From e18738c97ca8aee5d1afccb8e8c7bb9313b9dcc2 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:52:32 -0400 Subject: [PATCH 1/6] test(integration): add timeout golden coverage --- .github/workflows/integration_tests_local.yml | 3 + .../container/cjs/timeout-handler.js | 11 + integration_tests/container/cjs/timeout.js | 7 + integration_tests_local/README.md | 104 +- integration_tests_local/check-timeout-logs.js | 90 ++ .../check-timeout-logs.test.js | 121 ++ integration_tests_local/run.sh | 90 +- .../snapshots/logs/cjs-timeout.log | 1282 +++++++++++++++++ .../snapshots/logs/manual-timeout.log | 1255 ++++++++++++++++ .../snapshots/return_values/cjs-timeout.txt | 1 + .../return_values/manual-timeout.txt | 1 + 11 files changed, 2955 insertions(+), 10 deletions(-) create mode 100644 integration_tests/container/cjs/timeout-handler.js create mode 100644 integration_tests/container/cjs/timeout.js create mode 100644 integration_tests_local/check-timeout-logs.js create mode 100644 integration_tests_local/check-timeout-logs.test.js create mode 100644 integration_tests_local/snapshots/logs/cjs-timeout.log create mode 100644 integration_tests_local/snapshots/logs/manual-timeout.log create mode 100644 integration_tests_local/snapshots/return_values/cjs-timeout.txt create mode 100644 integration_tests_local/snapshots/return_values/manual-timeout.txt diff --git a/.github/workflows/integration_tests_local.yml b/.github/workflows/integration_tests_local.yml index bb246776..7900f9df 100644 --- a/.github/workflows/integration_tests_local.yml +++ b/.github/workflows/integration_tests_local.yml @@ -36,6 +36,9 @@ jobs: with: node-version: 18.12 + - name: Test timeout trace assertions + run: node --test integration_tests_local/check-timeout-logs.test.js + - name: Run local container integration tests (nodejs${{ matrix.node_major }}.x) run: PLATFORM=linux/amd64 RUNTIME_PARAM=${{ matrix.node_major }} ./integration_tests_local/run.sh diff --git a/integration_tests/container/cjs/timeout-handler.js b/integration_tests/container/cjs/timeout-handler.js new file mode 100644 index 00000000..5895c932 --- /dev/null +++ b/integration_tests/container/cjs/timeout-handler.js @@ -0,0 +1,11 @@ +const tracer = require("dd-trace"); + +// Both entry paths initialize dd-trace before loading this unwrapped handler. +// The child stays open past the real RIE deadline: only the timeout monitor's +// killAll() can flush it along with the error-tagged invocation span. +exports.handle = async function handle() { + return tracer.trace("timeout.unfinished", async () => { + await new Promise((resolve) => setTimeout(resolve, 60_000)); + return { statusCode: 200, body: "unreachable" }; + }); +}; diff --git a/integration_tests/container/cjs/timeout.js b/integration_tests/container/cjs/timeout.js new file mode 100644 index 00000000..3dcbc06a --- /dev/null +++ b/integration_tests/container/cjs/timeout.js @@ -0,0 +1,7 @@ +// Manual wrapping does not initialize the tracer. Load it first so its +// datadog-lambda-js hook installs the pre-migration timeout monitor. +require("dd-trace").init(); +const { datadog } = require("datadog-lambda-js"); +const { handle } = require("./timeout-handler"); + +module.exports.handle = datadog(handle); diff --git a/integration_tests_local/README.md b/integration_tests_local/README.md index c0079caf..fa1c0af9 100644 --- a/integration_tests_local/README.md +++ b/integration_tests_local/README.md @@ -1,7 +1,7 @@ # Local integration tests (docker + AWS RIE, no AWS account required) This directory contains a **local** integration test harness for -datadog-lambda-js. It runs eleven cases — container-image, layer-mode, and +datadog-lambda-js. It runs container-image, layer-mode, and manual-wrap handlers, plus targeted feature cases (HTTP header injection, custom trace extractors, proactive initialization) — inside Docker against the [AWS Lambda Runtime Interface Emulator (RIE)](https://github.com/aws/aws-lambda-runtime-interface-emulator), @@ -42,6 +42,10 @@ SKIP_PACK=true RUNTIME_PARAM=18 CASE_PARAM=container-esm ./integration_tests_loc # Force amd64 images instead of arm64 PLATFORM=linux/amd64 ./integration_tests_local/run.sh + +# Bypass a broken local VM host-port forward without changing the function. +# Runs the same HTTP requests using curl inside the target container. +RIE_HTTP_TRANSPORT=container RUNTIME_PARAM=22 CASE_PARAM=manual-timeout ./integration_tests_local/run.sh ``` On a tree that pins dd-trace v6 (which older runtimes cannot install), the @@ -62,6 +66,8 @@ The case names are: | `manual-send-metrics` | manual wrap calling `sendDistributionMetric` inside and outside the handler; per-event return values | | `manual-process-input` | manual wrap with userland `dd-trace` init reading the active span; per-event return values | | `manual-callback` | manual wrap of a callback-style `(event, context, callback)` handler; pins the `promisifiedHandler` seam end to end (the migration spike broke exactly this) | +| `manual-timeout` | explicit userland tracer init, then manual wrap; impending-timeout error on the invocation and `killAll()` flushing an unfinished child before RIE terminates the runtime | +| `cjs-timeout` | the same timeout contract through `DD_LAMBDA_HANDLER` and the npm redirect entrypoint, exercising the raw-handler hook also used by layers | | `manual-metrics-only` | `DD_TRACE_ENABLED=false` (metrics-only customers): enhanced + custom metrics still flush, no `aws.lambda` span, no trace JSON, no `dd.trace_id` log correlation | | `cjs-capture-payload` | `DD_CAPTURE_LAMBDA_PAYLOAD=true` in redirect mode; span meta gains `function.request` / `function.response` with the captured payloads | | `cjs-http-requests` | downstream HTTP calls against a hermetic mock server in redirect mode; asserts injected `x-datadog-*`/`traceparent` headers and log injection via dd-trace's http plugin | @@ -104,6 +110,7 @@ snapshots/logs/.log # shared across runtimes snapshots/return_values/default.json # return mode "default": every event, every case snapshots/return_values/.json # return mode "case": one payload for all 9 events snapshots/return_values/_.json # return mode "per-event": payload embeds event data +snapshots/return_values/.txt # RIE's plain-text timeout response ``` The normalized logs of most cases are identical across all five runtimes, so @@ -172,6 +179,99 @@ while `manual-http-requests` is manual-wrapped with no tracer (TraceListener falls back to the library's own `patchHttp`). See "Known emulation gaps" for what the manual case cannot pin locally. +## Impending timeouts (manual-timeout, cjs-timeout) + +These cases exercise both pre-migration dd-trace hook paths: manual +`datadog(handler)` wrapping and the `DD_LAMBDA_HANDLER` redirect. The manual +fixture initializes dd-trace **before** importing the shim; importing the shim +alone does not initialize tracing or install the timeout monitor. + +The handler opens a `timeout.unfinished` child span and waits 60 seconds. +`AWS_LAMBDA_FUNCTION_TIMEOUT=5` gives the invocation a real RIE deadline, and +`DD_APM_FLUSH_DEADLINE_MILLISECONDS=3500` makes dd-trace flush roughly 1.5 +seconds into the invocation (less the init time). RIE then kills the runtime +at five seconds. All nine input events are exercised, with a fresh runtime +after each timeout. The two cases add about 90 seconds per runtime leg. + +RIE v1.36 returns HTTP 200 with the **plain text** +`Task timed out after 5.00 seconds`, not Lambda's JSON error envelope. +Consequently, these return goldens use `.txt`. Completion is gated on REPORT, +timeout-reset, and SIGKILL records; a killed runtime never emits INVOKE RTDONE. +The HTTP client is bounded to 15 seconds so a broken deadline cannot hang CI. + +Before normalization or snapshot creation, `check-timeout-logs.js` reads all +trace payloads and asserts for every raw request ID: + +- exactly one `aws.lambda` span across all payloads, with `error=1`, + `error.type=Impending Timeout`, and the expected error message; +- exactly one unfinished child in that invocation's trace, with the invocation + as its parent and no error on the child; +- both spans were exported before REPORT, and the invocation span finished + within 2.5 seconds (the configured 1.5-second guard plus scheduling headroom, + which also rejects silently falling back to the default 100ms flush deadline); +- RIE actually reset and killed the runtime once per invocation. + +This catches duplicate invocation spans even when they appear in separate +traces, and prevents a metrics-only run from becoming a passing golden. The +helper preserves trace payloads unchanged. It normalizes only volatile IDs +and timestamps in RIE timeout diagnostics, including Go's varying log-field +order. Every diagnostic record and severity remains in the snapshot; the +shared AWS/RIE normalizer is unchanged. + +Run the helper's regression tests independently with: + +```bash +node --test integration_tests_local/check-timeout-logs.test.js +``` + +### Golden provenance + +The timeout goldens were captured from pre-migration library commit +`cf751a76003e9bdf18a4283bccdfa54785943b51` (`datadog-lambda-js` 12.142.0), +with no production-source changes. Both cases used RIE v1.36 on +`linux/arm64`, across Node 18/20/22/24/26, with dd-trace 5.105.0 — the version +`integration_tests/container/cjs/package.json` pins — on every runtime. The +per-runtime `DD_TRACE_VERSION` build-arg in `run.sh` is inert here: it only +applies when `scripts/dd_trace_versions.sh` exists, which is a v6-era file. +All five runtimes produced the same shared goldens; no runtime-specific +timeout overrides were needed. + +Both cases then passed a comparison-only rerun across all five runtimes +(90 invocations total), leaving all four timeout snapshot files byte-for-byte +unchanged. The existing `manual-throw-error` and `container-cjs` goldens also +passed unchanged on Node 22 through the same transport. The checker has nine +regression tests, verified on host Node 25 and container Node 18. + +An earlier capture used a different baseline, `4b9f41d2`, with dd-trace 5.126.0 +on Node 18/20 and 6.12.0 on Node 22/24/26. It passed on that baseline, with a +nonempty `dd_trace` tag and no `links` field. The goldens were recaptured for +the current branch and its 5.105.0 fixture pin: this build emits `links: []` +and an empty `dd_trace` tag (see below). + +When intentionally recapturing an existing shared golden, first review why +the baseline changed, then remove only the affected golden. `UPDATE_SNAPSHOTS=true` +deliberately refuses to overwrite an existing shared golden, so a divergent +runtime cannot silently replace the expectation from an earlier leg. + +**Known unrelated defect these goldens bake in.** The `dd_trace` span tag is +the empty string. `scripts/update_dist_version.sh` derives it with +`sed -n -E "s/dd-trace@([0-9]*\.[0-9]*\.[0-9]*):/\1/p" yarn.lock`, which +expects a pinned entry, but `yarn.lock` now records the range +`dd-trace@^5.113.0:` — so the substitution yields nothing. The committed +`container-cjs` golden shows the same empty tag, so this predates the timeout +work and is not caused by it. Worth fixing on its own, together with a recapture +of every golden carrying `dd_trace`. + +The local capture used `RIE_HTTP_TRANSPORT=container` because Colima's +published host ports were unreachable. RIE was reachable over IPv4 inside +the containers, so this was not an IPv6-only RIE listener. This transport +does not alter the handler, tracer, invocation body, or RIE timeout. + +These are L2 emulator goldens, not evidence of real AWS termination behavior +or layer packaging. The redirect case covers the raw-handler hook used by +layers; real AWS timeout behavior still needs L3 coverage. The current CI +workflow compares these goldens on native amd64; the local capture used arm64. + ## Proactive initialization (cjs-proactive-init) The library stamps `initTime = Date.now()` at wrapper-module load @@ -235,6 +335,8 @@ under test. A base-image change must be reviewed, not hidden by normalization. Reads stdin, writes stdout; honors `RUN_ID` for optional per-run ID stripping. - `prepare-layer.js` — assembles the layer fixture's build context from the repo build, mirroring the release Dockerfile's `/opt` layout +- `check-timeout-logs.js` — validates raw timeout traces and normalizes only + the timeout-specific RIE diagnostics before the shared normalizer - `bin/` — downloaded RIE binary (gitignored) - `snapshots/logs/` — normalized log snapshots, shared per case across runtimes, with optional `_node.log` overrides diff --git a/integration_tests_local/check-timeout-logs.js b/integration_tests_local/check-timeout-logs.js new file mode 100644 index 00000000..33741923 --- /dev/null +++ b/integration_tests_local/check-timeout-logs.js @@ -0,0 +1,90 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); + +// Check raw identities and complete trace payloads before the shared golden +// normalizer removes IDs or the runner sorts lines. A "some trace matches" +// assertion cannot catch two invocation spans exported in different payloads. +const expected = Number(process.argv[2]); +assert.ok(Number.isInteger(expected) && expected > 0, "expected invocation count is required"); +const raw = fs.readFileSync(0, "utf8"); +const lines = raw.trimEnd().split("\n"); +const starts = new Map(); +const reports = new Map(); +const spans = []; + +for (const [index, line] of lines.entries()) { + const start = /^START RequestId: (\S+)/.exec(line); + const report = /^REPORT RequestId: (\S+)/.exec(line); + if (start) { + assert.ok(!starts.has(start[1]), "duplicate START request ID"); + starts.set(start[1], index); + } + if (report) { + assert.ok(!reports.has(report[1]), "duplicate REPORT request ID"); + reports.set(report[1], index); + } + let record; + try { + record = JSON.parse(line); + } catch { + continue; + } + if (Array.isArray(record.traces)) { + for (const trace of record.traces) { + for (const span of trace) spans.push({ span, index }); + } + } +} + +assert.equal(starts.size, expected, "START count"); +assert.equal(reports.size, expected, "REPORT count"); +assert.equal(lines.filter((line) => line.includes("Reset initiated: Timeout")).length, expected, "timeout resets"); +assert.equal(lines.filter((line) => line.includes("Sending SIGKILL to runtime-")).length, expected, "runtime kills"); + +const invocations = spans.filter(({ span }) => span.name === "aws.lambda"); +const children = spans.filter(({ span }) => span.name === "timeout.unfinished"); +assert.equal(invocations.length, expected, "exactly one Lambda span per invocation across all trace payloads"); +assert.equal(children.length, expected, "killAll must flush every unfinished child"); +for (const [requestId, startIndex] of starts) { + const matching = invocations.filter(({ span }) => span.meta?.request_id === requestId); + assert.equal(matching.length, 1, `Lambda span count for ${requestId}`); + const { span, index } = matching[0]; + assert.equal(span.error, 1, "invocation must be marked as an error"); + assert.equal(span.meta["error.type"], "Impending Timeout"); + assert.equal(span.meta["error.message"], "Datadog detected an impending timeout"); + assert.ok(index > startIndex && index < reports.get(requestId), "span exported before runtime timeout REPORT"); + // The configured guard fires ~1.5s in, allowing 1s of scheduling overhead. + // Merely checking <5s would also accept a broken/default 100ms flush deadline. + assert.ok(span.duration > 0 && span.duration < 2.5e9, "span must honor the 3500ms flush deadline"); + const matchingChildren = children.filter( + ({ span: child }) => child.trace_id === span.trace_id && child.parent_id === span.span_id, + ); + assert.equal(matchingChildren.length, 1, "unfinished child must stay in its invocation's trace"); + const child = matchingChildren[0]; + assert.equal(child.span.error, 0, "timeout must decorate the invocation rather than the child"); + assert.ok( + child.index > startIndex && child.index < reports.get(requestId), + "child exported before runtime timeout REPORT", + ); +} + +// Only RIE's new timeout diagnostics need extra normalization. Keep every +// record, severity, and error value; replace volatile runtime/PID/request IDs +// and canonicalize Go's nondeterministic order of these two log fields. +process.stdout.write( + lines + .map((line) => { + if (!/^\d{2} \w{3} \d{4} .*\[(?:INFO|ERROR|WARNING)\] \(rapid\)/.test(line)) return line; + return line + .replace(/^\d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2},\d{3} /, "XXXX ") + .replace(/InvokeID=[0-9a-f-]+/g, "InvokeID=XXXX") + .replace(/runtime-\d+\(\d+\)/g, "runtime-XX(XX)") + .replace( + "Invoke failed error=errResetReceived InvokeID=XXXX", + "Invoke failed InvokeID=XXXX error=errResetReceived", + ); + }) + .join("\n") + "\n", +); diff --git a/integration_tests_local/check-timeout-logs.test.js b/integration_tests_local/check-timeout-logs.test.js new file mode 100644 index 00000000..f7d40deb --- /dev/null +++ b/integration_tests_local/check-timeout-logs.test.js @@ -0,0 +1,121 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const { spawnSync } = require("node:child_process"); +const path = require("node:path"); +const { test } = require("node:test"); + +function fixture() { + const invocation = { + name: "aws.lambda", + trace_id: "trace", + span_id: "lambda", + parent_id: "0", + error: 1, + duration: 1.5e9, + meta: { + request_id: "abc-def", + "error.type": "Impending Timeout", + "error.message": "Datadog detected an impending timeout", + }, + }; + const child = { + name: "timeout.unfinished", + trace_id: "trace", + span_id: "child", + parent_id: "lambda", + error: 0, + }; + return { invocation, child }; +} + +function rawLog(payloads, { late = false, kill = true } = {}) { + const records = payloads.map((spans) => JSON.stringify({ traces: [spans] })); + const report = "REPORT RequestId: abc-def Duration: 5000.00 ms"; + return ( + [ + "START RequestId: abc-def Version: $LATEST", + ...(late ? [report, ...records] : [...records, report]), + "25 Sep 2026 02:03:57,172 [ERROR] (rapid) Invoke failed error=errResetReceived InvokeID=abc-def", + "25 Sep 2026 02:03:57,173 [WARNING] (rapid) Reset initiated: Timeout", + ...(kill ? ["25 Sep 2026 02:03:57,173 [INFO] (rapid) Sending SIGKILL to runtime-1(28)."] : []), + ].join("\n") + "\n" + ); +} + +function check(input) { + return spawnSync(process.execPath, [path.join(__dirname, "check-timeout-logs.js"), "1"], { + input, + encoding: "utf8", + }); +} + +test("accepts complete exports and normalizes only volatile RIE diagnostics", () => { + const { invocation, child } = fixture(); + const result = check(rawLog([[invocation], [child]])); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.stdout.includes(JSON.stringify({ traces: [[invocation]] })), "trace payload unchanged"); + assert.ok(result.stdout.includes("XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived")); + assert.ok(result.stdout.includes("XXXX [WARNING] (rapid) Reset initiated: Timeout")); + assert.ok(result.stdout.includes("XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX).")); +}); + +test("rejects a second Lambda span in a separate trace payload", () => { + const { invocation, child } = fixture(); + const duplicate = { ...invocation, trace_id: "other-trace", span_id: "other-lambda" }; + const result = check(rawLog([[invocation, child], [duplicate]])); + assert.equal(result.status, 1); + assert.match(result.stderr, /exactly one Lambda span/); +}); + +test("rejects metrics-only logs without an invocation span", () => { + const result = check(rawLog([])); + assert.equal(result.status, 1); + assert.match(result.stderr, /exactly one Lambda span/); +}); + +test("rejects a missing unfinished child", () => { + const { invocation } = fixture(); + const result = check(rawLog([[invocation]])); + assert.equal(result.status, 1); + assert.match(result.stderr, /killAll must flush every unfinished child/); +}); + +test("rejects a child exported in the wrong trace", () => { + const { invocation, child } = fixture(); + child.trace_id = "unrelated-trace"; + const result = check(rawLog([[invocation, child]])); + assert.equal(result.status, 1); + assert.match(result.stderr, /unfinished child must stay/); +}); + +test("rejects timeout decoration on the child instead of the invocation", () => { + const { invocation, child } = fixture(); + invocation.error = 0; + child.error = 1; + const result = check(rawLog([[invocation, child]])); + assert.equal(result.status, 1); + assert.match(result.stderr, /invocation must be marked as an error/); +}); + +test("rejects an invocation exported after the runtime reports its timeout", () => { + const { invocation, child } = fixture(); + const result = check(rawLog([[invocation, child]], { late: true })); + assert.equal(result.status, 1); + assert.match(result.stderr, /span exported before runtime timeout REPORT/); +}); + +test("rejects falling back to the default flush deadline", () => { + const { invocation, child } = fixture(); + invocation.duration = 4.9e9; + const result = check(rawLog([[invocation, child]])); + assert.equal(result.status, 1); + assert.match(result.stderr, /span must honor the 3500ms flush deadline/); +}); + +test("requires evidence that RIE actually terminated the runtime", () => { + const { invocation, child } = fixture(); + const result = check(rawLog([[invocation, child]], { kill: false })); + assert.equal(result.status, 1); + assert.match(result.stderr, /runtime kills/); +}); diff --git a/integration_tests_local/run.sh b/integration_tests_local/run.sh index d821d1b9..ef2364e0 100755 --- a/integration_tests_local/run.sh +++ b/integration_tests_local/run.sh @@ -23,6 +23,8 @@ # manual-send-metrics | cjs | send-metrics.handle | manual wrap; sendDistributionMetric via DD_FLUSH_TO_LOG # manual-process-input | cjs | process-input.handle | manual wrap; dd-trace child spans via tracer.wrap # manual-callback | cjs | callback.handle | manual wrap; callback-style (event, context, callback) handler — the spike's proven break seam +# manual-timeout | cjs | timeout.handle | manual wrap; impending timeout -> aws.lambda span tagged error.type=Impending Timeout, flushed before the runtime kills the invoke +# cjs-timeout | cjs | node_modules/datadog-lambda-js/dist/handler.handler | DD_LAMBDA_HANDLER path; same timeout and unfinished-child flush contract # cjs-http-requests | cjs | node_modules/datadog-lambda-js/dist/handler.handler | npm redirect; downstream HTTP header injection via dd-trace's http plugin (hermetic mock server) # manual-http-requests | cjs | http-requests-manual.handle | manual wrap; patchHttp fallback wrapping + per-request logging (hermetic mock) # cjs-fetch-requests | cjs | node_modules/datadog-lambda-js/dist/handler.handler | npm redirect; header injection on global fetch via dd-trace's undici plugin (hermetic mock) @@ -43,6 +45,8 @@ # VARIANT_PARAM - legacy alias: cjs -> container-cjs, esm -> container-esm # SIMULATE_PROACTIVE_INIT - legacy alias for CASE_PARAM=cjs-proactive-init # PLATFORM - docker platform (default: linux/arm64) +# RIE_HTTP_TRANSPORT - host (default) or container; container uses docker exec +# when the local VM cannot forward published ports # UPDATE_SNAPSHOTS=true - overwrite local snapshots instead of diffing # SKIP_PACK=true - reuse existing container/*/datadog-lambda-js-local.tgz # and the existing layer fixture context @@ -63,6 +67,11 @@ repo_dir=$(dirname "$local_dir") integration_tests_dir="$repo_dir/integration_tests" PLATFORM=${PLATFORM:-linux/arm64} +RIE_HTTP_TRANSPORT=${RIE_HTTP_TRANSPORT:-host} +case "$RIE_HTTP_TRANSPORT" in + host|container) ;; + *) echo "Unsupported RIE_HTTP_TRANSPORT: $RIE_HTTP_TRANSPORT (use host or container)"; exit 1 ;; +esac RIE_VERSION=1.36 case "$PLATFORM" in linux/arm64) @@ -116,6 +125,8 @@ ALL_CASES=( "manual-send-metrics" "manual-process-input" "manual-callback" + "manual-timeout" + "cjs-timeout" "cjs-http-requests" "manual-http-requests" "cjs-fetch-requests" @@ -134,6 +145,9 @@ function configure_case() { case_expect_error=false case_proactive=false case_needs_mock=false + case_invoke_timeout=0 + case_timeout=false + case_return_extension=json # Return-value golden shape: # default - every event returns the shared default.json payload # case - every event returns one case-specific payload (error bodies) @@ -202,6 +216,27 @@ function configure_case() { case_image=cjs case_entry_handler="callback.handle" ;; + manual-timeout|cjs-timeout) + # dd-trace's pre-migration monitor flushes the invocation and an + # unfinished child ~1.5s in; RIE kills the runtime at 5s. Exercise + # both hook branches: the shim export and DD_LAMBDA_HANDLER. + case_image=cjs + case_entry_handler="timeout.handle" + case_extra_env=( + -e AWS_LAMBDA_FUNCTION_TIMEOUT=5 + -e DD_APM_FLUSH_DEADLINE_MILLISECONDS=3500 + ) + if [ "$case_name" = cjs-timeout ]; then + case_entry_handler="node_modules/datadog-lambda-js/dist/handler.handler" + case_extra_env+=(-e DD_LAMBDA_HANDLER=timeout-handler.handle) + fi + # RIE v1.36 returns a plain-text timeout body, not an error JSON + # envelope, and emits Reset/SIGKILL instead of INVOKE RTDONE. + case_timeout=true + case_return_mode=case + case_return_extension=txt + case_invoke_timeout=15 + ;; manual-metrics-only) # DD_TRACE_ENABLED=false — metrics-only customers. The golden pins # that enhanced + custom metrics still flush (via DD_FLUSH_TO_LOG) @@ -428,6 +463,18 @@ function prepare_layer_context() { # golden shows the injected downstream trace context. mock_script='const http=require("http");http.createServer((req,res)=>{res.setHeader("content-type","application/json");res.end(JSON.stringify({url:req.url,headers:req.headers}));}).listen(8080);' mock_started=false +function container_http() { + local target_cid=$1 + local host_port=$2 + local resource=$3 + shift 3 + if [ "$RIE_HTTP_TRANSPORT" = container ]; then + docker exec -i "$target_cid" curl -s "$@" "http://127.0.0.1:8080$resource" + else + curl -s "$@" "http://127.0.0.1:$host_port$resource" + fi +} + function start_mock() { local base_image=$1 if [ "$mock_started" = true ]; then @@ -446,7 +493,7 @@ function start_mock() { # Desktop port proxy before the backend is reachable, which then # drops the next real request ("empty reply"). Any HTTP response — # the mock 200s every path — proves the full path works. - if curl -s -o /dev/null --max-time 2 "http://localhost:$port/"; then + if container_http "$mock_cid" "$port" / -o /dev/null --max-time 2; then ready=true break fi @@ -629,7 +676,7 @@ for node_version in "${RUNTIMES[@]}"; do # exit status proves the whole proxy + HTTP path. ready=false for i in $(seq 1 60); do - if curl -s -o /dev/null --max-time 2 "http://localhost:$port/"; then + if container_http "$cid" "$port" / -o /dev/null --max-time 2; then ready=true break fi @@ -653,9 +700,9 @@ for node_version in "${RUNTIMES[@]}"; do for input_event_file in "${input_event_files[@]}"; do input_event_name=$(echo "$input_event_file" | sed "s/.json//") # curl does not fail on HTTP errors, so validate the response status. - invoke_response=$(curl -s -w '\n%{http_code}' -XPOST \ - "http://localhost:$port/2015-03-31/functions/function/invocations" \ - -d @"$integration_tests_dir/input_events/$input_event_file") + invoke_response=$(container_http "$cid" "$port" /2015-03-31/functions/function/invocations \ + --max-time "$case_invoke_timeout" -w '\n%{http_code}' -XPOST \ + --data-binary @- < "$integration_tests_dir/input_events/$input_event_file") invoke_success=$? http_code=$(printf '%s\n' "$invoke_response" | tail -n1) return_value=$(printf '%s\n' "$invoke_response" | sed '$d') @@ -684,6 +731,11 @@ for node_version in "${RUNTIMES[@]}"; do continue fi fi + if [ "$case_timeout" = true ] && [ "$return_value" != "Task timed out after 5.00 seconds" ]; then + echo "Failed: expected the RIE timeout body for $handler_name with $input_event_name, got: $return_value" + mismatch_found=true + continue + fi echo " $input_event_name -> $return_value" # Return-value goldens resolve from most to least specific; the @@ -709,7 +761,7 @@ for node_version in "${RUNTIMES[@]}"; do case "$case_return_mode" in case) # Same payload for every event (e.g. an error body). - return_snapshot="$local_dir/snapshots/return_values/${case_name}.json" + return_snapshot="$local_dir/snapshots/return_values/${case_name}.${case_return_extension}" ;; per-event) # Payload embeds event data (record ids, request ids), @@ -746,7 +798,7 @@ for node_version in "${RUNTIMES[@]}"; do # The managed-instances path used by the proactive-init case emits # REPORT but never RTDONE, so requiring it there would always time out. expected_invocation_count=${#input_event_files[@]} - if [ "$case_proactive" = true ]; then + if [ "$case_proactive" = true ] || [ "$case_timeout" = true ]; then expected_rtdone_count=0 else expected_rtdone_count=$expected_invocation_count @@ -757,8 +809,16 @@ for node_version in "${RUNTIMES[@]}"; do raw_logs=$(docker logs "$cid" 2>&1) report_count=$(printf '%s\n' "$raw_logs" | grep -c '^REPORT RequestId:' || true) rtdone_count=$(printf '%s\n' "$raw_logs" | grep -c 'INVOKE RTDONE' || true) + timeout_logs_ready=true + if [ "$case_timeout" = true ]; then + reset_count=$(printf '%s\n' "$raw_logs" | grep -c 'Reset initiated: Timeout' || true) + kill_count=$(printf '%s\n' "$raw_logs" | grep -c 'Sending SIGKILL to runtime-' || true) + if [ "$reset_count" -lt "$expected_invocation_count" ] || [ "$kill_count" -lt "$expected_invocation_count" ]; then + timeout_logs_ready=false + fi + fi if [ "$report_count" -ge "$expected_invocation_count" ] && \ - [ "$rtdone_count" -ge "$expected_rtdone_count" ]; then + [ "$rtdone_count" -ge "$expected_rtdone_count" ] && [ "$timeout_logs_ready" = true ]; then logs_ready=true break fi @@ -790,7 +850,19 @@ for node_version in "${RUNTIMES[@]}"; do done fi - logs=$(printf '%s\n' "$raw_logs" | "$repo_dir/scripts/normalize_integration_logs.sh" rie) + snapshot_logs=$raw_logs + if [ "$case_timeout" = true ]; then + snapshot_logs=$(printf '%s\n' "$raw_logs" | node "$local_dir/check-timeout-logs.js" "$expected_invocation_count") + if [ "$?" -ne 0 ]; then + echo "FAILURE: timeout trace assertions failed for $function_name" >&2 + mismatch_found=true + docker rm -f "$cid" >/dev/null 2>&1 + container_ids=("${container_ids[@]/$cid}") + continue + fi + echo "Ok: exactly one timeout-tagged Lambda span and one flushed child per invocation" + fi + logs=$(printf '%s\n' "$snapshot_logs" | "$repo_dir/scripts/normalize_integration_logs.sh" rie) # `runtime:nodejsNN.x` is the only genuinely runtime-specific line in # the whole log — everything else is identical across 18/20/22/24/26. diff --git a/integration_tests_local/snapshots/logs/cjs-timeout.log b/integration_tests_local/snapshots/logs/cjs-timeout.log new file mode 100644 index 00000000..1bad8aab --- /dev/null +++ b/integration_tests_local/snapshots/logs/cjs-timeout.log @@ -0,0 +1,1282 @@ +XXXX [INFO] (rapid) exec '/lambda-entrypoint.sh' (cwd=/var/task, handler=node_modules/datadog-lambda-js/dist/handler.handler) +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: init) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.apigateway", + "resource": "GET /{proxy+}", + "service": "remappedApiGatewayServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedApiGatewayServiceName", + "runtime-id":"XXXX", + "http.url": "https://undefined", + "resource_names": "GET /{proxy+}", + "request_id":"XXXX", + "span.kind": "server", + "apiid":"XXXX", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "sync", + "http.method": "GET", + "stage": "test", + "domain_name": "", + "dd_resource_key": "arn:aws:apigateway:us-east-1::/restapis/wt6mne2s9k", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "api-gateway", + "function_trigger.event_source_arn": "arn:aws:apigateway:us-east-1::/restapis/wt6mne2s9k/stages/test", + "http.method": "GET", + "http.route": "/{proxy+}", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +END RequestId: XXXX +REPORT RequestId: XXXX Init Duration: XXXX ms Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.dynamodb", + "resource": "MODIFY someTableName", + "service": "remappedDynamoDbServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedDynamoDbServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.dynamodb", + "tablename": "someTableName", + "resource_names": "MODIFY someTableName", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "event_name": "MODIFY", + "event_version": "1.1", + "event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "event_id": "0123456789abcdef09123456789abcdef", + "stream_view_type": "KEYS_ONLY", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "size_bytes": 37, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "dynamodb", + "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.dynamodb", + "resource": "REMOVE someTableName", + "service": "remappedDynamoDbServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedDynamoDbServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.dynamodb", + "tablename": "someTableName", + "resource_names": "REMOVE someTableName", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "event_name": "REMOVE", + "event_version": "1.1", + "event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "event_id": "0123456789abcdef09123456789abcdef", + "stream_view_type": "KEYS_ONLY", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "size_bytes": 37, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "dynamodb", + "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.dynamodb", + "resource": "INSERT someTableName", + "service": "remappedDynamoDbServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedDynamoDbServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.dynamodb", + "tablename": "someTableName", + "resource_names": "INSERT someTableName", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "event_name": "INSERT", + "event_version": "1.1", + "event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "event_id": "0123456789abcdef09123456789abcdef", + "stream_view_type": "KEYS_ONLY", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "size_bytes": 37, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "dynamodb", + "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.s3", + "resource": "my-bucket-name", + "service": "remappedS3ServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedS3ServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.s3", + "resource_names": "my-bucket-name", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "bucketname": "my-bucket-name", + "bucket_arn": "arn:aws:s3:::my-bucket-name", + "event_name": "ObjectCreated:CompleteMultipartUpload", + "object_key": "multipart_object.txt", + "object_etag": "09876543210987654321098765432109-2", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "object_size": 8388608, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "s3", + "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.s3", + "resource": "my-bucket-name", + "service": "remappedS3ServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedS3ServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.s3", + "resource_names": "my-bucket-name", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "bucketname": "my-bucket-name", + "bucket_arn": "arn:aws:s3:::my-bucket-name", + "event_name": "ObjectCreated:Copy", + "object_key": "copied_object.txt", + "object_etag": "01234567890123456789012345678901", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "object_size": 100, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "s3", + "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.s3", + "resource": "my-bucket-name", + "service": "remappedS3ServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedS3ServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.s3", + "resource_names": "my-bucket-name", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "bucketname": "my-bucket-name", + "bucket_arn": "arn:aws:s3:::my-bucket-name", + "event_name": "ObjectCreated:Put", + "object_key": "test_object.txt", + "object_etag": "abcdef0123456789abcdef01234567890", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "object_size": 100, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "s3", + "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.sns", + "resource": "sns-lambda", + "service": "remappedSnsServiceName", + "type": "sns", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedSnsServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.sns", + "resource_names": "sns-lambda", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "type": "Notification", + "subject": "TestInvoke", + "message_id": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", + "topicname": "sns-lambda", + "topic_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", + "event_subscription_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda:21be56ed-a058-49f5-8c98-aedd2564c486", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "sns", + "function_trigger.event_source_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-cjs-timeout", + "resource:integration-tests-js-local-cjs-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.sqs", + "resource": "my-queue", + "service": "remappedSqsServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-cjs-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "_dd.origin": "lambda", + "service": "remappedSqsServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.sqs", + "resource_names": "my-queue", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "queuename": "my-queue", + "event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", + "receipt_handle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", + "sender_id": "AIDAIENQZJOLO23YVJ4VO", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-cjs-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "retry_count": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-cjs-timeout", + "service": "integration-tests-js-local-cjs-timeout", + "type": "serverless", + "error": 1, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-cjs-timeout", + "functionname": "integration-tests-js-local-cjs-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "sqs", + "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-cjs-timeout", + "error": 0, + "meta": { + "_dd.origin": "lambda", + "version": "1.0.0", + "service": "integration-tests-js-local-cjs-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination diff --git a/integration_tests_local/snapshots/logs/manual-timeout.log b/integration_tests_local/snapshots/logs/manual-timeout.log new file mode 100644 index 00000000..ee981cc2 --- /dev/null +++ b/integration_tests_local/snapshots/logs/manual-timeout.log @@ -0,0 +1,1255 @@ +XXXX [INFO] (rapid) exec '/lambda-entrypoint.sh' (cwd=/var/task, handler=timeout.handle) +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: init) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.apigateway", + "resource": "GET /{proxy+}", + "service": "remappedApiGatewayServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedApiGatewayServiceName", + "runtime-id":"XXXX", + "http.url": "https://undefined", + "resource_names": "GET /{proxy+}", + "request_id":"XXXX", + "span.kind": "server", + "apiid":"XXXX", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "sync", + "http.method": "GET", + "stage": "test", + "domain_name": "", + "dd_resource_key": "arn:aws:apigateway:us-east-1::/restapis/wt6mne2s9k", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "api-gateway", + "function_trigger.event_source_arn": "arn:aws:apigateway:us-east-1::/restapis/wt6mne2s9k/stages/test", + "http.method": "GET", + "http.route": "/{proxy+}", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +END RequestId: XXXX +REPORT RequestId: XXXX Init Duration: XXXX ms Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.dynamodb", + "resource": "MODIFY someTableName", + "service": "remappedDynamoDbServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedDynamoDbServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.dynamodb", + "tablename": "someTableName", + "resource_names": "MODIFY someTableName", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "event_name": "MODIFY", + "event_version": "1.1", + "event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "event_id": "0123456789abcdef09123456789abcdef", + "stream_view_type": "KEYS_ONLY", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "size_bytes": 37, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "dynamodb", + "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.dynamodb", + "resource": "REMOVE someTableName", + "service": "remappedDynamoDbServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedDynamoDbServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.dynamodb", + "tablename": "someTableName", + "resource_names": "REMOVE someTableName", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "event_name": "REMOVE", + "event_version": "1.1", + "event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "event_id": "0123456789abcdef09123456789abcdef", + "stream_view_type": "KEYS_ONLY", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "size_bytes": 37, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "dynamodb", + "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.dynamodb", + "resource": "INSERT someTableName", + "service": "remappedDynamoDbServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedDynamoDbServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.dynamodb", + "tablename": "someTableName", + "resource_names": "INSERT someTableName", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "event_name": "INSERT", + "event_version": "1.1", + "event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "event_id": "0123456789abcdef09123456789abcdef", + "stream_view_type": "KEYS_ONLY", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "size_bytes": 37, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "dynamodb", + "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.s3", + "resource": "my-bucket-name", + "service": "remappedS3ServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedS3ServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.s3", + "resource_names": "my-bucket-name", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "bucketname": "my-bucket-name", + "bucket_arn": "arn:aws:s3:::my-bucket-name", + "event_name": "ObjectCreated:CompleteMultipartUpload", + "object_key": "multipart_object.txt", + "object_etag": "09876543210987654321098765432109-2", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "object_size": 8388608, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "s3", + "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.s3", + "resource": "my-bucket-name", + "service": "remappedS3ServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedS3ServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.s3", + "resource_names": "my-bucket-name", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "bucketname": "my-bucket-name", + "bucket_arn": "arn:aws:s3:::my-bucket-name", + "event_name": "ObjectCreated:Copy", + "object_key": "copied_object.txt", + "object_etag": "01234567890123456789012345678901", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "object_size": 100, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "s3", + "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.s3", + "resource": "my-bucket-name", + "service": "remappedS3ServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedS3ServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.s3", + "resource_names": "my-bucket-name", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "bucketname": "my-bucket-name", + "bucket_arn": "arn:aws:s3:::my-bucket-name", + "event_name": "ObjectCreated:Put", + "object_key": "test_object.txt", + "object_etag": "abcdef0123456789abcdef01234567890", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "object_size": 100, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "s3", + "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.sns", + "resource": "sns-lambda", + "service": "remappedSnsServiceName", + "type": "sns", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedSnsServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.sns", + "resource_names": "sns-lambda", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "type": "Notification", + "subject": "TestInvoke", + "message_id": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", + "topicname": "sns-lambda", + "topic_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", + "event_subscription_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda:21be56ed-a058-49f5-8c98-aedd2564c486", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "sns", + "function_trigger.event_source_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false +XXXX [INFO] (rapid) INIT RTDONE(status: success) +XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) +XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) +{ + "e": XXXX, + "m": "aws.lambda.enhanced.invocations", + "t": [ + "region:us-east-1", + "account_id:XXXX", + "functionname:integration-tests-js-local-manual-timeout", + "resource:integration-tests-js-local-manual-timeout", + "memorysize:3008", + "cold_start:true", + "datadog_lambda:vX.X.X", + "runtime:nodejsXX.x" + ], + "v": 1 +} +{ + "traces": [ + [ + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.sqs", + "resource": "my-queue", + "service": "remappedSqsServiceName", + "type": "web", + "error": 0, + "meta": { + "_dd.tags.process": "entrypoint.type:script,entrypoint.workdir:task,package.json.name:container-cjs-test,svc.auto:integration-tests-js-local-manual-timeout", + "_dd.p.tid": "XXXX", + "_dd.p.dm": "-0", + "service": "remappedSqsServiceName", + "runtime-id":"XXXX", + "operation_name": "aws.sqs", + "resource_names": "my-queue", + "request_id":"XXXX", + "span.kind": "server", + "_inferred_span.tag_source": "self", + "_inferred_span.synchronicity": "async", + "queuename": "my-queue", + "event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", + "receipt_handle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", + "sender_id": "AIDAIENQZJOLO23YVJ4VO", + "_dd.integration": "opentracing", + "_dd.svc_src": "m", + "_dd.base_service": "integration-tests-js-local-manual-timeout", + "language": "javascript" + }, + "metrics": { + "_dd.agent_psr": 1, + "_dd.top_level": 1, + "_dd.measured": 1, + "retry_count": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "aws.lambda", + "resource": "integration-tests-js-local-manual-timeout", + "service": "integration-tests-js-local-manual-timeout", + "type": "serverless", + "error": 1, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "span.kind": "server", + "cold_start": "true", + "function_arn":"XXXX", + "function_version": "$LATEST", + "request_id":"XXXX", + "resource_names": "integration-tests-js-local-manual-timeout", + "functionname": "integration-tests-js-local-manual-timeout", + "datadog_lambda":"XXXX", + "dd_trace": "", + "function_trigger.event_source": "sqs", + "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", + "error.message": "Datadog detected an impending timeout", + "error.type": "Impending Timeout", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + }, + { + "trace_id":"XXXX", + "span_id":"XXXX", + "parent_id":"XXXX", + "name": "timeout.unfinished", + "resource": "timeout.unfinished", + "service": "integration-tests-js-local-manual-timeout", + "error": 0, + "meta": { + "version": "1.0.0", + "service": "integration-tests-js-local-manual-timeout", + "runtime-id":"XXXX", + "_dd.integration": "opentracing", + "language": "javascript" + }, + "metrics": { + "_dd.measured": 1, + "process_id":XXXX, + "_sampling_priority_v1": 1 + }, + "start":XXXX, + "duration":XXXX, + "links": [] + } + ] + ] +} +XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived +XXXX [WARNING] (rapid) Reset initiated: Timeout +XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB diff --git a/integration_tests_local/snapshots/return_values/cjs-timeout.txt b/integration_tests_local/snapshots/return_values/cjs-timeout.txt new file mode 100644 index 00000000..a0f172e4 --- /dev/null +++ b/integration_tests_local/snapshots/return_values/cjs-timeout.txt @@ -0,0 +1 @@ +Task timed out after 5.00 seconds diff --git a/integration_tests_local/snapshots/return_values/manual-timeout.txt b/integration_tests_local/snapshots/return_values/manual-timeout.txt new file mode 100644 index 00000000..a0f172e4 --- /dev/null +++ b/integration_tests_local/snapshots/return_values/manual-timeout.txt @@ -0,0 +1 @@ +Task timed out after 5.00 seconds From 3750f6abf779d1f9cff06c6ca34b7a399ef07907 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:11:46 -0400 Subject: [PATCH 2/6] ci(integration): resolve timeout workflow conflict with main --- .github/workflows/integration_tests_local.yml | 37 +++++++++++++++---- integration_tests_local/README.md | 5 ++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration_tests_local.yml b/.github/workflows/integration_tests_local.yml index 7900f9df..3eb84561 100644 --- a/.github/workflows/integration_tests_local.yml +++ b/.github/workflows/integration_tests_local.yml @@ -4,8 +4,8 @@ name: local integration tests # against the real AWS Lambda base images + the Runtime Interface Emulator # (RIE). No AWS account or credentials required. # -# One job per Node major version; each job builds the container/layer fixture -# images and runs every case (default: all of them) against that runtime. +# One job per Node major version and CPU architecture; each job builds the +# container/layer fixture images and runs every case against that runtime. # The fixtures share the same ~1 GB base image, so each job pulls it exactly # once. If public.ecr.aws anonymous pull limits ever bite, the fix is to # mirror the base images to GHCR (same pattern dd-trace-js uses in its @@ -20,27 +20,50 @@ on: jobs: container: - runs-on: ubuntu-latest + name: nodejs${{ matrix.node_major }}.x (${{ matrix.architecture }}) + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: node_major: ["18", "20", "22", "24", "26"] + architecture: ["amd64", "arm64"] + include: + - architecture: "amd64" + platform: "linux/amd64" + runner: "ubuntu-latest" + - architecture: "arm64" + platform: "linux/arm64" + runner: "ubuntu-24.04-arm" + # Host Node must match the Lambda runtime so yarn installs the + # matching dd-trace line (v5 on 18/20, v6 on 22+) with working + # native prebuilds. Packing v6 on Node 18 (--ignore-engines) is + # what local Node 22+ never hits. + - node_major: "18" + node_version: "18.12" + - node_major: "20" + node_version: "20.19" + - node_major: "22" + node_version: "22.11" + - node_major: "24" + node_version: "24.11" + - node_major: "26" + node_version: "26.1" steps: - name: Checkout uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - name: Set up Node 18.12 + - name: Set up Node ${{ matrix.node_version }} uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1 with: - node-version: 18.12 + node-version: ${{ matrix.node_version }} - name: Test timeout trace assertions run: node --test integration_tests_local/check-timeout-logs.test.js - - name: Run local container integration tests (nodejs${{ matrix.node_major }}.x) - run: PLATFORM=linux/amd64 RUNTIME_PARAM=${{ matrix.node_major }} ./integration_tests_local/run.sh + - name: Run local container integration tests (nodejs${{ matrix.node_major }}.x, ${{ matrix.architecture }}) + run: PLATFORM=${{ matrix.platform }} RUNTIME_PARAM=${{ matrix.node_major }} ./integration_tests_local/run.sh # Every leg is a strict comparison, so a run must leave the snapshots # untouched. Writing here would mean the harness recorded its own output diff --git a/integration_tests_local/README.md b/integration_tests_local/README.md index fa1c0af9..e5ddf74a 100644 --- a/integration_tests_local/README.md +++ b/integration_tests_local/README.md @@ -269,8 +269,9 @@ does not alter the handler, tracer, invocation body, or RIE timeout. These are L2 emulator goldens, not evidence of real AWS termination behavior or layer packaging. The redirect case covers the raw-handler hook used by -layers; real AWS timeout behavior still needs L3 coverage. The current CI -workflow compares these goldens on native amd64; the local capture used arm64. +layers; real AWS timeout behavior still needs L3 coverage. The CI workflow is +configured to compare these goldens on native amd64 and arm64; the local capture +used arm64. ## Proactive initialization (cjs-proactive-init) From 8785aeeeb9fea310214d5368dc8a8842f4a87544 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:34:44 -0400 Subject: [PATCH 3/6] test(integration): refresh timeout goldens after main merge --- integration_tests_local/README.md | 66 +++++----- .../snapshots/logs/cjs-timeout.log | 115 +++++++----------- .../snapshots/logs/manual-timeout.log | 115 +++++++----------- 3 files changed, 116 insertions(+), 180 deletions(-) diff --git a/integration_tests_local/README.md b/integration_tests_local/README.md index 3b012258..9c5e8408 100644 --- a/integration_tests_local/README.md +++ b/integration_tests_local/README.md @@ -166,10 +166,11 @@ filter to absorb it would be invisible. Current overrides: assertions — the proactive-initialization markers — are grep-checked on the raw logs and are identical on every runtime. -In update mode, a leg that disagrees with an existing shared golden **fails** -instead of overwriting it — otherwise the last runtime to run would silently -define the expectation for all of them. To capture a genuine per-runtime -divergence, `touch` the override file first so the write targets it. +In update mode, the first leg writes the shared golden. Later legs in the +same run **fail** if they disagree instead of overwriting it — otherwise the +last runtime to run would silently define the expectation for all of them. +To capture a genuine per-runtime divergence, `touch` the override file first +so the write targets it. ## The mock HTTP server (cjs-http-requests, manual-http-requests) @@ -234,32 +235,29 @@ node --test integration_tests_local/check-timeout-logs.test.js ### Golden provenance -The timeout goldens were captured from pre-migration library commit -`cf751a76003e9bdf18a4283bccdfa54785943b51` (`datadog-lambda-js` 12.142.0), -with no production-source changes. Both cases used RIE v1.36 on -`linux/arm64`, across Node 18/20/22/24/26, with dd-trace 5.105.0 — the version -`integration_tests/container/cjs/package.json` pinned at that commit — on every -runtime. The per-runtime `DD_TRACE_VERSION` build-arg was inert on that baseline -because it did not yet have `scripts/dd_trace_versions.sh`. -All five runtimes produced the same shared goldens; no runtime-specific -timeout overrides were needed. - -The branch has since incorporated current main to resolve the workflow conflict. -Main uses the v5/v6 runtime split and runtime tracer-version reporting. These -timeout goldens still record the older baseline above and need recapture and -strict comparison against the updated baseline before this PR is ready. - -Both cases then passed a comparison-only rerun across all five runtimes -(90 invocations total), leaving all four timeout snapshot files byte-for-byte -unchanged. The existing `manual-throw-error` and `container-cjs` goldens also -passed unchanged on Node 22 through the same transport. The checker has nine -regression tests, verified on host Node 25 and container Node 18. - -An earlier capture used a different baseline, `4b9f41d2`, with dd-trace 5.126.0 -on Node 18/20 and 6.12.0 on Node 22/24/26. It passed on that baseline, with a -nonempty `dd_trace` tag and no `links` field. The goldens were recaptured for -the `cf751a76` baseline and its 5.105.0 fixture pin: that build emits `links: []` -and an empty `dd_trace` tag (see below). +The timeout goldens were recaptured after merging main, from pre-migration +library commit `291fd14e9b8c54b94ba5b57998f726231839dc92` +(`datadog-lambda-js` 12.143.0), with no additional production-source changes. +Both cases used RIE v1.36 on `linux/arm64`, across Node 18/20/22/24/26. The +fixture runner installed dd-trace 5.126.0 on Node 18/20 and the lockfile-resolved +6.15.0 on Node 22/24/26. All five runtimes produced the same shared goldens; +no runtime-specific timeout overrides were needed. + +Both cases passed a comparison-only rerun across all five runtimes (90 +invocations), leaving all four timeout snapshot files byte-for-byte unchanged. + +The existing `manual-throw-error` and `container-cjs` goldens also passed +unchanged on Node 22 through the same transport. The checker has nine +regression tests. + +The previous files recorded the older `cf751a76` baseline with dd-trace +5.105.0 on every runtime. Those snapshots stopped matching after the main +merge: the current tracer omits empty `links: []` fields, and the shim resolves +the loaded tracer's version at runtime instead of recording an empty +`dd_trace` tag. The recapture changes only those fields and ordinary RIE +record ordering under the existing line-order comparison. Timeout error +decoration, span counts, parent/child relationships, and flush checks remain +unchanged. Neither the raw checker nor the shared normalizer was loosened. When intentionally recapturing an existing shared golden, first review why the baseline changed. The current runner lets the first leg overwrite a shared @@ -267,14 +265,6 @@ golden in `UPDATE_SNAPSHOTS=true` mode and requires later legs in that run to agree. Unlike the older capture baseline, it no longer requires deleting the existing golden first. -**Historical capture-baseline defect.** These goldens record an empty `dd_trace` -span tag. At `cf751a76`, `scripts/update_dist_version.sh` derived it with -`sed -n -E "s/dd-trace@([0-9]*\.[0-9]*\.[0-9]*):/\1/p" yarn.lock`, which -expected a pinned entry, but `yarn.lock` recorded the range -`dd-trace@^5.113.0:` — so the substitution yielded nothing. The `container-cjs` -golden at that baseline had the same empty tag. Main has since fixed version -reporting; the pending timeout recapture must reflect that fix. - The local capture used `RIE_HTTP_TRANSPORT=container` because Colima's published host ports were unreachable. RIE was reachable over IPv4 inside the containers, so this was not an IPv6-only RIE listener. This transport diff --git a/integration_tests_local/snapshots/logs/cjs-timeout.log b/integration_tests_local/snapshots/logs/cjs-timeout.log index 1bad8aab..98d23561 100644 --- a/integration_tests_local/snapshots/logs/cjs-timeout.log +++ b/integration_tests_local/snapshots/logs/cjs-timeout.log @@ -64,8 +64,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -89,7 +88,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "api-gateway", "function_trigger.event_source_arn": "arn:aws:apigateway:us-east-1::/restapis/wt6mne2s9k/stages/test", "http.method": "GET", @@ -105,8 +104,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -130,8 +128,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -139,9 +136,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). -XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Init Duration: XXXX ms Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -209,8 +206,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -234,7 +230,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "dynamodb", "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", "error.message": "Datadog detected an impending timeout", @@ -248,8 +244,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -273,8 +268,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -282,9 +276,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB -XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -352,8 +346,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -377,7 +370,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "dynamodb", "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", "error.message": "Datadog detected an impending timeout", @@ -391,8 +384,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -416,8 +408,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -425,9 +416,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB -XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -495,8 +486,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -520,7 +510,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "dynamodb", "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", "error.message": "Datadog detected an impending timeout", @@ -534,8 +524,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -559,8 +548,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -569,10 +557,10 @@ XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB XXXX [INFO] (rapid) Waiting for runtime domain processes termination -XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB START RequestId: XXXX Version: $LATEST +XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false XXXX [INFO] (rapid) INIT RTDONE(status: success) @@ -637,8 +625,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -662,7 +649,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "s3", "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", "error.message": "Datadog detected an impending timeout", @@ -676,8 +663,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -701,8 +687,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -710,9 +695,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB -XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -779,8 +764,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -804,7 +788,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "s3", "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", "error.message": "Datadog detected an impending timeout", @@ -818,8 +802,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -843,8 +826,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -921,8 +903,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -946,7 +927,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "s3", "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", "error.message": "Datadog detected an impending timeout", @@ -960,8 +941,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -985,8 +965,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -1063,8 +1042,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1088,7 +1066,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "sns", "function_trigger.event_source_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", "error.message": "Datadog detected an impending timeout", @@ -1102,8 +1080,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1127,8 +1104,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -1136,9 +1112,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). -XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -1204,8 +1180,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1229,7 +1204,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-cjs-timeout", "functionname": "integration-tests-js-local-cjs-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", "error.message": "Datadog detected an impending timeout", @@ -1243,8 +1218,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1268,8 +1242,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -1277,6 +1250,6 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB -XXXX [INFO] (rapid) Waiting for runtime domain processes termination diff --git a/integration_tests_local/snapshots/logs/manual-timeout.log b/integration_tests_local/snapshots/logs/manual-timeout.log index ee981cc2..a3580052 100644 --- a/integration_tests_local/snapshots/logs/manual-timeout.log +++ b/integration_tests_local/snapshots/logs/manual-timeout.log @@ -1,8 +1,8 @@ XXXX [INFO] (rapid) exec '/lambda-entrypoint.sh' (cwd=/var/task, handler=timeout.handle) +START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: init) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false -START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT RTDONE(status: success) XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) @@ -63,8 +63,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -87,7 +86,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "api-gateway", "function_trigger.event_source_arn": "arn:aws:apigateway:us-east-1::/restapis/wt6mne2s9k/stages/test", "http.method": "GET", @@ -103,8 +102,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -127,8 +125,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -136,9 +133,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). -XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Init Duration: XXXX ms Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB +XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -205,8 +202,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -229,7 +225,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "dynamodb", "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", "error.message": "Datadog detected an impending timeout", @@ -243,8 +239,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -267,8 +262,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -276,9 +270,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB -XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -345,8 +339,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -369,7 +362,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "dynamodb", "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", "error.message": "Datadog detected an impending timeout", @@ -383,8 +376,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -407,8 +399,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -416,12 +407,12 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB -XXXX [INFO] (rapid) Waiting for runtime domain processes termination -START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. +START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) Starting runtime without AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN , Expected?: false XXXX [INFO] (rapid) INIT RTDONE(status: success) XXXX [INFO] (rapid) INIT REPORT(durationMs: XXXX) @@ -485,8 +476,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -509,7 +499,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "dynamodb", "function_trigger.event_source_arn": "arn:aws:dynamodb:us-east-1:1234567890:table/someTableName/stream/2024-12-11T20:00:00.000", "error.message": "Datadog detected an impending timeout", @@ -523,8 +513,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -547,8 +536,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -556,8 +544,8 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). -END RequestId: XXXX XXXX [INFO] (rapid) Waiting for runtime domain processes termination +END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) @@ -624,8 +612,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -648,7 +635,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "s3", "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", "error.message": "Datadog detected an impending timeout", @@ -662,8 +649,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -686,8 +672,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -696,8 +681,8 @@ XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB XXXX [INFO] (rapid) Waiting for runtime domain processes termination +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -763,8 +748,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -787,7 +771,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "s3", "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", "error.message": "Datadog detected an impending timeout", @@ -801,8 +785,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -825,8 +808,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -902,8 +884,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -926,7 +907,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "s3", "function_trigger.event_source_arn": "arn:aws:s3:::my-bucket-name", "error.message": "Datadog detected an impending timeout", @@ -940,8 +921,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -964,8 +944,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -973,9 +952,9 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) XXXX [ERROR] (rapid) Invoke failed InvokeID=XXXX error=errResetReceived XXXX [WARNING] (rapid) Reset initiated: Timeout XXXX [INFO] (rapid) Sending SIGKILL to runtime-XX(XX). +XXXX [INFO] (rapid) Waiting for runtime domain processes termination END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 3008 MB Max Memory Used: XXXX MB -XXXX [INFO] (rapid) Waiting for runtime domain processes termination START RequestId: XXXX Version: $LATEST XXXX [INFO] (rapid) INIT START(type: on-demand, phase: invoke) XXXX [INFO] (rapid) The extension's directory "/opt/extensions" does not exist, assuming no extensions to be loaded. @@ -1041,8 +1020,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1065,7 +1043,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "sns", "function_trigger.event_source_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", "error.message": "Datadog detected an impending timeout", @@ -1079,8 +1057,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1103,8 +1080,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] @@ -1179,8 +1155,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1203,7 +1178,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "resource_names": "integration-tests-js-local-manual-timeout", "functionname": "integration-tests-js-local-manual-timeout", "datadog_lambda":"XXXX", - "dd_trace": "", + "dd_trace":"XXXX", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", "error.message": "Datadog detected an impending timeout", @@ -1217,8 +1192,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX }, { "trace_id":"XXXX", @@ -1241,8 +1215,7 @@ XXXX [INFO] (rapid) INVOKE START(requestId: XXXX) "_sampling_priority_v1": 1 }, "start":XXXX, - "duration":XXXX, - "links": [] + "duration":XXXX } ] ] From 85fdb2ee2368e4ef03268277e01b9e99119ca4c1 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:00:03 -0400 Subject: [PATCH 4/6] test(integration): pin Lambda runtime images and refresh goldens --- integration_tests_local/README.md | 29 ++++++++++++++++++- integration_tests_local/run.sh | 23 ++++++++++----- .../snapshots/logs/cjs-proactive-init.log | 18 +++++++++--- .../logs/cjs-proactive-init_node24.log | 26 ++++++++++++----- .../manual-throw-error_node24.json | 2 +- 5 files changed, 76 insertions(+), 22 deletions(-) diff --git a/integration_tests_local/README.md b/integration_tests_local/README.md index 9c5e8408..2ed37009 100644 --- a/integration_tests_local/README.md +++ b/integration_tests_local/README.md @@ -305,6 +305,33 @@ the three markers: ## Pinned runtime infrastructure +All five Lambda base images are pinned to multi-architecture manifest digests +in `run.sh`'s `lambda_node_image_tag()`. The same reference is used by the CJS, +ESM, layer, and mock-server fixtures, with `PLATFORM` selecting amd64 or arm64. +The pins come from [CI run 36168116285](https://github.com/DataDog/datadog-lambda-js/actions/runs/36168116285) +on September 25, 2026. A warm local Docker cache and a clean CI runner must not +silently test different runtime releases under the same major-version tag. + +That discrepancy caused two baseline changes with no library-source changes: +the newer Node 22/24 managed runtime emits a structured +`runtime_worker_pool_initializing` DEBUG record, and Node 24 now includes +`requestId` in its thrown-error response. The corresponding proactive-init log +goldens and Node 24 error-return golden were recaptured from the pre-migration +library at `8785aeee`, using the pinned images on arm64. The runtime record is +preserved in full, including `workerCount: 4` (the harness specifies `--cpus 4`) +and `executionEnvironmentMaxConcurrency: 1`; the response retains `requestId` +with its existing volatile-value normalization. The shared normalizer and the +three raw proactive-init assertions are unchanged. + +After recapture, all 18 cases passed in comparison-only mode on both Node 22 +and Node 24 / arm64 (324 invocations). The three refreshed expectations kept +the same hashes, and the timeout goldens were not changed by this refresh. + +To update a runtime, change its digest in `lambda_node_image_tag()`, inspect +the runtime differences, and recapture only the affected expectations before +running comparison-only tests on both architectures. Do not remove runtime +records to make a new base image match an older golden. + The harness pins AWS Runtime Interface Emulator (RIE) `v1.36` and verifies the cached binary on every run before mounting it into a container: @@ -320,7 +347,7 @@ Node 26 is still preview-only in ECR Public: the bare `public.ecr.aws/lambda/nodejs:26` tag does not exist. The logical runtime stays `26` for image names, function names, and snapshot paths, while the Docker base-image build argument maps to the dated multi-arch tag -`26-preview.2026.08.21.22`. +`26-preview.2026.08.21.22` plus its manifest digest. Node 26 is a strict leg like every other; where its preview runtime genuinely diverges (error stack frames, warning emission) it carries diff --git a/integration_tests_local/run.sh b/integration_tests_local/run.sh index 19725e46..d65207e5 100755 --- a/integration_tests_local/run.sh +++ b/integration_tests_local/run.sh @@ -187,8 +187,8 @@ function configure_case() { case_return_mode=case # Stack frames carry line:col that shift with fixture edits and # node-internal frames whose line numbers vary by Node major. - # The node26 preview RIC embeds a per-invocation requestId in the - # error body; strip it wherever it appears. + # Node 24 and the Node 26 preview RIC embed a per-invocation requestId + # in the error body; retain the field and normalize its value. case_return_filter='s/:[0-9]+:[0-9]+([\)"])/:XXX:XXX\1/g; s/"requestId":"[0-9a-f-]+"/"requestId":"XXXX"/g' ;; manual-status-500) @@ -514,11 +514,18 @@ input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_nam set +e # Don't exit this script if an invocation fails or there's a diff function lambda_node_image_tag() { - if [ "$1" = "26" ]; then - echo "26-preview.2026.08.21.22" - else - echo "$1" - fi + # Multi-architecture manifest digests keep local caches and clean CI runners + # on the same runtime. Floating major tags can change logs and error bodies + # independently of the library under test. Review snapshot changes when + # intentionally updating these pins; do not normalize runtime changes away. + case "$1" in + 18) echo "18@sha256:daf6a5c0a2b36153b94c91f3563e8ef89b3b19e4129963c6ccb07b8c5251f7ef" ;; + 20) echo "20@sha256:a4440274d6f0fb4e6cb92cd5f2a97254efe6612f631d9974b197ecbd4a61fcab" ;; + 22) echo "22@sha256:1922086069c2effeb6b5c3f3b839d393c21dadb28df5bea0920cdb97a71451bd" ;; + 24) echo "24@sha256:1c7e718a6973b27b31cd3535ac58537b242c932427bbf4ca72b27bc14977b656" ;; + 26) echo "26-preview.2026.08.21.22@sha256:b9d6953f685b42667843f9bbc4f77d4711749c93df8fef984baa44bfdea89195" ;; + *) echo "Unsupported Lambda Node image: $1" >&2; return 1 ;; + esac } function compare_snapshot() { @@ -595,7 +602,7 @@ function write_snapshot() { } for node_version in "${RUNTIMES[@]}"; do - node_image_tag=$(lambda_node_image_tag "$node_version") + node_image_tag=$(lambda_node_image_tag "$node_version") || exit 1 # Resolve both tracer lines to exact versions. The v5 compatibility pin is # maintained explicitly; the v6 pin follows the root Yarn lockfile, so the # regular dependency-update workflow updates the RIE fixtures as well. diff --git a/integration_tests_local/snapshots/logs/cjs-proactive-init.log b/integration_tests_local/snapshots/logs/cjs-proactive-init.log index 303d92fa..87c5f53e 100644 --- a/integration_tests_local/snapshots/logs/cjs-proactive-init.log +++ b/integration_tests_local/snapshots/logs/cjs-proactive-init.log @@ -1,6 +1,16 @@ time=XXXX-XX-XXTXX:XX:XX.XXXZ level=INFO msg="executing bootstrap" command=/lambda-entrypoint.sh time=XXXX-XX-XXTXX:XX:XX.XXXZ level=INFO msg="The extension's directory does not exist, assuming no extensions to be loaded" fullDir=/opt/extensions time=XXXX-XX-XXTXX:XX:XX.XXXZ level=INFO msg=LocalProcessSupervisor.Exec pid=XX path=/var/runtime/bootstrap args=[/var/runtime/bootstrap] +{ + "timestamp": "XXXX-XX-XXTXX:XX:XX.XXXZ", + "level": "DEBUG", + "requestId": "XXXX", + "message": { + "event": "runtime_worker_pool_initializing", + "workerCount": 4, + "executionEnvironmentMaxConcurrency": 1 + } +} START RequestId: XXXX Version: $LATEST { "e": XXXX, @@ -135,6 +145,8 @@ START RequestId: XXXX Version: $LATEST "requestId": "XXXX", "message": "[dd.trace_id=XXXX dd.span_id=XXXX] (node:XX) TimeoutOverflowWarning: XXXX does not fit into a 32-bit signed integer.\nTimeout duration was set to 1." } +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms { "traces": [ [ @@ -222,8 +234,6 @@ START RequestId: XXXX Version: $LATEST ] ] } -END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms START RequestId: XXXX Version: $LATEST { "e": XXXX, @@ -246,6 +256,8 @@ START RequestId: XXXX Version: $LATEST "requestId": "XXXX", "message": "[dd.trace_id=XXXX dd.span_id=XXXX] (node:XX) TimeoutOverflowWarning: XXXX does not fit into a 32-bit signed integer.\nTimeout duration was set to 1." } +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms { "traces": [ [ @@ -333,8 +345,6 @@ START RequestId: XXXX Version: $LATEST ] ] } -END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms START RequestId: XXXX Version: $LATEST { "e": XXXX, diff --git a/integration_tests_local/snapshots/logs/cjs-proactive-init_node24.log b/integration_tests_local/snapshots/logs/cjs-proactive-init_node24.log index 303d92fa..f596d2a6 100644 --- a/integration_tests_local/snapshots/logs/cjs-proactive-init_node24.log +++ b/integration_tests_local/snapshots/logs/cjs-proactive-init_node24.log @@ -1,6 +1,16 @@ time=XXXX-XX-XXTXX:XX:XX.XXXZ level=INFO msg="executing bootstrap" command=/lambda-entrypoint.sh time=XXXX-XX-XXTXX:XX:XX.XXXZ level=INFO msg="The extension's directory does not exist, assuming no extensions to be loaded" fullDir=/opt/extensions time=XXXX-XX-XXTXX:XX:XX.XXXZ level=INFO msg=LocalProcessSupervisor.Exec pid=XX path=/var/runtime/bootstrap args=[/var/runtime/bootstrap] +{ + "timestamp": "XXXX-XX-XXTXX:XX:XX.XXXZ", + "level": "DEBUG", + "requestId": "XXXX", + "message": { + "event": "runtime_worker_pool_initializing", + "workerCount": 4, + "executionEnvironmentMaxConcurrency": 1 + } +} START RequestId: XXXX Version: $LATEST { "e": XXXX, @@ -135,6 +145,8 @@ START RequestId: XXXX Version: $LATEST "requestId": "XXXX", "message": "[dd.trace_id=XXXX dd.span_id=XXXX] (node:XX) TimeoutOverflowWarning: XXXX does not fit into a 32-bit signed integer.\nTimeout duration was set to 1." } +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms { "traces": [ [ @@ -222,8 +234,6 @@ START RequestId: XXXX Version: $LATEST ] ] } -END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms START RequestId: XXXX Version: $LATEST { "e": XXXX, @@ -578,6 +588,8 @@ START RequestId: XXXX Version: $LATEST "requestId": "XXXX", "message": "[dd.trace_id=XXXX dd.span_id=XXXX] (node:XX) TimeoutOverflowWarning: XXXX does not fit into a 32-bit signed integer.\nTimeout duration was set to 1." } +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms { "traces": [ [ @@ -664,8 +676,6 @@ START RequestId: XXXX Version: $LATEST ] ] } -END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms START RequestId: XXXX Version: $LATEST { "e": XXXX, @@ -688,6 +698,8 @@ START RequestId: XXXX Version: $LATEST "requestId": "XXXX", "message": "[dd.trace_id=XXXX dd.span_id=XXXX] (node:XX) TimeoutOverflowWarning: XXXX does not fit into a 32-bit signed integer.\nTimeout duration was set to 1." } +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms { "traces": [ [ @@ -774,8 +786,6 @@ START RequestId: XXXX Version: $LATEST ] ] } -END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms START RequestId: XXXX Version: $LATEST { "e": XXXX, @@ -798,6 +808,8 @@ START RequestId: XXXX Version: $LATEST "requestId": "XXXX", "message": "[dd.trace_id=XXXX dd.span_id=XXXX] (node:XX) TimeoutOverflowWarning: XXXX does not fit into a 32-bit signed integer.\nTimeout duration was set to 1." } +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms { "traces": [ [ @@ -883,8 +895,6 @@ START RequestId: XXXX Version: $LATEST ] ] } -END RequestId: XXXX -REPORT RequestId: XXXX Duration: XXXX ms START RequestId: XXXX Version: $LATEST { "e": XXXX, diff --git a/integration_tests_local/snapshots/return_values/manual-throw-error_node24.json b/integration_tests_local/snapshots/return_values/manual-throw-error_node24.json index 997d8531..89615645 100644 --- a/integration_tests_local/snapshots/return_values/manual-throw-error_node24.json +++ b/integration_tests_local/snapshots/return_values/manual-throw-error_node24.json @@ -1 +1 @@ -{"errorType":"Error","errorMessage":"Hello","trace":["Error: Hello"," at handle (/var/task/throw-error.js:XXX:XXX)"," at /var/task/node_modules/datadog-lambda-js/dist/utils/handler.js:XXX:XXX"," at traceListenerOnWrap (/var/task/node_modules/datadog-lambda-js/dist/index.js:XXX:XXX)"," at BufferedInvokeProcessor.wrappedFunc [as handler] (/var/task/node_modules/datadog-lambda-js/dist/index.js:XXX:XXX)"," at async BufferedInvokeProcessor.processInvoke (file:///var/runtime/index.mjs:XXX:XXX)"," at async _Runtime.processSingleConcurrent (file:///var/runtime/index.mjs:XXX:XXX)"," at async _Runtime.start (file:///var/runtime/index.mjs:XXX:XXX)"," at async ignition (file:///var/runtime/index.mjs:XXX:XXX)"]} +{"errorType":"Error","errorMessage":"Hello","trace":["Error: Hello"," at handle (/var/task/throw-error.js:XXX:XXX)"," at /var/task/node_modules/datadog-lambda-js/dist/utils/handler.js:XXX:XXX"," at traceListenerOnWrap (/var/task/node_modules/datadog-lambda-js/dist/index.js:XXX:XXX)"," at BufferedInvokeProcessor.wrappedFunc [as handler] (/var/task/node_modules/datadog-lambda-js/dist/index.js:XXX:XXX)"," at async BufferedInvokeProcessor.processInvoke (file:///var/runtime/index.mjs:XXX:XXX)"," at async _Runtime.processSingleConcurrent (file:///var/runtime/index.mjs:XXX:XXX)"," at async _Runtime.start (file:///var/runtime/index.mjs:XXX:XXX)"," at async ignition (file:///var/runtime/index.mjs:XXX:XXX)"],"requestId":"XXXX"} From e66c4b67c429f10e15f2cbec46d5e01f4f25fe52 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:23:29 -0400 Subject: [PATCH 5/6] update lower bound --- integration_tests_local/check-timeout-logs.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integration_tests_local/check-timeout-logs.js b/integration_tests_local/check-timeout-logs.js index 33741923..2ce30aa1 100644 --- a/integration_tests_local/check-timeout-logs.js +++ b/integration_tests_local/check-timeout-logs.js @@ -56,8 +56,11 @@ for (const [requestId, startIndex] of starts) { assert.equal(span.meta["error.message"], "Datadog detected an impending timeout"); assert.ok(index > startIndex && index < reports.get(requestId), "span exported before runtime timeout REPORT"); // The configured guard fires ~1.5s in, allowing 1s of scheduling overhead. - // Merely checking <5s would also accept a broken/default 100ms flush deadline. - assert.ok(span.duration > 0 && span.duration < 2.5e9, "span must honor the 3500ms flush deadline"); + // Expected around 1.5s; allow generous tolerance while rejecting immediate firing. + assert.ok( + span.duration >= 0.5e9 && span.duration < 2.5e9, + `timeout span duration outside expected window: ${span.duration / 1e9}s`, + ); const matchingChildren = children.filter( ({ span: child }) => child.trace_id === span.trace_id && child.parent_id === span.span_id, ); From da9fc3c0c9dca00893f70c9a1bd5ac1254af3b01 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:36:23 -0400 Subject: [PATCH 6/6] wording --- integration_tests_local/check-timeout-logs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests_local/check-timeout-logs.js b/integration_tests_local/check-timeout-logs.js index 2ce30aa1..6bf75006 100644 --- a/integration_tests_local/check-timeout-logs.js +++ b/integration_tests_local/check-timeout-logs.js @@ -59,7 +59,7 @@ for (const [requestId, startIndex] of starts) { // Expected around 1.5s; allow generous tolerance while rejecting immediate firing. assert.ok( span.duration >= 0.5e9 && span.duration < 2.5e9, - `timeout span duration outside expected window: ${span.duration / 1e9}s`, + `span must honor the 3500ms flush deadline: got ${span.duration / 1e9}s`, ); const matchingChildren = children.filter( ({ span: child }) => child.trace_id === span.trace_id && child.parent_id === span.span_id,