From 431707bf4cdecbaceaf70b1195b699a35ec9fd5c Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Thu, 2 May 2024 13:15:00 -0400 Subject: [PATCH 1/8] emit enhanced error metric and create span when an exception is raised outside of the handler function --- src/handler.cjs | 8 +++++- src/handler.mjs | 19 ++++++++++++-- src/index.ts | 38 ++++++++++++++++++++++++++++ src/metrics/enhanced-metrics.spec.ts | 6 ++--- src/metrics/enhanced-metrics.ts | 19 +++++++++----- 5 files changed, 77 insertions(+), 13 deletions(-) diff --git a/src/handler.cjs b/src/handler.cjs index 641aea93..6b391cc9 100644 --- a/src/handler.cjs +++ b/src/handler.cjs @@ -3,6 +3,7 @@ const { datadogHandlerEnvVar, lambdaTaskRootEnvVar, traceExtractorEnvVar, + emitTelemetryOnErrorOutsideHandler, getEnvValue, } = require("./index.js"); const { logDebug, logError } = require("./utils/index.js"); @@ -32,4 +33,9 @@ if (extractorEnv) { } } -exports.handler = datadog(loadSync(taskRootEnv, handlerEnv), { traceExtractor }); +try { + exports.handler = datadog(loadSync(taskRootEnv, handlerEnv), { traceExtractor }); +} catch (error) { + emitTelemetryOnErrorOutsideHandler(error, handlerEnv, Date.now()); + throw error; +} diff --git a/src/handler.mjs b/src/handler.mjs index 2db66c2d..a89ff6ac 100644 --- a/src/handler.mjs +++ b/src/handler.mjs @@ -1,4 +1,11 @@ -import { datadog, datadogHandlerEnvVar, lambdaTaskRootEnvVar, traceExtractorEnvVar, getEnvValue } from "./index.js"; +import { + datadog, + datadogHandlerEnvVar, + lambdaTaskRootEnvVar, + traceExtractorEnvVar, + getEnvValue, + emitTelemetryOnErrorOutsideHandler, +} from "./index.js"; import { logDebug, logError } from "./utils/index.js"; import { load } from "./runtime/index.js"; import { initTracer } from "./runtime/module_importer.js"; @@ -26,4 +33,12 @@ if (extractorEnv) { } } -export const handler = datadog(await load(taskRootEnv, handlerEnv), { traceExtractor }); +let wrapped_handler; +try { + wrapped_handler = datadog(await load(taskRootEnv, handlerEnv), { traceExtractor }); +} catch (error) { + emitTelemetryOnErrorOutsideHandler(error, handlerEnv, Date.now()); + throw error; +} + +export const handler = wrapped_handler; diff --git a/src/index.ts b/src/index.ts index 5678ba3e..3be40adf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,8 @@ import { } from "./utils"; import { getEnhancedMetricTags } from "./metrics/enhanced-metrics"; import { DatadogTraceHeaders } from "./trace/context/extractor"; +import { SpanWrapper } from "./trace/span-wrapper"; +import { SpanOptions, TracerWrapper } from "./trace/tracer-wrapper"; // Backwards-compatible export, TODO deprecate in next major export { DatadogTraceHeaders as TraceHeaders } from "./trace/context/extractor"; @@ -416,3 +418,39 @@ function getRuntimeTag(): string { const version = process.version; return `dd_lambda_layer:datadog-node${version}`; } + +export async function emitTelemetryOnErrorOutsideHandler( + error: Error, + functionName: string, + startTime: number, +): Promise { + const config = getConfig(); + const metricsListener = new MetricsListener(new KMSService(), config); + await metricsListener.onStartInvocation(undefined); + if (config.enhancedMetrics) { + incrementErrorsMetric(metricsListener); + } + + if (getEnvValue("DD_TRACE_ENABLED", "true").toLowerCase() === "true") { + const options: SpanOptions = { + tags: { + service: "aws.lambda", + operation_name: "aws.lambda", + resource_names: functionName, + "resource.name": functionName, + "span.type": "serverless", + "error.status": 500, + "error.type": error.name, + "error.message": error.message, + "error.stack": error.stack, + status: error, + }, + startTime, + }; + const tracerWrapper = new TracerWrapper(); + const span = new SpanWrapper(tracerWrapper.startSpan("aws.lambda", options), {}); + span.finish(); + } + + await metricsListener.onCompleteInvocation(); +} diff --git a/src/metrics/enhanced-metrics.spec.ts b/src/metrics/enhanced-metrics.spec.ts index b3bddf5b..6c83e898 100644 --- a/src/metrics/enhanced-metrics.spec.ts +++ b/src/metrics/enhanced-metrics.spec.ts @@ -55,8 +55,8 @@ describe("getEnhancedMetricTags", () => { "account_id:123497598159", "functionname:my-test-lambda", "resource:my-test-lambda", - "cold_start:true", "memorysize:128", + "cold_start:true", "datadog_lambda:vX.X.X", "runtime:nodejs20.x", ]); @@ -66,8 +66,8 @@ describe("getEnhancedMetricTags", () => { mockedGetProcessVersion.mockReturnValue("v20.9.0"); expect(getEnhancedMetricTags(mockContextLocal)).toStrictEqual([ "functionname:my-test-lambda", - "cold_start:true", "memorysize:128", + "cold_start:true", "datadog_lambda:vX.X.X", "runtime:nodejs20.x", ]); @@ -80,8 +80,8 @@ describe("getEnhancedMetricTags", () => { "account_id:123497598159", "functionname:my-test-lambda", "resource:my-test-lambda", - "cold_start:true", "memorysize:128", + "cold_start:true", "datadog_lambda:vX.X.X", ]); }); diff --git a/src/metrics/enhanced-metrics.ts b/src/metrics/enhanced-metrics.ts index 2ce1b96f..fb5cf143 100644 --- a/src/metrics/enhanced-metrics.ts +++ b/src/metrics/enhanced-metrics.ts @@ -49,12 +49,17 @@ export function getRuntimeTag(): string | null { return `runtime:${processVersionTagString}`; } -export function getEnhancedMetricTags(context: Context): string[] { - let arnTags = [`functionname:${context.functionName}`]; - if (context.invokedFunctionArn) { - arnTags = parseTagsFromARN(context.invokedFunctionArn, context.functionVersion); +export function getEnhancedMetricTags(context?: Context): string[] { + let tags: string[] = []; + if (context) { + let arnTags = [`functionname:${context.functionName}`]; + if (context.invokedFunctionArn) { + arnTags = parseTagsFromARN(context.invokedFunctionArn, context.functionVersion); + } + tags = [...arnTags, `memorysize:${context.memoryLimitInMB}`]; } - const tags = [...arnTags, ...getSandboxInitTags(), `memorysize:${context.memoryLimitInMB}`, getVersionTag()]; + + tags.push(...getSandboxInitTags(), getVersionTag()); const runtimeTag = getRuntimeTag(); if (runtimeTag) { @@ -69,7 +74,7 @@ export function getEnhancedMetricTags(context: Context): string[] { * @param context object passed to invocation by AWS * @param metricName name of the enhanced metric without namespace prefix, i.e. "invocations" or "errors" */ -function incrementEnhancedMetric(listener: MetricsListener, metricName: string, context: Context) { +function incrementEnhancedMetric(listener: MetricsListener, metricName: string, context?: Context) { // Always write enhanced metrics to standard out listener.sendDistributionMetric(`aws.lambda.enhanced.${metricName}`, 1, true, ...getEnhancedMetricTags(context)); } @@ -78,7 +83,7 @@ export function incrementInvocationsMetric(listener: MetricsListener, context: C incrementEnhancedMetric(listener, "invocations", context); } -export function incrementErrorsMetric(listener: MetricsListener, context: Context): void { +export function incrementErrorsMetric(listener: MetricsListener, context?: Context): void { incrementEnhancedMetric(listener, "errors", context); } From 02e9ae7e1f5000dba69df4c0dd2fd2295ba0a42a Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Mon, 6 May 2024 21:07:10 -0400 Subject: [PATCH 2/8] add unit tests --- src/index.spec.ts | 58 +++++++++++++++++++++++++++++++++++++++++++++++ src/index.ts | 1 - 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/index.spec.ts b/src/index.spec.ts index 1143aa18..31028c84 100644 --- a/src/index.spec.ts +++ b/src/index.spec.ts @@ -8,6 +8,7 @@ import { sendDistributionMetric, sendDistributionMetricWithDate, _metricsQueue, + emitTelemetryOnErrorOutsideHandler, } from "./index"; import { incrementErrorsMetric, @@ -21,6 +22,8 @@ import { DatadogTraceHeaders } from "./trace/context/extractor"; import { SpanContextWrapper } from "./trace/span-context-wrapper"; import { TraceSource } from "./trace/trace-context-service"; import { inflateSync } from "zlib"; +import { MetricsListener } from "./metrics/listener"; +import { SpanOptions, TracerWrapper } from "./trace/tracer-wrapper"; jest.mock("./metrics/enhanced-metrics"); @@ -536,3 +539,58 @@ describe("sendDistributionMetricWithDate", () => { expect(_metricsQueue.length).toBe(1); }); }); + +describe("emitTelemetryOnErrorOutsideHandler", () => { + let mockedStartSpan = jest.spyOn(TracerWrapper.prototype, "startSpan"); + beforeEach(() => { + jest.spyOn(MetricsListener.prototype, "onStartInvocation").mockImplementation(); + jest.spyOn(TracerWrapper.prototype, "isTracerAvailable", "get").mockImplementation(() => true); + }); + afterEach(() => { + mockedIncrementErrors.mockClear(); + mockedStartSpan.mockClear(); + }); + it("emits a metric when enhanced metrics are enabled", async () => { + process.env.DD_ENHANCED_METRICS = "true"; + await emitTelemetryOnErrorOutsideHandler(new ReferenceError("some error"), "myFunction", Date.now()); + expect(mockedIncrementErrors).toBeCalledTimes(1); + }); + + it("does not emit a metric when enhanced metrics are disabled", async () => { + process.env.DD_ENHANCED_METRICS = "false"; + await emitTelemetryOnErrorOutsideHandler(new ReferenceError("some error"), "myFunction", Date.now()); + expect(mockedIncrementErrors).toBeCalledTimes(0); + }); + + it("creates a span when tracing is enabled", async () => { + process.env.DD_TRACE_ENABLED = "true"; + const functionName = "myFunction"; + const startTime = Date.now(); + const fakeError = new ReferenceError("some error"); + const spanName = "aws.lambda"; + + await emitTelemetryOnErrorOutsideHandler(fakeError, functionName, startTime); + + const options: SpanOptions = { + tags: { + service: spanName, + operation_name: spanName, + resource_names: functionName, + "resource.name": functionName, + "span.type": "serverless", + "error.status": 500, + "error.type": fakeError.name, + "error.message": fakeError.message, + "error.stack": fakeError.stack, + }, + startTime, + }; + expect(mockedStartSpan).toBeCalledWith(spanName, options); + }); + + it("does not create a span when tracing is disabled", async () => { + process.env.DD_TRACE_ENABLED = "false"; + await emitTelemetryOnErrorOutsideHandler(new ReferenceError("some error"), "myFunction", Date.now()); + expect(mockedStartSpan).toBeCalledTimes(0); + }); +}); diff --git a/src/index.ts b/src/index.ts index 3be40adf..09c420f4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -443,7 +443,6 @@ export async function emitTelemetryOnErrorOutsideHandler( "error.type": error.name, "error.message": error.message, "error.stack": error.stack, - status: error, }, startTime, }; From 050662c7680626b5b58fc7f3ecb679f4341d69ec Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Tue, 7 May 2024 11:31:35 -0400 Subject: [PATCH 3/8] add missing await and move metrics block --- src/handler.mjs | 2 +- src/index.ts | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/handler.mjs b/src/handler.mjs index a89ff6ac..facd4e22 100644 --- a/src/handler.mjs +++ b/src/handler.mjs @@ -37,7 +37,7 @@ let wrapped_handler; try { wrapped_handler = datadog(await load(taskRootEnv, handlerEnv), { traceExtractor }); } catch (error) { - emitTelemetryOnErrorOutsideHandler(error, handlerEnv, Date.now()); + await emitTelemetryOnErrorOutsideHandler(error, handlerEnv, Date.now()); throw error; } diff --git a/src/index.ts b/src/index.ts index 09c420f4..388d47bd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -424,13 +424,6 @@ export async function emitTelemetryOnErrorOutsideHandler( functionName: string, startTime: number, ): Promise { - const config = getConfig(); - const metricsListener = new MetricsListener(new KMSService(), config); - await metricsListener.onStartInvocation(undefined); - if (config.enhancedMetrics) { - incrementErrorsMetric(metricsListener); - } - if (getEnvValue("DD_TRACE_ENABLED", "true").toLowerCase() === "true") { const options: SpanOptions = { tags: { @@ -451,5 +444,11 @@ export async function emitTelemetryOnErrorOutsideHandler( span.finish(); } + const config = getConfig(); + const metricsListener = new MetricsListener(new KMSService(), config); + await metricsListener.onStartInvocation(undefined); + if (config.enhancedMetrics) { + incrementErrorsMetric(metricsListener); + } await metricsListener.onCompleteInvocation(); } From 4b441d37fd3d10ab227a2ea12a9cfd2819293918 Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Tue, 7 May 2024 13:48:20 -0400 Subject: [PATCH 4/8] test getEnhancedMetricTags with no context --- src/metrics/enhanced-metrics.spec.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/metrics/enhanced-metrics.spec.ts b/src/metrics/enhanced-metrics.spec.ts index 6c83e898..230852b3 100644 --- a/src/metrics/enhanced-metrics.spec.ts +++ b/src/metrics/enhanced-metrics.spec.ts @@ -85,4 +85,9 @@ describe("getEnhancedMetricTags", () => { "datadog_lambda:vX.X.X", ]); }); + + it("doesn't add context-based tags when context not provided", () => { + mockedGetProcessVersion.mockReturnValue("v20.9.0"); + expect(getEnhancedMetricTags()).toStrictEqual(["cold_start:true", "datadog_lambda:vX.X.X", "runtime:nodejs20.x"]); + }); }); From 56f21e42eb521b007d690b5f444a74fbd70de9d9 Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Tue, 7 May 2024 14:52:28 -0400 Subject: [PATCH 5/8] fix casing --- src/handler.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handler.mjs b/src/handler.mjs index facd4e22..8e6ea3db 100644 --- a/src/handler.mjs +++ b/src/handler.mjs @@ -33,12 +33,12 @@ if (extractorEnv) { } } -let wrapped_handler; +let wrappedHandler; try { - wrapped_handler = datadog(await load(taskRootEnv, handlerEnv), { traceExtractor }); + wrappedHandler = datadog(await load(taskRootEnv, handlerEnv), { traceExtractor }); } catch (error) { await emitTelemetryOnErrorOutsideHandler(error, handlerEnv, Date.now()); throw error; } -export const handler = wrapped_handler; +export const handler = wrappedHandler; From f14e40296e3e4f4b717a1604342b7983f9212ec1 Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Tue, 7 May 2024 14:59:33 -0400 Subject: [PATCH 6/8] push new tags onto existing array --- src/metrics/enhanced-metrics.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metrics/enhanced-metrics.ts b/src/metrics/enhanced-metrics.ts index fb5cf143..36eeb8da 100644 --- a/src/metrics/enhanced-metrics.ts +++ b/src/metrics/enhanced-metrics.ts @@ -50,13 +50,13 @@ export function getRuntimeTag(): string | null { } export function getEnhancedMetricTags(context?: Context): string[] { - let tags: string[] = []; + const tags: string[] = []; if (context) { let arnTags = [`functionname:${context.functionName}`]; if (context.invokedFunctionArn) { arnTags = parseTagsFromARN(context.invokedFunctionArn, context.functionVersion); } - tags = [...arnTags, `memorysize:${context.memoryLimitInMB}`]; + tags.push(...arnTags, `memorysize:${context.memoryLimitInMB}`); } tags.push(...getSandboxInitTags(), getVersionTag()); From 53652370262048ad5f419b8038f24c037093c94d Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Tue, 7 May 2024 15:09:39 -0400 Subject: [PATCH 7/8] handle promise rejection --- src/handler.cjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/handler.cjs b/src/handler.cjs index 6b391cc9..740a3f58 100644 --- a/src/handler.cjs +++ b/src/handler.cjs @@ -36,6 +36,8 @@ if (extractorEnv) { try { exports.handler = datadog(loadSync(taskRootEnv, handlerEnv), { traceExtractor }); } catch (error) { - emitTelemetryOnErrorOutsideHandler(error, handlerEnv, Date.now()); + emitTelemetryOnErrorOutsideHandler(error, handlerEnv, Date.now()).catch( + logDebug("failed to error telemetry on error outside handler"), + ); throw error; } From 75ae1c8f902618fa6d3d8786d4ebee1417eece00 Mon Sep 17 00:00:00 2001 From: Tal Usvyatsky Date: Wed, 8 May 2024 09:50:59 -0400 Subject: [PATCH 8/8] move all listener code inside condition --- src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/index.ts b/src/index.ts index 388d47bd..c3715aa0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -445,10 +445,10 @@ export async function emitTelemetryOnErrorOutsideHandler( } const config = getConfig(); - const metricsListener = new MetricsListener(new KMSService(), config); - await metricsListener.onStartInvocation(undefined); if (config.enhancedMetrics) { + const metricsListener = new MetricsListener(new KMSService(), config); + await metricsListener.onStartInvocation(undefined); incrementErrorsMetric(metricsListener); + await metricsListener.onCompleteInvocation(); } - await metricsListener.onCompleteInvocation(); }