test(maintenance): add ESM output to e2e test (#2370)

* test(maintenance): add esm output to e2e tests

* test: switch 50% tests to esm

* test: switch 50% tests to esm

* test: switch 50% tests to esm

* test: refactor layers test to deploy cjs & esm

* chore: marked @AWS-SDK as external

* chore: remove redundant boolean literals

* chore: add max-parallel to avoid timeouts

* chore: adjust max-parallel

* chore: aws-sdk workaround

* chore: override nodejs16 to bundle aws-sdk when working with esm

* chore: exclude ESM on Node.js 16 from layer tests

* docs: add callout about ESM & Layers in docs

---------

Co-authored-by: Alexander Schueren <sha@amazon.com>

Author: dreamorosi

.github/workflows/run-e2e-tests.yml

@@ -21,6 +21,7 @@ jobs:
       id-token: write # needed to interact with GitHub's OIDC Token endpoint.
       contents: read
     strategy:
+      max-parallel: 30
       matrix:
         package:
           [

docs/index.md

@@ -254,6 +254,8 @@ You can use Powertools for AWS Lambda (TypeScript) by installing it with your fa
 
 [Lambda Layer](https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html){target="_blank"} is a `.zip` file archive that can contain additional code, pre-packaged dependencies, data, or configuration files. We compile and optimize [all dependencies](#install) to achieve an optimal build.
 
+You can use the Lambda Layer both with CommonJS and ESM (ECMAScript modules) for Node.js 18.x and newer runtimes. **If you are using the managed Node.js 16.x runtime and cannot upgrade, you should use the CommonJS version only**.
+
 ??? note "Click to expand and copy any regional Lambda Layer ARN"
     | Region           | Layer ARN                                                                                                     |
     | ---------------- | ------------------------------------------------------------------------------------------------------------- |

layers/src/layer-publisher-stack.ts

@@ -100,9 +100,7 @@ export class LayerPublisherStack extends Stack {
                 'node_modules/@aws-lambda-powertools/*/lib/**/*.d.ts',
                 'node_modules/@aws-lambda-powertools/*/lib/**/*.d.ts.map',
                 'node_modules/@aws-sdk/*/dist-types',
-                'node_modules/@aws-sdk/*/dist-es',
                 'node_modules/@smithy/*/dist-types',
-                'node_modules/@smithy/*/dist-es',
                 'node_modules/@smithy/**/README.md ',
                 'node_modules/@aws-sdk/**/README.md ',
               ];

layers/tests/e2e/layerPublisher.test.ts

@@ -11,35 +11,52 @@ import {
   TestInvocationLogs,
   invokeFunctionOnce,
   generateTestUniqueName,
+  getRuntimeKey,
 } from '@aws-lambda-powertools/testing-utils';
 import { TestNodejsFunction } from '@aws-lambda-powertools/testing-utils/resources/lambda';
 import {
   RESOURCE_NAME_PREFIX,
   SETUP_TIMEOUT,
   TEARDOWN_TIMEOUT,
-  TEST_CASE_TIMEOUT,
 } from './constants';
 import { join } from 'node:path';
 import packageJson from '../../package.json';
 
 jest.spyOn(console, 'log').mockImplementation();
 
+// eslint-disable-next-line func-style -- type assertions can't be arrow functions
+function assertLogs(
+  logs: TestInvocationLogs | undefined
+): asserts logs is TestInvocationLogs {
+  if (!logs) {
+    throw new Error('Function logs are not available');
+  }
+}
+
 /**
  * This test has two stacks:
  * 1. LayerPublisherStack - publishes a layer version using the LayerPublisher construct and containing the Powertools utilities from the repo
- * 2. TestStack - uses the layer published in the first stack and contains a lambda function that uses the Powertools utilities from the layer
+ * 2. TestStack - uses the layer published in the first stack and contains two lambda functions that use the Powertools utilities from the layer
  *
  * The lambda function is invoked once and the logs are collected. The goal of the test is to verify that the layer creation and usage works as expected.
  */
-describe(`Layers E2E tests, publisher stack`, () => {
+describe(`Layers E2E tests`, () => {
   const testStack = new TestStack({
     stackNameProps: {
       stackNamePrefix: RESOURCE_NAME_PREFIX,
       testName: 'functionStack',
     },
   });
 
-  let invocationLogs: TestInvocationLogs;
+  /**
+   * Node.js 16.x does not support importing ESM modules from Lambda Layers reliably.
+   *
+   * The feature is available in Node.js 18.x and later.
+   * @see https://aws.amazon.com/blogs/compute/node-js-18-x-runtime-now-available-in-aws-lambda/
+   */
+  const cases = getRuntimeKey() === 'nodejs16x' ? ['CJS'] : ['CJS', 'ESM'];
+  const invocationLogsMap: Map<(typeof cases)[number], TestInvocationLogs> =
+    new Map();
 
   const ssmParameterLayerName = generateTestUniqueName({
     testPrefix: `${RESOURCE_NAME_PREFIX}`,
@@ -75,76 +92,83 @@ describe(`Layers E2E tests, publisher stack`, () => {
   });
 
   beforeAll(async () => {
+    // Deploy the stack that publishes the layer
     await testLayerStack.deploy();
 
+    // Import the layer version from the stack outputs into the test stack
     const layerVersion = LayerVersion.fromLayerVersionArn(
       testStack.stack,
       'LayerVersionArnReference',
       testLayerStack.findAndGetStackOutputValue('LatestLayerArn')
     );
-    new TestNodejsFunction(
-      testStack,
-      {
-        entry: lambdaFunctionCodeFilePath,
-        environment: {
-          LAYERS_PATH: '/opt/nodejs/node_modules',
-          POWERTOOLS_PACKAGE_VERSION: powerToolsPackageVersion,
-          POWERTOOLS_SERVICE_NAME: 'LayerPublisherStack',
-        },
-        bundling: {
-          externalModules: [
-            '@aws-lambda-powertools/commons',
-            '@aws-lambda-powertools/logger',
-            '@aws-lambda-powertools/metrics',
-            '@aws-lambda-powertools/tracer',
-            '@aws-lambda-powertools/parameter',
-            '@aws-lambda-powertools/idempotency',
-            '@aws-lambda-powertools/batch',
-          ],
+
+    // Add a lambda function for each output format to the test stack
+    cases.forEach((outputFormat) => {
+      new TestNodejsFunction(
+        testStack,
+        {
+          entry: lambdaFunctionCodeFilePath,
+          environment: {
+            LAYERS_PATH: '/opt/nodejs/node_modules',
+            POWERTOOLS_PACKAGE_VERSION: powerToolsPackageVersion,
+            POWERTOOLS_SERVICE_NAME: 'LayerPublisherStack',
+          },
+          bundling: {
+            externalModules: [
+              '@aws-lambda-powertools/*',
+              '@aws-sdk/*',
+              'aws-xray-sdk-node',
+            ],
+          },
+          layers: [layerVersion],
         },
-        layers: [layerVersion],
-      },
-      {
-        nameSuffix: 'testFn',
-      }
-    );
+        {
+          nameSuffix: `test${outputFormat}Fn`,
+          ...(outputFormat === 'ESM' && { outputFormat: 'ESM' }),
+        }
+      );
+    });
 
+    // Deploy the test stack
     await testStack.deploy();
 
-    const functionName = testStack.findAndGetStackOutputValue('testFn');
-
-    invocationLogs = await invokeFunctionOnce({
-      functionName,
-    });
+    // Invoke the lambda function once for each output format and collect the logs
+    for await (const outputFormat of cases) {
+      invocationLogsMap.set(
+        outputFormat,
+        await invokeFunctionOnce({
+          functionName: testStack.findAndGetStackOutputValue(
+            `test${outputFormat}Fn`
+          ),
+        })
+      );
+    }
   }, SETUP_TIMEOUT);
 
-  describe('package version and path check', () => {
-    it(
-      'should have no errors in the logs, which indicates the pacakges version matches the expected one',
-      () => {
+  describe.each(cases)(
+    'utilities tests for %s output format',
+    (outputFormat) => {
+      let invocationLogs: TestInvocationLogs;
+      beforeAll(() => {
+        const maybeInvocationLogs = invocationLogsMap.get(outputFormat);
+        assertLogs(maybeInvocationLogs);
+        invocationLogs = maybeInvocationLogs;
+      });
+
+      it('should have no errors in the logs, which indicates the pacakges version matches the expected one', () => {
         const logs = invocationLogs.getFunctionLogs('ERROR');
 
         expect(logs.length).toBe(0);
-      },
-      TEST_CASE_TIMEOUT
-    );
-  });
+      });
 
-  describe('utilities usage', () => {
-    it(
-      'should have one warning related to missing Metrics namespace',
-      () => {
+      it('should have one warning related to missing Metrics namespace', () => {
         const logs = invocationLogs.getFunctionLogs('WARN');
 
         expect(logs.length).toBe(1);
         expect(logs[0]).toContain('Namespace should be defined, default used');
-      },
-      TEST_CASE_TIMEOUT
-    );
+      });
 
-    it(
-      'should have one info log related to coldstart metric',
-      () => {
+      it('should have one info log related to coldstart metric', () => {
         const logs = invocationLogs.getFunctionLogs();
         const emfLogEntry = logs.find((log) =>
           log.match(
@@ -153,13 +177,9 @@ describe(`Layers E2E tests, publisher stack`, () => {
         );
 
         expect(emfLogEntry).toBeDefined();
-      },
-      TEST_CASE_TIMEOUT
-    );
+      });
 
-    it(
-      'should have one debug log with tracer subsegment info',
-      () => {
+      it('should have one debug log with tracer subsegment info', () => {
         const logs = invocationLogs.getFunctionLogs('DEBUG');
 
         expect(logs.length).toBe(1);
@@ -182,10 +202,9 @@ describe(`Layers E2E tests, publisher stack`, () => {
             trace_id: traceIdFromLog,
           })
         );
-      },
-      TEST_CASE_TIMEOUT
-    );
-  });
+      });
+    }
+  );
 
   afterAll(async () => {
     if (!process.env.DISABLE_TEARDOWN) {

packages/idempotency/tests/e2e/idempotentDecorator.test.ts

@@ -63,6 +63,7 @@ describe('Idempotency e2e test decorator, default settings', () => {
     },
     {
       nameSuffix: 'defaultParallel',
+      outputFormat: 'ESM',
     }
   );
 
@@ -79,6 +80,7 @@ describe('Idempotency e2e test decorator, default settings', () => {
     },
     {
       nameSuffix: 'timeout',
+      outputFormat: 'ESM',
     }
   );
 
@@ -95,6 +97,7 @@ describe('Idempotency e2e test decorator, default settings', () => {
     },
     {
       nameSuffix: 'expired',
+      outputFormat: 'ESM',
     }
   );
 
@@ -110,6 +113,7 @@ describe('Idempotency e2e test decorator, default settings', () => {
     },
     {
       nameSuffix: 'dataIndex',
+      outputFormat: 'ESM',
     }
   );
 
@@ -131,6 +135,7 @@ describe('Idempotency e2e test decorator, default settings', () => {
     },
     {
       nameSuffix: 'customConfig',
+      outputFormat: 'ESM',
     }
   );
 

packages/logger/tests/e2e/basicFeatures.middy.test.ts

@@ -48,6 +48,7 @@ describe(`Logger E2E tests, basic functionalities middy usage`, () => {
       {
         logGroupOutputKey: STACK_OUTPUT_LOG_GROUP,
         nameSuffix: 'BasicFeatures',
+        outputFormat: 'ESM',
       }
     );
 

packages/logger/tests/e2e/sampleRate.decorator.test.ts

@@ -52,6 +52,7 @@ describe(`Logger E2E tests, sample rate and injectLambdaContext()`, () => {
       {
         logGroupOutputKey: STACK_OUTPUT_LOG_GROUP,
         nameSuffix: 'BasicFeatures',
+        outputFormat: 'ESM',
       }
     );
 

packages/metrics/tests/e2e/basicFeatures.decorators.test.ts

@@ -50,6 +50,7 @@ describe(`Metrics E2E tests, basic features decorator usage`, () => {
     },
     {
       nameSuffix: 'BasicFeatures',
+      outputFormat: 'ESM',
     }
   );
 

packages/parameters/tests/e2e/appConfigProvider.class.test.ts

@@ -120,6 +120,7 @@ describe(`Parameters E2E tests, AppConfig provider`, () => {
       },
       {
         nameSuffix: 'appConfigProvider',
+        outputFormat: 'ESM',
       }
     );
 

packages/parameters/tests/e2e/secretsProvider.class.test.ts

@@ -60,6 +60,7 @@ describe(`Parameters E2E tests, Secrets Manager provider`, () => {
       },
       {
         nameSuffix: 'secretsProvider',
+        outputFormat: 'ESM',
       }
     );
 

packages/testing/src/resources/TestNodejsFunction.ts

@@ -1,6 +1,6 @@
 import { CfnOutput, Duration } from 'aws-cdk-lib';
 import { Tracing } from 'aws-cdk-lib/aws-lambda';
-import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs';
+import { NodejsFunction, OutputFormat } from 'aws-cdk-lib/aws-lambda-nodejs';
 import { RetentionDays } from 'aws-cdk-lib/aws-logs';
 import { randomUUID } from 'node:crypto';
 import { TEST_RUNTIMES, TEST_ARCHITECTURES } from '../constants.js';
@@ -23,11 +23,24 @@ class TestNodejsFunction extends NodejsFunction {
     props: TestNodejsFunctionProps,
     extraProps: ExtraTestProps
   ) {
+    const isESM = extraProps.outputFormat === 'ESM';
+    const { bundling, ...restProps } = props;
+
     super(stack.stack, `fn-${randomUUID().substring(0, 5)}`, {
       timeout: Duration.seconds(30),
-      memorySize: 256,
+      memorySize: 512,
       tracing: Tracing.ACTIVE,
-      ...props,
+      bundling: {
+        ...bundling,
+        minify: true,
+        mainFields: isESM ? ['module', 'main'] : ['main', 'module'],
+        sourceMap: false,
+        format: isESM ? OutputFormat.ESM : OutputFormat.CJS,
+        banner: isESM
+          ? `import { createRequire } from 'module';const require = createRequire(import.meta.url);`
+          : '',
+      },
+      ...restProps,
       functionName: concatenateResourceName({
         testName: stack.testName,
         resourceName: extraProps.nameSuffix,

packages/testing/src/types.ts

@@ -13,6 +13,12 @@ interface ExtraTestProps {
    * Note that the maximum length of the name is 64 characters, so the suffix might be truncated.
    */
   nameSuffix: string;
+  /**
+   * The output format of the bundled code.
+   *
+   * @default 'CJS'
+   */
+  outputFormat?: 'CJS' | 'ESM';
 }
 
 type TestDynamodbTableProps = Omit<
@@ -27,8 +33,13 @@ type TestDynamodbTableProps = Omit<
 
 type TestNodejsFunctionProps = Omit<
   NodejsFunctionProps,
-  'logRetention' | 'runtime' | 'functionName'
->;
+  'logRetention' | 'runtime' | 'functionName' | 'bundling'
+> & {
+  bundling?: Omit<
+    NodejsFunctionProps['bundling'],
+    'minify' | 'mainFields' | 'sourceMap' | 'format' | 'banner'
+  >;
+};
 
 type InvokeTestFunctionOptions = {
   functionName: string;