From e290845868d67790e0f4b1f7ae64a42fb9699838 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 8 Feb 2023 16:37:12 +0100 Subject: [PATCH 1/4] fix import to src, instead of lib --- .../tests/e2e/secretsProvider.class.test.functionCode.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index 655b9472a6..86ecc7d995 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -1,9 +1,9 @@ import { Context } from 'aws-lambda'; -import { SecretsProvider } from '../../lib/secrets'; import { TinyLogger } from '../helpers/tinyLogger'; -import { SecretsGetOptionsInterface } from '../../lib/types'; import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager'; import { middleware } from '../helpers/sdkMiddlewareRequestCounter'; +import { SecretsProvider } from '../../src/secrets'; +import { SecretsGetOptionsInterface } from '../../src/types'; const logger = new TinyLogger(); const defaultProvider = new SecretsProvider(); From f17c396c10de6fa9b3729251b6d2413ccf9293b1 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 8 Feb 2023 17:05:37 +0100 Subject: [PATCH 2/4] clearing cache to fix forceFetch test case --- .../tests/e2e/secretsProvider.class.test.functionCode.ts | 1 + packages/parameters/tests/e2e/secretsProvider.class.test.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index 86ecc7d995..c657bd0556 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -77,6 +77,7 @@ export const handler = async (_event: unknown, _context: Context): Promise // get parameter twice, but force fetch 2nd time, we count number of SDK requests and check that we made two API calls try { middleware.counter = 0; + providerWithMiddleware.clearCache(); await providerWithMiddleware.get(secretNamePlainChached); await providerWithMiddleware.get(secretNamePlainChached, { forceFetch: true }); logger.log({ diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.ts index a4ea351151..b8f7fbdf18 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.ts @@ -176,7 +176,7 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => expect(testLogFirst).toStrictEqual({ test: 'get-plain-force', - value: 1 + value: 2 }); }); From 74b7cc88f3b97a65387d2b363199a9e2ccbbdf96 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 8 Feb 2023 18:26:01 +0100 Subject: [PATCH 3/4] add more comments, create dedicated resource for test 7 --- ...secretsProvider.class.test.functionCode.ts | 19 ++-- .../tests/e2e/secretsProvider.class.test.ts | 86 +++++++++++++------ 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index c657bd0556..b57310d864 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -14,6 +14,7 @@ const secretNameBinary = process.env.SECRET_NAME_BINARY || ''; const secretNameObjectWithSuffix = process.env.SECRET_NAME_OBJECT_WITH_SUFFIX || ''; const secretNameBinaryWithSuffix = process.env.SECRET_NAME_BINARY_WITH_SUFFIX || ''; const secretNamePlainChached = process.env.SECRET_NAME_PLAIN_CACHED || ''; +const secretNamePlainForceFetch = process.env.SECRET_NAME_PLAIN_FORCE_FETCH || ''; // Provider test 8, 9 const customClient = new SecretsManagerClient({}); @@ -42,23 +43,23 @@ const _call_get = async (paramName: string, testName: string, options?: SecretsG export const handler = async (_event: unknown, _context: Context): Promise => { - // Test 1 get single param as plaintext + // Test 1 get single secrt as plaintext await _call_get(secretNamePlain, 'get-plain'); - // Test 2 get single param with transform json + // Test 2 get single secret with transform json await _call_get(secretNameObject, 'get-transform-json', { transform: 'json' }); - // Test 3 get single param with transform binary + // Test 3 get single secret with transform binary await _call_get(secretNameBinary, 'get-transform-binary', { transform: 'binary' }); - // Test 4 get single param with transform auto json + // Test 4 get single secret with transform auto json await _call_get(secretNameObjectWithSuffix, 'get-transform-auto-json', { transform: 'auto' }); - // Test 5 get single param with transform auto binary + // Test 5 get single secret with transform auto binary await _call_get(secretNameBinaryWithSuffix, 'get-transform-auto-binary', { transform: 'auto' }); // Test 6 - // get parameter twice with middleware, which counts number of SDK requests, we check later if we only called SecretManager API once + // get secret twice with middleware, which counts number of SDK requests, we check later if we only called SecretManager API once try { middleware.counter = 0; await providerWithMiddleware.get(secretNamePlainChached); @@ -74,12 +75,12 @@ export const handler = async (_event: unknown, _context: Context): Promise }); } // Test 7 - // get parameter twice, but force fetch 2nd time, we count number of SDK requests and check that we made two API calls + // get secret twice, but force fetch 2nd time, we count number of SDK requests and check that we made two API calls try { middleware.counter = 0; providerWithMiddleware.clearCache(); - await providerWithMiddleware.get(secretNamePlainChached); - await providerWithMiddleware.get(secretNamePlainChached, { forceFetch: true }); + await providerWithMiddleware.get(secretNamePlainForceFetch); + await providerWithMiddleware.get(secretNamePlainForceFetch, { forceFetch: true }); logger.log({ test: 'get-plain-force', value: middleware.counter // should be 2 diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.ts index b8f7fbdf18..48e2734974 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.ts @@ -24,29 +24,51 @@ const runtime: string = process.env.RUNTIME || 'nodejs18x'; if (!isValidRuntimeKey(runtime)) { throw new Error(`Invalid runtime key: ${runtime}`); } - -const uuid = v4(); -const stackName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'secretsProvider'); -const functionName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'secretsProvider'); -const lambdaFunctionCodeFile = 'secretsProvider.class.test.functionCode.ts'; - -const secretNamePlain = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlain'); -const secretNamePlainCached = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlainCached'); -const secretNameObject = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject'); -const secretNameBinary = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretBinary'); -const secretNameObjectWithSuffix = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject.json'); -const secretNameBinaryWithSuffix = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject.binary'); - -const invocationCount = 1; - -const integTestApp = new App(); -let stack: Stack; - +/** + * Collection of e2e tests for SecretsProvider utility. + * + * Test 1: create a secret with plain text value, fetch it with no additional options + * Test 2: create a secret with json value, fetch it using `transform: 'json'` option + * Test 3: create a secret with base64 encoded value (technicaly string), fetch it using `transform: 'binary'` option + * Test 4: create a secret with json value and secret name ends with .json, fetch it using `transform: 'auto'` option + * Test 5: create a secret with base64 encoded value (technicaly string) and secert name ends with .binary, fetch it using `transform: 'auto'` option + * Test 6: create a secret with plain text value, fetch it twice, check that value was cached, the number of SDK calls should be 1 + * Test 7: create a secret with plain text value, fetch it twice, second time with `forceFetch: true` option, check that value was not cached, the number of SDK calls should be 2 + * + * For tests 6 and 7 we use our own AWS SDK custom middleware plugin `sdkMiddlewareRequestCounter.ts` + * + * Adding new test: + * Please keep the state clean, and create dedicated resource for your test, don't reuse resources from other tests. + * Pass the necessary information to lambda function by using enviroment variables + * Make sure to add the right permissions to the lambda function to access the resources. We use our `ResourceAccessGranter` to add permissions. + * + */ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => { + const uuid = v4(); let invocationLogs: InvocationLogs[]; + const stackName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'secretsProvider'); + const functionName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'secretsProvider'); + const lambdaFunctionCodeFile = 'secretsProvider.class.test.functionCode.ts'; + + const invocationCount = 1; + + const integTestApp = new App(); + let stack: Stack; beforeAll(async () => { + + // use unique names for each test to keep a clean state + const secretNamePlain = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlain'); + const secretNameObject = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject'); + const secretNameBinary = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretBinary'); + const secretNameObjectWithSuffix = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject.json'); + const secretNameBinaryWithSuffix = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject.binary'); + const secretNamePlainCached = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlainCached'); + const secretNamePlainForceFetch = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlainForceFetch'); + + // creates the test fuction that uses powertools secret provider we want to test + // pass env vars with secret names we want to fetch stack = createStackWithLambdaFunction({ app: integTestApp, stackName: stackName, @@ -61,6 +83,7 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => SECRET_NAME_OBJECT_WITH_SUFFIX: secretNameObjectWithSuffix, SECRET_NAME_BINARY_WITH_SUFFIX: secretNameBinaryWithSuffix, SECRET_NAME_PLAIN_CACHED: secretNamePlainCached, + SECRET_NAME_PLAIN_FORCE_FETCH: secretNamePlainForceFetch, }, runtime: runtime }); @@ -99,7 +122,14 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => secretStringValue: SecretValue.unsafePlainText('foo') }); - Aspects.of(stack).add(new ResourceAccessGranter([ secretString, secretObject, secretBinary, secretObjectWithSuffix, secretBinaryWithSuffix, secretStringCached ])); + const secretStringForceFetch = new Secret(stack, 'testSecretStringForceFetch', { + secretName: secretNamePlainForceFetch, + secretStringValue: SecretValue.unsafePlainText('foo') + }); + + // add secrets here to grant lambda permisisons to access secrets + Aspects.of(stack).add(new ResourceAccessGranter([ + secretString, secretObject, secretBinary, secretObjectWithSuffix, secretBinaryWithSuffix, secretStringCached, secretStringForceFetch ])); await deployStack(integTestApp, stack); @@ -108,7 +138,8 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => }, SETUP_TIMEOUT); describe('SecretsProvider usage', () => { - it('should retrieve a single parameter', async () => { + + it('should retrieve a secret as plain string', async () => { const logs = invocationLogs[0].getFunctionLogs(); const testLog = InvocationLogs.parseFunctionLog(logs[0]); @@ -119,7 +150,7 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => }); }, TEST_CASE_TIMEOUT); - it('should retrieve a single parameter with transform json', async () => { + it('should retrieve a secret using transform json option', async () => { const logs = invocationLogs[0].getFunctionLogs(); const testLog = InvocationLogs.parseFunctionLog(logs[1]); @@ -129,7 +160,7 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => }); }, TEST_CASE_TIMEOUT); - it('should retrieve single param with transform binary', async () => { + it('should retrieve a secret using transform binary option', async () => { const logs = invocationLogs[0].getFunctionLogs(); const testLog = InvocationLogs.parseFunctionLog(logs[2]); @@ -140,17 +171,18 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => }, TEST_CASE_TIMEOUT); }); - it('should retrieve single param with transform auto json', async () => { + it('should retrieve a secret using transform auto option with implicit json', async () => { const logs = invocationLogs[0].getFunctionLogs(); const testLog = InvocationLogs.parseFunctionLog(logs[3]); + // result should be a json object expect(testLog).toStrictEqual({ test: 'get-transform-auto-json', value: { foo: 'bar' } }); }, TEST_CASE_TIMEOUT); - it('should retrieve single param wit transform auto binary', async () => { + it('should retrieve a secret using transform auto option with implicit binary', async () => { const logs = invocationLogs[0].getFunctionLogs(); const testLog = InvocationLogs.parseFunctionLog(logs[4]); @@ -160,20 +192,22 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => }); }); - it('should retrieve single parameter cached', async () => { + it('should retrieve a secret twice with cached value', async () => { const logs = invocationLogs[0].getFunctionLogs(); const testLogFirst = InvocationLogs.parseFunctionLog(logs[5]); + // we fetch twice, but we expect to make an API call only once expect(testLogFirst).toStrictEqual({ test: 'get-plain-cached', value: 1 }); }); - it('should retrieve single parameter twice without caching', async () => { + it('should retrieve a secret twice with forceFetch second time', async () => { const logs = invocationLogs[0].getFunctionLogs(); const testLogFirst = InvocationLogs.parseFunctionLog(logs[6]); + // we fetch twice, 2nd time with forceFetch: true flag, we expect two api calls expect(testLogFirst).toStrictEqual({ test: 'get-plain-force', value: 2 From 1bbb514e00d80f4ed65bccf81832d4423e893f78 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Thu, 9 Feb 2023 11:56:42 +0100 Subject: [PATCH 4/4] Update packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts --- .../tests/e2e/secretsProvider.class.test.functionCode.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index b57310d864..a1bd955919 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -43,7 +43,7 @@ const _call_get = async (paramName: string, testName: string, options?: SecretsG export const handler = async (_event: unknown, _context: Context): Promise => { - // Test 1 get single secrt as plaintext + // Test 1 get single secret as plaintext await _call_get(secretNamePlain, 'get-plain'); // Test 2 get single secret with transform json