From eaa862ad8fec6e1249b4f465c943153448a8a975 Mon Sep 17 00:00:00 2001 From: seebees Date: Fri, 14 Feb 2020 15:39:41 -0800 Subject: [PATCH 1/2] feat: Add concurrency for running tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running the integration tests can be speed up significantly by adding concurrency. This adds a `—concurrency` and `-c` option to the cli that can be used to run multiple tests in parallel. --- modules/integration-node/src/cli.ts | 12 +- .../integration-node/src/integration_tests.ts | 111 ++++++++++++++---- package.json | 4 +- 3 files changed, 100 insertions(+), 27 deletions(-) diff --git a/modules/integration-node/src/cli.ts b/modules/integration-node/src/cli.ts index 0186cb873..f98da8034 100644 --- a/modules/integration-node/src/cli.ts +++ b/modules/integration-node/src/cli.ts @@ -57,18 +57,24 @@ const cli = yargs describe: 'an optional test name to execute', type: 'string' }) + .option('concurrency', { + alias: 'c', + describe: 'an optional concurrency for running tests', + type: 'number', + default: 1 + }) .demandCommand() ;(async (argv) => { - const { _: [ command ], tolerateFailures, testName } = argv + const { _: [ command ], tolerateFailures, testName, concurrency } = argv /* I set the result to 1 so that if I fall through the exit condition is a failure */ let result = 1 if (command === 'decrypt') { const { vectorFile } = argv as unknown as { vectorFile: string} - result = await integrationDecryptTestVectors(vectorFile, tolerateFailures, testName) + result = await integrationDecryptTestVectors(vectorFile, tolerateFailures, testName, concurrency) } else if (command === 'encrypt') { const { manifestFile, keyFile, decryptOracle } = argv as unknown as { manifestFile: string, keyFile: string, decryptOracle: string} - result = await integrationEncryptTestVectors(manifestFile, keyFile, decryptOracle, tolerateFailures, testName) + result = await integrationEncryptTestVectors(manifestFile, keyFile, decryptOracle, tolerateFailures, testName, concurrency) } else { console.log(`Unknown command ${command}`) cli.showHelp() diff --git a/modules/integration-node/src/integration_tests.ts b/modules/integration-node/src/integration_tests.ts index 6b7e7d22a..34b0ffa28 100644 --- a/modules/integration-node/src/integration_tests.ts +++ b/modules/integration-node/src/integration_tests.ts @@ -72,57 +72,124 @@ export async function testEncryptVector ({ name, keysInfo, encryptOp, plainTextD } } -export async function integrationDecryptTestVectors (vectorFile: string, tolerateFailures: number = 0, testName?: string) { +export async function integrationDecryptTestVectors (vectorFile: string, tolerateFailures: number = 0, testName?: string, concurrency: number = 1) { const tests = await getDecryptTestVectorIterator(vectorFile) - let failureCount = 0 - for (const test of tests) { + + return parallelTests(concurrency, tolerateFailures, runTest, tests) + + async function runTest (test: TestVectorInfo): Promise { if (testName) { - if (test.name !== testName) continue + if (test.name !== testName) return true } const { result, name, err } = await testDecryptVector(test) if (result) { console.log({ name, result }) + return true } else { if (err && notSupportedDecryptMessages.includes(err.message)) { console.log({ name, result: `Not supported: ${err.message}` }) - continue + return true } console.log({ name, result, err }) - } - if (!result) { - failureCount += 1 - if (!tolerateFailures) return failureCount - tolerateFailures-- + return false } } - return failureCount } -export async function integrationEncryptTestVectors (manifestFile: string, keyFile: string, decryptOracle: string, tolerateFailures: number = 0, testName?: string) { +export async function integrationEncryptTestVectors (manifestFile: string, keyFile: string, decryptOracle: string, tolerateFailures: number = 0, testName?: string, concurrency: number = 1) { const decryptOracleUrl = new URL(decryptOracle) const tests = await getEncryptTestVectorIterator(manifestFile, keyFile) - let failureCount = 0 - for (const test of tests) { + + return parallelTests(concurrency, tolerateFailures, runTest, tests) + + async function runTest (test: EncryptTestVectorInfo): Promise { if (testName) { - if (test.name !== testName) continue + if (test.name !== testName) return true } const { result, name, err } = await testEncryptVector(test, decryptOracleUrl) if (result) { console.log({ name, result }) + return true } else { if (err && notSupportedEncryptMessages.includes(err.message)) { console.log({ name, result: `Not supported: ${err.message}` }) - continue + return true } console.log({ name, result, err }) + return false } - if (!result) { - failureCount += 1 - if (!tolerateFailures) return failureCount - tolerateFailures-- - } } - return failureCount +} + +async function parallelTests< + Test extends EncryptTestVectorInfo|TestVectorInfo, + work extends (test: Test) => Promise +>(max: number, tolerateFailures: number, runTest: work, tests: IterableIterator) { + let _resolve: (failureCount: number) => void + const queue = new Set>() + let failureCount = 0 + + return new Promise((resolve) => { + _resolve = resolve + enqueue() + }) + + function enqueue (): void { + /* If there are more failures than I am willing to tolerate, stop. */ + if (failureCount > tolerateFailures) return _resolve(failureCount) + /* Do not over-fill the queue! */ + if (queue.size > max) return + + const { value, done } = tests.next() + /* There is an edge here, + * you _could_ return a value *and* be done. + * Most iterators don't but in this case + * we just process the value and ask for another. + * Which will return done as true again. + */ + if (!value && done) return _resolve(failureCount) + + /* I need to define the work to be enqueue + * and a way to dequeue this work when complete. + * A Set of promises works nicely. + * Hold the variable here + * put it in the Set, take it out, and Bob's your uncle. + */ + const work: Promise = runTest(value) + .then((pass: boolean) => { + if (!pass) failureCount += 1 + }) + /* If there is some unknown error, + * it's just an error... + * Treat it like a test failure. + */ + .catch((err) => { + console.log(err) + failureCount += 1 + }) + .then(() => { + /* Dequeue this work.*/ + queue.delete(work) + /* More to eat? */ + enqueue() + }) + + /* Enqueue this work */ + queue.add(work) + + /* Fill the queue. + * The over-fill check above protects me. + * Sure, it is possible to exceed the stack depth. + * If you are trying to run ~10K tests in parallel + * on a system where that is actually faster, + * I want to talk to you. + * It is true that node can be configured to process + * > 10K HTTP requests no problem, + * but even the decrypt tests require that you first + * encrypt something locally before making the http call. + */ + enqueue() + } } interface TestVectorResults { diff --git a/package.json b/package.json index 93c662225..b6dc74b13 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,8 @@ "integration-browser-decrypt": "npm run build; lerna run build_fixtures --stream --no-prefix -- -- decrypt -v $npm_package_config_localTestVectors --karma", "integration-browser-encrypt": "npm run build; lerna run build_fixtures --stream --no-prefix -- -- encrypt -m $npm_package_config_encryptManifestList -k $npm_package_config_encryptKeyManifest -o $npm_package_config_decryptOracle --karma", "browser-integration": "run-s integration-browser-*", - "integration-node-decrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- decrypt -v $npm_package_config_localTestVectors", - "integration-node-encrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- encrypt -m $npm_package_config_encryptManifestList -k $npm_package_config_encryptKeyManifest -o $npm_package_config_decryptOracle", + "integration-node-decrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- decrypt -v $npm_package_config_localTestVectors -c 10", + "integration-node-encrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- encrypt -m $npm_package_config_encryptManifestList -k $npm_package_config_encryptKeyManifest -o $npm_package_config_decryptOracle -c 20", "node-integration": "run-s integration-node-*", "integration": "run-s integration-*", "test_conditions": "./util/test_conditions" From b12db1d45c261e9005d071842dee302f88906104 Mon Sep 17 00:00:00 2001 From: seebees Date: Fri, 14 Feb 2020 16:08:58 -0800 Subject: [PATCH 2/2] lint --- modules/integration-node/src/integration_tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/integration-node/src/integration_tests.ts b/modules/integration-node/src/integration_tests.ts index 34b0ffa28..6fc72659d 100644 --- a/modules/integration-node/src/integration_tests.ts +++ b/modules/integration-node/src/integration_tests.ts @@ -168,7 +168,7 @@ async function parallelTests< failureCount += 1 }) .then(() => { - /* Dequeue this work.*/ + /* Dequeue this work. */ queue.delete(work) /* More to eat? */ enqueue()