feat: Add concurrency for running tests (#243)

Running the integration tests
can be speed up significantly by adding concurrency.
This adds a `—concurrency` and `-c` option to the cli
that can be used to run multiple tests in parallel.

Author: seebees

modules/integration-node/src/cli.ts

@@ -57,18 +57,24 @@ const cli = yargs
     describe: 'an optional test name to execute',
     type: 'string'
   })
+  .option('concurrency', {
+    alias: 'c',
+    describe: 'an optional concurrency for running tests',
+    type: 'number',
+    default: 1
+  })
   .demandCommand()
 
 ;(async (argv) => {
-  const { _: [ command ], tolerateFailures, testName } = argv
+  const { _: [ command ], tolerateFailures, testName, concurrency } = argv
   /* I set the result to 1 so that if I fall through the exit condition is a failure */
   let result = 1
   if (command === 'decrypt') {
     const { vectorFile } = argv as unknown as { vectorFile: string}
-    result = await integrationDecryptTestVectors(vectorFile, tolerateFailures, testName)
+    result = await integrationDecryptTestVectors(vectorFile, tolerateFailures, testName, concurrency)
   } else if (command === 'encrypt') {
     const { manifestFile, keyFile, decryptOracle } = argv as unknown as { manifestFile: string, keyFile: string, decryptOracle: string}
-    result = await integrationEncryptTestVectors(manifestFile, keyFile, decryptOracle, tolerateFailures, testName)
+    result = await integrationEncryptTestVectors(manifestFile, keyFile, decryptOracle, tolerateFailures, testName, concurrency)
   } else {
     console.log(`Unknown command ${command}`)
     cli.showHelp()

modules/integration-node/src/integration_tests.ts

@@ -72,57 +72,124 @@ export async function testEncryptVector ({ name, keysInfo, encryptOp, plainTextD
   }
 }
 
-export async function integrationDecryptTestVectors (vectorFile: string, tolerateFailures: number = 0, testName?: string) {
+export async function integrationDecryptTestVectors (vectorFile: string, tolerateFailures: number = 0, testName?: string, concurrency: number = 1) {
   const tests = await getDecryptTestVectorIterator(vectorFile)
-  let failureCount = 0
-  for (const test of tests) {
+
+  return parallelTests(concurrency, tolerateFailures, runTest, tests)
+
+  async function runTest (test: TestVectorInfo): Promise<boolean> {
     if (testName) {
-      if (test.name !== testName) continue
+      if (test.name !== testName) return true
     }
     const { result, name, err } = await testDecryptVector(test)
     if (result) {
       console.log({ name, result })
+      return true
     } else {
       if (err && notSupportedDecryptMessages.includes(err.message)) {
         console.log({ name, result: `Not supported: ${err.message}` })
-        continue
+        return true
       }
       console.log({ name, result, err })
-    }
-    if (!result) {
-      failureCount += 1
-      if (!tolerateFailures) return failureCount
-      tolerateFailures--
+      return false
     }
   }
-  return failureCount
 }
 
-export async function integrationEncryptTestVectors (manifestFile: string, keyFile: string, decryptOracle: string, tolerateFailures: number = 0, testName?: string) {
+export async function integrationEncryptTestVectors (manifestFile: string, keyFile: string, decryptOracle: string, tolerateFailures: number = 0, testName?: string, concurrency: number = 1) {
   const decryptOracleUrl = new URL(decryptOracle)
   const tests = await getEncryptTestVectorIterator(manifestFile, keyFile)
-  let failureCount = 0
-  for (const test of tests) {
+
+  return parallelTests(concurrency, tolerateFailures, runTest, tests)
+
+  async function runTest (test: EncryptTestVectorInfo): Promise<boolean> {
     if (testName) {
-      if (test.name !== testName) continue
+      if (test.name !== testName) return true
     }
     const { result, name, err } = await testEncryptVector(test, decryptOracleUrl)
     if (result) {
       console.log({ name, result })
+      return true
     } else {
       if (err && notSupportedEncryptMessages.includes(err.message)) {
         console.log({ name, result: `Not supported: ${err.message}` })
-        continue
+        return true
       }
       console.log({ name, result, err })
+      return false
     }
-    if (!result) {
-      failureCount += 1
-      if (!tolerateFailures) return failureCount
-      tolerateFailures--
-    }
   }
-  return failureCount
+}
+
+async function parallelTests<
+  Test extends EncryptTestVectorInfo|TestVectorInfo,
+  work extends (test: Test) => Promise<boolean>
+>(max: number, tolerateFailures: number, runTest: work, tests: IterableIterator<Test>) {
+  let _resolve: (failureCount: number) => void
+  const queue = new Set<Promise<void>>()
+  let failureCount = 0
+
+  return new Promise<number>((resolve) => {
+    _resolve = resolve
+    enqueue()
+  })
+
+  function enqueue (): void {
+    /* If there are more failures than I am willing to tolerate, stop. */
+    if (failureCount > tolerateFailures) return _resolve(failureCount)
+    /* Do not over-fill the queue! */
+    if (queue.size > max) return
+
+    const { value, done } = tests.next()
+    /* There is an edge here,
+     * you _could_ return a value *and* be done.
+     * Most iterators don't but in this case
+     * we just process the value and ask for another.
+     * Which will return done as true again.
+     */
+    if (!value && done) return _resolve(failureCount)
+
+    /* I need to define the work to be enqueue
+     * and a way to dequeue this work when complete.
+     * A Set of promises works nicely.
+     * Hold the variable here
+     * put it in the Set, take it out, and Bob's your uncle.
+     */
+    const work: Promise<void> = runTest(value)
+      .then((pass: boolean) => {
+        if (!pass) failureCount += 1
+      })
+      /* If there is some unknown error,
+       * it's just an error...
+       * Treat it like a test failure.
+       */
+      .catch((err) => {
+        console.log(err)
+        failureCount += 1
+      })
+      .then(() => {
+        /* Dequeue this work. */
+        queue.delete(work)
+        /* More to eat? */
+        enqueue()
+      })
+
+    /* Enqueue this work */
+    queue.add(work)
+
+    /* Fill the queue.
+     * The over-fill check above protects me.
+     * Sure, it is possible to exceed the stack depth.
+     * If you are trying to run ~10K tests in parallel
+     * on a system where that is actually faster,
+     * I want to talk to you.
+     * It is true that node can be configured to process
+     * > 10K HTTP requests no problem,
+     * but even the decrypt tests require that you first
+     * encrypt something locally before making the http call.
+     */
+    enqueue()
+  }
 }
 
 interface TestVectorResults {

package.json

@@ -31,8 +31,8 @@
     "integration-browser-decrypt": "npm run build; lerna run build_fixtures --stream --no-prefix -- -- decrypt -v $npm_package_config_localTestVectors --karma",
     "integration-browser-encrypt": "npm run build; lerna run build_fixtures --stream --no-prefix -- -- encrypt -m $npm_package_config_encryptManifestList -k $npm_package_config_encryptKeyManifest -o $npm_package_config_decryptOracle --karma",
     "browser-integration": "run-s integration-browser-*",
-    "integration-node-decrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- decrypt -v $npm_package_config_localTestVectors",
-    "integration-node-encrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- encrypt -m $npm_package_config_encryptManifestList -k $npm_package_config_encryptKeyManifest -o $npm_package_config_decryptOracle",
+    "integration-node-decrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- decrypt -v $npm_package_config_localTestVectors -c 10",
+    "integration-node-encrypt": "npm run build; lerna run integration_node --stream --no-prefix -- -- encrypt -m $npm_package_config_encryptManifestList -k $npm_package_config_encryptKeyManifest -o $npm_package_config_decryptOracle -c 20",
     "node-integration": "run-s integration-node-*",
     "integration": "run-s integration-*",
     "test_conditions": "./util/test_conditions"