feat: Encryption tests for integration-browser

https://github.com/awslabs/aws-crypto-tools-test-vector-framework defines an encryption manifest.
This manifest should be used to maintain cross compatibility of language implementations.

Then integrate this into the CI integration tests.
Also update package.json to better organize parameters.

Author: seebees

modules/integration-browser/karma.conf.js

@@ -6,13 +6,20 @@ module.exports = function (config) {
     basePath: '',
     frameworks: ['jasmine'],
     files: [
-      'fixtures/tests.json',
+      'fixtures/decrypt_tests.json',
+      'fixtures/encrypt_tests.json',
+      'fixtures/decrypt_oracle.json',
       { pattern: 'fixtures/*.json', included: false, served: true, watched: false, nocache: true },
-      'build/module/integration.test.js'
+      'build/module/integration.decrypt.test.js',
+      'build/module/integration.encrypt.test.js',
     ],
     preprocessors: {
-      'build/module/integration.test.js': ['webpack', 'credentials'],
-      './fixtures/tests.json': ['json_fixtures']
+      'build/module/integration.decrypt.test.js': ['webpack', 'credentials'],
+      'build/module/integration.encrypt.test.js': ['webpack', 'credentials'],
+      './fixtures/decrypt_tests.json': ['json_fixtures'],
+      './fixtures/encrypt_tests.json': ['json_fixtures'],
+      './fixtures/decrypt_oracle.json': ['json_fixtures']
+
     },
     webpack: {
       mode: 'development',

modules/integration-browser/package.json

@@ -18,21 +18,25 @@
     "@aws-crypto/client-browser": "^0.1.0-preview.2",
     "@aws-sdk/karma-credential-loader": "0.1.0-preview.2",
     "@aws-sdk/util-base64-browser": "0.1.0-preview.1",
+    "@aws-sdk/util-utf8-browser": "0.1.0-preview.1",
     "@trust/keyto": "^0.3.7",
+    "@types/got": "^9.6.2",
+    "@types/stream-to-promise": "^2.2.0",
     "@types/unzipper": "^0.9.1",
     "@types/yargs": "^13.0.0",
-    "stream-to-promise": "^2.2.0",
-    "tslib": "^1.9.3",
-    "unzipper": "^0.9.11",
-    "yargs": "^13.2.2",
+    "got": "^9.6.0",
     "jasmine-core": "^3.4.0",
     "karma": "^4.1.0",
     "karma-chrome-launcher": "^2.2.0",
     "karma-jasmine": "^2.0.1",
     "karma-json-fixtures-preprocessor": "0.0.6",
     "karma-webpack": "^3.0.5",
+    "puppeteer": "^1.14.0",
+    "stream-to-promise": "^2.2.0",
+    "tslib": "^1.9.3",
+    "unzipper": "^0.9.11",
     "webpack": "^4.30.0",
-    "puppeteer": "^1.14.0"
+    "yargs": "^13.2.2"
   },
   "devDependencies": {
     "@types/node": "^11.11.4",
@@ -46,10 +50,9 @@
   "main": "./build/main/index.js",
   "module": "./build/module/index.js",
   "types": "./build/main/index.d.ts",
-  "bin": "./build_fixtures",
+  "bin": "./build/main/cli.js",
   "files": [
     "build/**/*",
-    "build_fixtures",
     "karma.conf.js"
   ],
   "standard": {

modules/integration-browser/build_fixtures renamed to modules/integration-browser/src/build_decrypt_fixtures.ts

@@ -14,51 +14,21 @@
  * limitations under the License.
  */
 
-const argv = require('yargs')
-  .option('vectorFile', {
-    alias: 'v',
-    describe: 'a vector zip file from aws-encryption-sdk-test-vectors',
-    demandOption: true,
-    type: 'string'
-  })
-  .option('testName', {
-    alias: 't',
-    describe: 'an optional test name to execute',
-    type: 'string'
-  })
-  .option('slice', {
-    alias: 's',
-    describe: 'an optional range start:end e.g. 100:200',
-    type: 'string'
-  })
-  .options('karma', {
-    alias: 'k',
-    describe: 'start karma and run the tests',
-    type: 'boolean'
-  })
-  .argv
-
-const { vectorFile, testName, slice, karma } = argv
-const {Open} = require('unzipper')
-const streamToPromise = require('stream-to-promise')
-const fs = require('fs')
-const path = require('path')
-const { spawnSync } = require('child_process')
-const fixtures = path.join(__dirname, './fixtures')
-
-const [start=0, end=9999] = (slice || '').split(':').map(n => parseInt(n, 10))
-
-if (!fs.existsSync(fixtures)){
-  fs.mkdirSync(fixtures)
-}
+import { Open } from 'unzipper'
+import streamToPromise from 'stream-to-promise'
+import { writeFileSync } from 'fs'
+
+import { DecryptManifestList } from './types' // eslint-disable-line no-unused-vars
+
+export async function buildDecryptFixtures (fixtures: string, vectorFile: string, testName: string, slice: string) {
+  const [start = 0, end = 9999] = (slice || '').split(':').map(n => parseInt(n, 10))
 
-;(async () => {
   const centralDirectory = await Open.file(vectorFile)
   const filesMap = new Map(centralDirectory.files.map(file => [file.path, file]))
 
   const readUriOnce = (() => {
     const cache = new Map()
-    return async (uri) => {
+    return async (uri: string) => {
       const has = cache.get(uri)
       if (has) return has
       const fileInfo = filesMap.get(testUri2Path(uri))
@@ -70,7 +40,7 @@ if (!fs.existsSync(fixtures)){
   })()
 
   const manifestBuffer = await readUriOnce('manifest.json')
-  const { keys: keysFile, tests } = JSON.parse(manifestBuffer.toString('utf8'))
+  const { keys: keysFile, tests }: DecryptManifestList = JSON.parse(manifestBuffer.toString('utf8'))
   const keysBuffer = await readUriOnce(keysFile)
   const { keys } = JSON.parse(keysBuffer.toString('utf8'))
   const testNames = []
@@ -94,8 +64,8 @@ if (!fs.existsSync(fixtures)){
     const plainTextInfo = filesMap.get(testUri2Path(plaintextFile))
     const cipherInfo = filesMap.get(testUri2Path(ciphertext))
     if (!cipherInfo || !plainTextInfo) throw new Error(`no file for ${name}: ${ciphertext} | ${plaintextFile}`)
-    
-    const cipherText = await streamToPromise(cipherInfo.stream())
+
+    const cipherText = await streamToPromise(<NodeJS.ReadableStream>cipherInfo.stream())
     const plainText = await readUriOnce(plainTextInfo.path)
     const keysInfo = masterKeys.map(keyInfo => {
       const key = keys[keyInfo.key]
@@ -111,19 +81,12 @@ if (!fs.existsSync(fixtures)){
       plainText: plainText.toString('base64')
     })
 
-    fs.writeFileSync(`${fixtures}/${name}.json`, test)
+    writeFileSync(`${fixtures}/${name}.json`, test)
   }
 
-  fs.writeFileSync(`${fixtures}/tests.json`, JSON.stringify(testNames))
-
-  if (karma) {
-    spawnSync('npm', ['run', 'karma'], {
-      cwd: __dirname,
-      stdio: 'inherit'
-    })
-  }
-})()
+  writeFileSync(`${fixtures}/decrypt_tests.json`, JSON.stringify(testNames))
+}
 
-function testUri2Path (uri) {
+function testUri2Path (uri: string) {
   return uri.replace('file://', '')
 }

modules/integration-browser/src/build_encrypt_fixtures.ts

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ * this file except in compliance with the License. A copy of the License is
+ * located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  EncryptManifestList, // eslint-disable-line no-unused-vars
+  KeyList, // eslint-disable-line no-unused-vars
+  KeyInfoTuple // eslint-disable-line no-unused-vars
+} from './types'
+import { randomBytes } from 'crypto'
+import {
+  AlgorithmSuiteIdentifier, // eslint-disable-line no-unused-vars
+  EncryptionContext // eslint-disable-line no-unused-vars
+} from '@aws-crypto/client-browser'
+import { URL } from 'url'
+import { readFileSync, writeFileSync } from 'fs'
+import got from 'got'
+
+export async function buildEncryptFixtures (fixtures: string, manifestFile: string, keyFile: string, testName: string, slice: string) {
+  const [start = 0, end = 9999] = (slice || '').split(':').map(n => parseInt(n, 10))
+  const { tests, plaintexts }: EncryptManifestList = await getParsedJSON(manifestFile)
+  const { keys }: KeyList = await getParsedJSON(keyFile)
+
+  const plaintextBytes: {[name: string]: string} = {}
+
+  Object
+    .keys(plaintexts)
+    .forEach(name => {
+      plaintextBytes[name] = randomBytes(10).toString('base64')
+    })
+
+  const testNames = []
+  let count = 0
+
+  for (const [name, testInfo] of Object.entries(tests)) {
+    count += 1
+
+    if (testName) {
+      if (name !== testName) continue
+    }
+
+    if (slice) {
+      if (start >= count) continue
+      if (count > end) continue
+    }
+
+    testNames.push(name)
+
+    const {
+      plaintext,
+      'master-keys': masterKeys,
+      algorithm,
+      'frame-size': frameLength,
+      'encryption-context': encryptionContext
+    } = testInfo
+
+    const keysInfo = <KeyInfoTuple[]>masterKeys.map(keyInfo => {
+      const key = keys[keyInfo.key]
+      if (!key) throw new Error(`no key for ${name}`)
+      return [keyInfo, key]
+    })
+
+    /* I'm expecting that the encrypt function will throw if this is not a supported AlgorithmSuiteIdentifier */
+    const suiteId = <AlgorithmSuiteIdentifier>parseInt(algorithm, 16)
+
+    const test: EncryptTestVectorInfo = {
+      name,
+      keysInfo,
+      plainTextData: plaintextBytes[plaintext],
+      encryptOp: { suiteId, frameLength, encryptionContext }
+    }
+
+    writeFileSync(`${fixtures}/${name}.json`, JSON.stringify(test))
+  }
+
+  writeFileSync(`${fixtures}/encrypt_tests.json`, JSON.stringify(testNames))
+}
+
+export interface EncryptTestVectorInfo {
+  name: string,
+  keysInfo: KeyInfoTuple[],
+  plainTextData: string,
+  encryptOp: {
+    suiteId: AlgorithmSuiteIdentifier,
+    frameLength: number,
+    encryptionContext: EncryptionContext
+  }
+}
+
+async function getParsedJSON (thing: string) {
+  try {
+    const url = new URL(thing)
+    if (url.protocol === 'file:') {
+      return jsonAtPath(thing)
+    } else {
+      return jsonAtUrl(url)
+    }
+  } catch (ex) {
+    return jsonAtPath(thing)
+  }
+}
+async function jsonAtUrl (url: URL) {
+  const { body } = await got(url)
+  return JSON.parse(body)
+}
+
+function jsonAtPath (path: string) {
+  const json = readFileSync(path, { encoding: 'utf-8' })
+  return JSON.parse(json)
+}

modules/integration-browser/src/cli.ts

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ * this file except in compliance with the License. A copy of the License is
+ * located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import yargs from 'yargs'
+import { spawnSync } from 'child_process'
+
+import { join } from 'path'
+import { existsSync, mkdirSync, writeFileSync } from 'fs'
+import { buildDecryptFixtures } from './build_decrypt_fixtures'
+import { buildEncryptFixtures } from './build_encrypt_fixtures'
+
+const cli = yargs
+  .command('decrypt', 'verify decrypt vectors', y => y
+    .option('vectorFile', {
+      alias: 'v',
+      describe: 'a vector zip file from aws-encryption-sdk-test-vectors',
+      demandOption: true,
+      type: 'string'
+    })
+  )
+  .command('encrypt', 'verify encrypt manifest', y => y
+    .option('manifestFile', {
+      alias: 'm',
+      describe: 'a path/url to aws-crypto-tools-test-vector-framework canonical manifest',
+      demandOption: true,
+      type: 'string'
+    })
+    .option('keyFile', {
+      alias: 'k',
+      describe: 'a path/url to aws-crypto-tools-test-vector-framework canonical key list',
+      demandOption: true,
+      type: 'string'
+    })
+    .option('decryptOracle', {
+      alias: 'o',
+      describe: 'a url to the decrypt oracle',
+      demandOption: true,
+      type: 'string'
+    })
+  )
+  .option('testName', {
+    alias: 't',
+    describe: 'an optional test name to execute',
+    type: 'string'
+  })
+  .option('slice', {
+    alias: 's',
+    describe: 'an optional range start:end e.g. 100:200',
+    type: 'string'
+  })
+  .options('karma', {
+    describe: 'start karma and run the tests',
+    type: 'boolean'
+  })
+  .demandCommand()
+const fixtures = join(__dirname, '../../fixtures')
+/* Sad side effect. */
+if (!existsSync(fixtures)) {
+  mkdirSync(fixtures)
+}
+
+;(async (argv) => {
+  const { _: [ command ], testName, slice, karma, decryptOracle = '' } = argv
+
+  writeFileSync(`${fixtures}/decrypt_tests.json`, JSON.stringify([]))
+  writeFileSync(`${fixtures}/encrypt_tests.json`, JSON.stringify([]))
+  writeFileSync(`${fixtures}/decrypt_oracle.json`, JSON.stringify(decryptOracle))
+
+  if (command === 'decrypt') {
+    const { vectorFile } = argv
+    // @ts-ignore
+    await buildDecryptFixtures(fixtures, vectorFile, testName, slice)
+  } else if (command === 'encrypt') {
+    const { manifestFile, keyFile } = argv
+    // @ts-ignore
+    await buildEncryptFixtures(fixtures, manifestFile, keyFile, testName, slice)
+  } else {
+    console.log(`Unknown command ${command}`)
+    cli.showHelp()
+  }
+
+  if (karma) {
+    spawnSync('npm', ['run', 'karma'], {
+      cwd: __dirname,
+      stdio: 'inherit'
+    })
+  }
+})(cli.argv)