feat: Encryption tests for integration-browser (#159)

https://github.com/awslabs/aws-crypto-tools-test-vector-framework defines an encryption manifest.
This manifest should be used to maintain cross compatibility of language implementations.

Then integrate this into the CI integration tests.
Also update package.json to better organize parameters.

Author: seebees

modules/integration-browser/karma.conf.js

@@ -6,13 +6,20 @@ module.exports = function (config) {
     basePath: '',
     frameworks: ['jasmine'],
     files: [
-      'fixtures/tests.json',
+      'fixtures/decrypt_tests.json',
+      'fixtures/encrypt_tests.json',
+      'fixtures/decrypt_oracle.json',
       { pattern: 'fixtures/*.json', included: false, served: true, watched: false, nocache: true },
-      'build/module/integration.test.js'
+      'build/module/integration.decrypt.test.js',
+      'build/module/integration.encrypt.test.js',
     ],
     preprocessors: {
-      'build/module/integration.test.js': ['webpack', 'credentials'],
-      './fixtures/tests.json': ['json_fixtures']
+      'build/module/integration.decrypt.test.js': ['webpack', 'credentials'],
+      'build/module/integration.encrypt.test.js': ['webpack', 'credentials'],
+      './fixtures/decrypt_tests.json': ['json_fixtures'],
+      './fixtures/encrypt_tests.json': ['json_fixtures'],
+      './fixtures/decrypt_oracle.json': ['json_fixtures']
+
     },
     webpack: {
       mode: 'development',

modules/integration-browser/package.json

@@ -18,21 +18,25 @@
     "@aws-crypto/client-browser": "^0.1.0-preview.2",
     "@aws-sdk/karma-credential-loader": "0.1.0-preview.2",
     "@aws-sdk/util-base64-browser": "0.1.0-preview.1",
+    "@aws-sdk/util-utf8-browser": "0.1.0-preview.1",
     "@trust/keyto": "^0.3.7",
+    "@types/got": "^9.6.2",
+    "@types/stream-to-promise": "^2.2.0",
     "@types/unzipper": "^0.9.1",
     "@types/yargs": "^13.0.0",
-    "stream-to-promise": "^2.2.0",
-    "tslib": "^1.9.3",
-    "unzipper": "^0.9.11",
-    "yargs": "^13.2.2",
+    "got": "^9.6.0",
     "jasmine-core": "^3.4.0",
     "karma": "^4.1.0",
     "karma-chrome-launcher": "^2.2.0",
     "karma-jasmine": "^2.0.1",
     "karma-json-fixtures-preprocessor": "0.0.6",
     "karma-webpack": "^3.0.5",
+    "puppeteer": "^1.14.0",
+    "stream-to-promise": "^2.2.0",
+    "tslib": "^1.9.3",
+    "unzipper": "^0.9.11",
     "webpack": "^4.30.0",
-    "puppeteer": "^1.14.0"
+    "yargs": "^13.2.2"
   },
   "devDependencies": {
     "@types/node": "^11.11.4",
@@ -46,10 +50,9 @@
   "main": "./build/main/index.js",
   "module": "./build/module/index.js",
   "types": "./build/main/index.d.ts",
-  "bin": "./build_fixtures",
+  "bin": "./build/main/cli.js",
   "files": [
     "build/**/*",
-    "build_fixtures",
     "karma.conf.js"
   ],
   "standard": {

modules/integration-browser/build_fixtures renamed to modules/integration-browser/src/build_decrypt_fixtures.ts

@@ -14,51 +14,28 @@
  * limitations under the License.
  */
 
-const argv = require('yargs')
-  .option('vectorFile', {
-    alias: 'v',
-    describe: 'a vector zip file from aws-encryption-sdk-test-vectors',
-    demandOption: true,
-    type: 'string'
-  })
-  .option('testName', {
-    alias: 't',
-    describe: 'an optional test name to execute',
-    type: 'string'
-  })
-  .option('slice', {
-    alias: 's',
-    describe: 'an optional range start:end e.g. 100:200',
-    type: 'string'
-  })
-  .options('karma', {
-    alias: 'k',
-    describe: 'start karma and run the tests',
-    type: 'boolean'
-  })
-  .argv
-
-const { vectorFile, testName, slice, karma } = argv
-const {Open} = require('unzipper')
-const streamToPromise = require('stream-to-promise')
-const fs = require('fs')
-const path = require('path')
-const { spawnSync } = require('child_process')
-const fixtures = path.join(__dirname, './fixtures')
-
-const [start=0, end=9999] = (slice || '').split(':').map(n => parseInt(n, 10))
-
-if (!fs.existsSync(fixtures)){
-  fs.mkdirSync(fixtures)
-}
+import { Open } from 'unzipper'
+import streamToPromise from 'stream-to-promise'
+import { writeFileSync } from 'fs'
+
+import { DecryptManifestList } from './types' // eslint-disable-line no-unused-vars
+
+/* This function interacts with manifest information
+ * and produces the fixtures in the `fixtures`
+ * that the karma server will consume to run tests.
+ * This gives us 2 useful freedoms.
+ * 1. The code is not tied to a specific copy of the manifest information
+ * 2. The tests can be run on a subset of tests for debugging.
+ */
+export async function buildDecryptFixtures (fixtures: string, vectorFile: string, testName: string, slice: string) {
+  const [start = 0, end = 9999] = (slice || '').split(':').map(n => parseInt(n, 10))
 
-;(async () => {
   const centralDirectory = await Open.file(vectorFile)
   const filesMap = new Map(centralDirectory.files.map(file => [file.path, file]))
 
   const readUriOnce = (() => {
     const cache = new Map()
-    return async (uri) => {
+    return async (uri: string) => {
       const has = cache.get(uri)
       if (has) return has
       const fileInfo = filesMap.get(testUri2Path(uri))
@@ -70,7 +47,7 @@ if (!fs.existsSync(fixtures)){
   })()
 
   const manifestBuffer = await readUriOnce('manifest.json')
-  const { keys: keysFile, tests } = JSON.parse(manifestBuffer.toString('utf8'))
+  const { keys: keysFile, tests }: DecryptManifestList = JSON.parse(manifestBuffer.toString('utf8'))
   const keysBuffer = await readUriOnce(keysFile)
   const { keys } = JSON.parse(keysBuffer.toString('utf8'))
   const testNames = []
@@ -94,8 +71,8 @@ if (!fs.existsSync(fixtures)){
     const plainTextInfo = filesMap.get(testUri2Path(plaintextFile))
     const cipherInfo = filesMap.get(testUri2Path(ciphertext))
     if (!cipherInfo || !plainTextInfo) throw new Error(`no file for ${name}: ${ciphertext} | ${plaintextFile}`)
-    
-    const cipherText = await streamToPromise(cipherInfo.stream())
+
+    const cipherText = await streamToPromise(<NodeJS.ReadableStream>cipherInfo.stream())
     const plainText = await readUriOnce(plainTextInfo.path)
     const keysInfo = masterKeys.map(keyInfo => {
       const key = keys[keyInfo.key]
@@ -111,19 +88,12 @@ if (!fs.existsSync(fixtures)){
       plainText: plainText.toString('base64')
     })
 
-    fs.writeFileSync(`${fixtures}/${name}.json`, test)
+    writeFileSync(`${fixtures}/${name}.json`, test)
   }
 
-  fs.writeFileSync(`${fixtures}/tests.json`, JSON.stringify(testNames))
-
-  if (karma) {
-    spawnSync('npm', ['run', 'karma'], {
-      cwd: __dirname,
-      stdio: 'inherit'
-    })
-  }
-})()
+  writeFileSync(`${fixtures}/decrypt_tests.json`, JSON.stringify(testNames))
+}
 
-function testUri2Path (uri) {
+function testUri2Path (uri: string) {
   return uri.replace('file://', '')
 }

modules/integration-browser/src/build_encrypt_fixtures.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ * this file except in compliance with the License. A copy of the License is
+ * located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  EncryptManifestList, // eslint-disable-line no-unused-vars
+  KeyList, // eslint-disable-line no-unused-vars
+  KeyInfoTuple // eslint-disable-line no-unused-vars
+} from './types'
+import { randomBytes } from 'crypto'
+import {
+  AlgorithmSuiteIdentifier, // eslint-disable-line no-unused-vars
+  EncryptionContext // eslint-disable-line no-unused-vars
+} from '@aws-crypto/client-browser'
+import { URL } from 'url'
+import { readFileSync, writeFileSync } from 'fs'
+import got from 'got'
+
+/* This function interacts with manifest information
+ * and produces the fixtures in the `fixtures`
+ * that the karma server will consume to run tests.
+ * This gives us 2 useful freedoms.
+ * 1. The code is not tied to a specific copy of the manifest information
+ * 2. The tests can be run on a subset of tests for debugging.
+ */
+export async function buildEncryptFixtures (fixtures: string, manifestFile: string, keyFile: string, testName: string, slice: string) {
+  const [start = 0, end = 9999] = (slice || '').split(':').map(n => parseInt(n, 10))
+  const { tests, plaintexts }: EncryptManifestList = await getParsedJSON(manifestFile)
+  const { keys }: KeyList = await getParsedJSON(keyFile)
+
+  const plaintextBytes: {[name: string]: string} = {}
+
+  Object
+    .keys(plaintexts)
+    .forEach(name => {
+      /* Generate random bites as per spec.
+       * See: https://github.com/awslabs/aws-crypto-tools-test-vector-framework/blob/master/features/0003-awses-message-encryption.md#plaintexts
+       */
+      plaintextBytes[name] = randomBytes(10).toString('base64')
+    })
+
+  const testNames = []
+  let count = 0
+
+  for (const [name, testInfo] of Object.entries(tests)) {
+    count += 1
+
+    if (testName) {
+      if (name !== testName) continue
+    }
+
+    if (slice) {
+      if (start >= count) continue
+      if (count > end) continue
+    }
+
+    testNames.push(name)
+
+    const {
+      plaintext,
+      'master-keys': masterKeys,
+      algorithm,
+      'frame-size': frameLength,
+      'encryption-context': encryptionContext
+    } = testInfo
+
+    const keysInfo = <KeyInfoTuple[]>masterKeys.map(keyInfo => {
+      const key = keys[keyInfo.key]
+      if (!key) throw new Error(`no key for ${name}`)
+      return [keyInfo, key]
+    })
+
+    /* I'm expecting that the encrypt function will throw if this is not a supported AlgorithmSuiteIdentifier */
+    const suiteId = <AlgorithmSuiteIdentifier>parseInt(algorithm, 16)
+
+    const test: EncryptTestVectorInfo = {
+      name,
+      keysInfo,
+      plainTextData: plaintextBytes[plaintext],
+      encryptOp: { suiteId, frameLength, encryptionContext }
+    }
+
+    writeFileSync(`${fixtures}/${name}.json`, JSON.stringify(test))
+  }
+
+  writeFileSync(`${fixtures}/encrypt_tests.json`, JSON.stringify(testNames))
+}
+
+export interface EncryptTestVectorInfo {
+  name: string,
+  keysInfo: KeyInfoTuple[],
+  plainTextData: string,
+  encryptOp: {
+    suiteId: AlgorithmSuiteIdentifier,
+    frameLength: number,
+    encryptionContext: EncryptionContext
+  }
+}
+
+async function getParsedJSON (thing: string) {
+  try {
+    const url = new URL(thing)
+    if (url.protocol === 'file:') {
+      return jsonAtPath(thing)
+    } else {
+      return jsonAtUrl(url)
+    }
+  } catch (ex) {
+    return jsonAtPath(thing)
+  }
+}
+async function jsonAtUrl (url: URL) {
+  const { body } = await got(url)
+  return JSON.parse(body)
+}
+
+function jsonAtPath (path: string) {
+  const json = readFileSync(path, { encoding: 'utf-8' })
+  return JSON.parse(json)
+}