test: integrate browser testing for differential loading

.circleci/config.yml

@@ -212,6 +212,29 @@ jobs:
             PATH=~/.npm-global/bin:$PATH npm install --global npm
       - run: PATH=~/.npm-global/bin:$PATH node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX}
 
+  test-browsers:
+    executor:
+      name: test-executor
+    environment:
+      E2E_BROWSERS: true
+    steps:
+      - attach_workspace: *attach_options
+      - run:
+          name: Initialize Environment
+          command: ./.circleci/env.sh
+      - run:
+          name: Initialize Saucelabs
+          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - run:
+          name: Start Saucelabs Tunnel
+          command: ./scripts/saucelabs/start-tunnel.sh
+          background: true
+        # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
+        # too early without Saucelabs not being ready.
+      - run: ./scripts/saucelabs/wait-for-tunnel.sh
+      - run: PATH=~/.npm-global/bin:$PATH node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts
+      - run: ./scripts/saucelabs/stop-tunnel.sh
+
   build:
     executor: action-executor
     steps:
@@ -346,6 +369,9 @@ workflows:
           <<: *ignore_pull_requests
           requires:
             - e2e-cli
+      - test-browsers:
+          requires:
+            - build
       - snapshot_publish:
           <<: *ignore_pull_requests
           requires:

package.json

@@ -128,6 +128,7 @@
     "pidtree": "^0.3.0",
     "pidusage": "^2.0.17",
     "rxjs": "~6.4.0",
+    "sauce-connect": "https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz",
     "semver": "6.2.0",
     "source-map": "^0.5.6",
     "source-map-support": "^0.5.0",

scripts/saucelabs/start-tunnel.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -x -u -e -o pipefail
+
+readonly currentDir=$(cd $(dirname $0); pwd)
+
+# Command arguments that will be passed to sauce-connect. By default we disable SSL bumping for
+# all requests. This is because SSL bumping is not needed for our test setup and in order
+# to perform the SSL bumping, Saucelabs intercepts all HTTP requests in the tunnel VM and modifies
+# them. This can cause flakiness as it makes all requests dependent on the SSL bumping middleware.
+# See: https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect#TroubleshootingSauceConnect-DisablingSSLBumping
+sauceArgs="--no-ssl-bump-domains all"
+
+if [[ ! -z "${SAUCE_LOG_FILE:-}" ]]; then
+  mkdir -p $(dirname ${SAUCE_LOG_FILE})
+  sauceArgs="${sauceArgs} --logfile ${SAUCE_LOG_FILE}"
+fi
+
+if [[ ! -z "${SAUCE_READY_FILE:-}" ]]; then
+  mkdir -p $(dirname ${SAUCE_READY_FILE})
+  sauceArgs="${sauceArgs} --readyfile ${SAUCE_READY_FILE}"
+fi
+
+if [[ ! -z "${SAUCE_PID_FILE:-}" ]]; then
+  mkdir -p $(dirname ${SAUCE_PID_FILE})
+  sauceArgs="${sauceArgs} --pidfile ${SAUCE_PID_FILE}"
+fi
+
+if [[ ! -z "${SAUCE_TUNNEL_IDENTIFIER:-}" ]]; then
+  sauceArgs="${sauceArgs} --tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}"
+fi
+
+echo "Starting Sauce Connect. Passed arguments: ${sauceArgs}"
+
+${currentDir}/../../node_modules/sauce-connect/bin/sc -u ${SAUCE_USERNAME} -k ${SAUCE_ACCESS_KEY} ${sauceArgs}

scripts/saucelabs/stop-tunnel.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Disable printing of any executed command because this would cause a lot
+# of spam due to the loop.
+set +x -u -e -o pipefail
+
+if [[ ! -f ${SAUCE_PID_FILE} ]]; then
+  echo "Could not find Saucelabs tunnel PID file. Cannot stop tunnel.."
+  exit 1
+fi
+
+echo "Shutting down Sauce Connect tunnel"
+
+# The process id for the sauce-connect instance is stored inside of the pidfile.
+tunnelProcessId=$(cat ${SAUCE_PID_FILE})
+
+# Kill the process by using the PID that has been read from the pidfile. Note that
+# we cannot use killall because CircleCI base container images don't have it installed.
+kill ${tunnelProcessId}
+
+while (ps -p ${tunnelProcessId} &> /dev/null); do
+  printf "."
+  sleep .5
+done
+
+echo ""
+echo "Sauce Connect tunnel has been shut down"

scripts/saucelabs/wait-for-tunnel.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Disable printing of any executed command because this would cause a lot
+# of spam due to the loop.
+set +x -u -e -o pipefail
+
+# Waits for Saucelabs Connect to be ready before executing any tests.
+counter=0
+
+while [[ ! -f ${SAUCE_READY_FILE} ]]; do
+  counter=$((counter + 1))
+
+  # Counter needs to be multiplied by two because the while loop only sleeps a half second.
+  # This has been made in favor of better progress logging (printing dots every half second)
+  if [ $counter -gt $[${SAUCE_READY_FILE_TIMEOUT} * 2] ]; then
+    echo "Timed out after ${SAUCE_READY_FILE_TIMEOUT} seconds waiting for tunnel ready file."
+    echo "Printing logfile output:"
+    echo ""
+    cat ${SAUCE_LOG_FILE}
+    exit 5
+  fi
+
+  printf "."
+  sleep 0.5
+done
+
+echo ""
+echo "Connected to Saucelabs"

tests/legacy-cli/e2e/assets/protractor-saucelabs.conf.js

@@ -0,0 +1,120 @@
+// @ts-check
+// Protractor configuration file, see link for more information
+// https://github.com/angular/protractor/blob/master/lib/config.ts
+
+const { SpecReporter } = require('jasmine-spec-reporter');
+
+const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER'];
+
+/**
+ * @type { import("protractor").Config }
+ */
+exports.config = {
+  sauceUser: process.env['SAUCE_USERNAME'],
+  sauceKey: process.env['SAUCE_ACCESS_KEY'],
+
+  allScriptsTimeout: 11000,
+  specs: ['./src/**/*.e2e-spec.ts'],
+
+  multiCapabilities: [
+    {
+      browserName: 'chrome',
+      version: '41',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'chrome',
+      version: '75',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'safari',
+      platform: 'OS X 10.11',
+      version: '9.0',
+      tunnelIdentifier,
+    },
+    // TODO: Investigate. Failure:
+    // Failed: Error while running testForAngular: undefined is not an object (evaluating 'd.prototype[b].apply')
+    // {
+    //   browserName: 'safari',
+    //   platform: 'OS X 10.12',
+    //   version: '10.1',
+    //   tunnelIdentifier,
+    // },
+    {
+      browserName: 'safari',
+      platform: 'macOS 10.13',
+      version: '11.1',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'safari',
+      platform: 'macOS 10.13',
+      version: '12.1',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'firefox',
+      version: '48',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'firefox',
+      version: '60',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'firefox',
+      version: '68',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'internet explorer',
+      platform: 'Windows 8',
+      version: '10',
+      tunnelIdentifier,
+    },
+    {
+      browserName: 'internet explorer',
+      platform: 'Windows 8.1',
+      version: '11',
+      tunnelIdentifier,
+    },
+    {
+      browserName: "MicrosoftEdge",
+      platform: 'Windows 10',
+      version: "14.14393",
+      tunnelIdentifier,
+    },
+    {
+      browserName: "MicrosoftEdge",
+      platform: 'Windows 10',
+      version: "17.17134",
+      tunnelIdentifier,
+    },
+    {
+      browserName: "MicrosoftEdge",
+      platform: 'Windows 10',
+      version: "18.17763",
+      tunnelIdentifier,
+    },
+  ],
+
+  baseUrl: 'http://localhost:2000/',
+  framework: 'jasmine',
+  jasmineNodeOpts: {
+    showColors: true,
+    defaultTimeoutInterval: 30000,
+    print: function() {},
+  },
+
+  onPrepare() {
+    // Fix for Safari 12 -- https://github.com/angular/protractor/issues/4964
+    browser.resetUrl = 'about:blank';
+
+    require('ts-node').register({
+      project: require('path').join(__dirname, './tsconfig.json'),
+    });
+    jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } }));
+  },
+};

tests/legacy-cli/e2e/tests/misc/browsers.ts

@@ -0,0 +1,63 @@
+import * as express from 'express';
+import * as path from 'path';
+import { copyProjectAsset } from '../../utils/assets';
+import { replaceInFile } from '../../utils/fs';
+import { ng } from '../../utils/process';
+
+export default async function () {
+  if (!process.env['E2E_BROWSERS']) {
+    return;
+  }
+
+  // Ensure SauceLabs configuration
+  if (!process.env['SAUCE_USERNAME'] || !process.env['SAUCE_ACCESS_KEY']) {
+    throw new Error('SauceLabs is not configured.');
+  }
+
+  await ng('build', '--prod');
+
+  // Add Protractor configuration
+  await copyProjectAsset('protractor-saucelabs.conf.js', 'e2e/protractor-saucelabs.conf.js');
+
+  // Remove browser log checks as they are only supported with the chrome webdriver
+  await replaceInFile(
+    'e2e/src/app.e2e-spec.ts',
+    'await browser.manage().logs().get(logging.Type.BROWSER)',
+    '[]',
+  );
+
+  // Workaround defect in getText WebDriver implementation for Safari/Edge
+  // Leading and trailing space is not removed
+  await replaceInFile(
+    'e2e/src/app.e2e-spec.ts',
+    '\'should display welcome message\',',
+    '\'should display welcome message\', async',
+  );
+  await replaceInFile(
+    'e2e/src/app.e2e-spec.ts',
+    'page.navigateTo();',
+    'await page.navigateTo();',
+  );
+  await replaceInFile(
+    'e2e/src/app.e2e-spec.ts',
+    'page.getTitleText()',
+    '(await page.getTitleText()).trim()',
+  );
+
+  // Setup server
+  const app = express();
+  app.use(express.static(path.resolve('dist/test-project')));
+  const server = app.listen(2000, 'localhost');
+
+  try {
+    // Execute application's E2E tests with SauceLabs
+    await ng(
+      'e2e',
+      'test-project',
+      '--protractorConfig=e2e/protractor-saucelabs.conf.js',
+      '--devServerTarget=',
+    );
+  } finally {
+    server.close();
+  }
+}

yarn.lock

@@ -9096,6 +9096,10 @@ sauce-connect-launcher@^1.2.4:
     lodash "^4.16.6"
     rimraf "^2.5.4"
 
+"sauce-connect@https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz":
+  version "0.0.0"
+  resolved "https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz#dc5efcd2be24ddb099a85b923d6e754754651fa8"
+
 saucelabs@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/saucelabs/-/saucelabs-1.5.0.tgz#9405a73c360d449b232839919a86c396d379fd9d"