diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b098810cd51..67bd5fd2ac9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -212,6 +212,29 @@ jobs: PATH=~/.npm-global/bin:$PATH npm install --global npm - run: PATH=~/.npm-global/bin:$PATH node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} + test-browsers: + executor: + name: test-executor + environment: + E2E_BROWSERS: true + steps: + - attach_workspace: *attach_options + - run: + name: Initialize Environment + command: ./.circleci/env.sh + - run: + name: Initialize Saucelabs + command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev) + - run: + name: Start Saucelabs Tunnel + command: ./scripts/saucelabs/start-tunnel.sh + background: true + # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests + # too early without Saucelabs not being ready. + - run: ./scripts/saucelabs/wait-for-tunnel.sh + - run: PATH=~/.npm-global/bin:$PATH node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts + - run: ./scripts/saucelabs/stop-tunnel.sh + build: executor: action-executor steps: @@ -346,6 +369,9 @@ workflows: <<: *ignore_pull_requests requires: - e2e-cli + - test-browsers: + requires: + - build - snapshot_publish: <<: *ignore_pull_requests requires: diff --git a/package.json b/package.json index c5080083cd12..d3393a199855 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "pidtree": "^0.3.0", "pidusage": "^2.0.17", "rxjs": "~6.4.0", + "sauce-connect": "https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz", "semver": "6.2.0", "source-map": "^0.5.6", "source-map-support": "^0.5.0", diff --git a/scripts/saucelabs/start-tunnel.sh b/scripts/saucelabs/start-tunnel.sh new file mode 100755 index 000000000000..34bf0a7b5bc9 --- /dev/null +++ b/scripts/saucelabs/start-tunnel.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -x -u -e -o pipefail + +readonly currentDir=$(cd $(dirname $0); pwd) + +# Command arguments that will be passed to sauce-connect. By default we disable SSL bumping for +# all requests. This is because SSL bumping is not needed for our test setup and in order +# to perform the SSL bumping, Saucelabs intercepts all HTTP requests in the tunnel VM and modifies +# them. This can cause flakiness as it makes all requests dependent on the SSL bumping middleware. +# See: https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect#TroubleshootingSauceConnect-DisablingSSLBumping +sauceArgs="--no-ssl-bump-domains all" + +if [[ ! -z "${SAUCE_LOG_FILE:-}" ]]; then + mkdir -p $(dirname ${SAUCE_LOG_FILE}) + sauceArgs="${sauceArgs} --logfile ${SAUCE_LOG_FILE}" +fi + +if [[ ! -z "${SAUCE_READY_FILE:-}" ]]; then + mkdir -p $(dirname ${SAUCE_READY_FILE}) + sauceArgs="${sauceArgs} --readyfile ${SAUCE_READY_FILE}" +fi + +if [[ ! -z "${SAUCE_PID_FILE:-}" ]]; then + mkdir -p $(dirname ${SAUCE_PID_FILE}) + sauceArgs="${sauceArgs} --pidfile ${SAUCE_PID_FILE}" +fi + +if [[ ! -z "${SAUCE_TUNNEL_IDENTIFIER:-}" ]]; then + sauceArgs="${sauceArgs} --tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}" +fi + +echo "Starting Sauce Connect. Passed arguments: ${sauceArgs}" + +${currentDir}/../../node_modules/sauce-connect/bin/sc -u ${SAUCE_USERNAME} -k ${SAUCE_ACCESS_KEY} ${sauceArgs} diff --git a/scripts/saucelabs/stop-tunnel.sh b/scripts/saucelabs/stop-tunnel.sh new file mode 100755 index 000000000000..c53a7e31ca08 --- /dev/null +++ b/scripts/saucelabs/stop-tunnel.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Disable printing of any executed command because this would cause a lot +# of spam due to the loop. +set +x -u -e -o pipefail + +if [[ ! -f ${SAUCE_PID_FILE} ]]; then + echo "Could not find Saucelabs tunnel PID file. Cannot stop tunnel.." + exit 1 +fi + +echo "Shutting down Sauce Connect tunnel" + +# The process id for the sauce-connect instance is stored inside of the pidfile. +tunnelProcessId=$(cat ${SAUCE_PID_FILE}) + +# Kill the process by using the PID that has been read from the pidfile. Note that +# we cannot use killall because CircleCI base container images don't have it installed. +kill ${tunnelProcessId} + +while (ps -p ${tunnelProcessId} &> /dev/null); do + printf "." + sleep .5 +done + +echo "" +echo "Sauce Connect tunnel has been shut down" diff --git a/scripts/saucelabs/wait-for-tunnel.sh b/scripts/saucelabs/wait-for-tunnel.sh new file mode 100755 index 000000000000..feda9a85b694 --- /dev/null +++ b/scripts/saucelabs/wait-for-tunnel.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Disable printing of any executed command because this would cause a lot +# of spam due to the loop. +set +x -u -e -o pipefail + +# Waits for Saucelabs Connect to be ready before executing any tests. +counter=0 + +while [[ ! -f ${SAUCE_READY_FILE} ]]; do + counter=$((counter + 1)) + + # Counter needs to be multiplied by two because the while loop only sleeps a half second. + # This has been made in favor of better progress logging (printing dots every half second) + if [ $counter -gt $[${SAUCE_READY_FILE_TIMEOUT} * 2] ]; then + echo "Timed out after ${SAUCE_READY_FILE_TIMEOUT} seconds waiting for tunnel ready file." + echo "Printing logfile output:" + echo "" + cat ${SAUCE_LOG_FILE} + exit 5 + fi + + printf "." + sleep 0.5 +done + +echo "" +echo "Connected to Saucelabs" diff --git a/tests/legacy-cli/e2e/assets/protractor-saucelabs.conf.js b/tests/legacy-cli/e2e/assets/protractor-saucelabs.conf.js new file mode 100644 index 000000000000..d3f5cd26834c --- /dev/null +++ b/tests/legacy-cli/e2e/assets/protractor-saucelabs.conf.js @@ -0,0 +1,120 @@ +// @ts-check +// Protractor configuration file, see link for more information +// https://github.com/angular/protractor/blob/master/lib/config.ts + +const { SpecReporter } = require('jasmine-spec-reporter'); + +const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER']; + +/** + * @type { import("protractor").Config } + */ +exports.config = { + sauceUser: process.env['SAUCE_USERNAME'], + sauceKey: process.env['SAUCE_ACCESS_KEY'], + + allScriptsTimeout: 11000, + specs: ['./src/**/*.e2e-spec.ts'], + + multiCapabilities: [ + { + browserName: 'chrome', + version: '41', + tunnelIdentifier, + }, + { + browserName: 'chrome', + version: '75', + tunnelIdentifier, + }, + { + browserName: 'safari', + platform: 'OS X 10.11', + version: '9.0', + tunnelIdentifier, + }, + // TODO: Investigate. Failure: + // Failed: Error while running testForAngular: undefined is not an object (evaluating 'd.prototype[b].apply') + // { + // browserName: 'safari', + // platform: 'OS X 10.12', + // version: '10.1', + // tunnelIdentifier, + // }, + { + browserName: 'safari', + platform: 'macOS 10.13', + version: '11.1', + tunnelIdentifier, + }, + { + browserName: 'safari', + platform: 'macOS 10.13', + version: '12.1', + tunnelIdentifier, + }, + { + browserName: 'firefox', + version: '48', + tunnelIdentifier, + }, + { + browserName: 'firefox', + version: '60', + tunnelIdentifier, + }, + { + browserName: 'firefox', + version: '68', + tunnelIdentifier, + }, + { + browserName: 'internet explorer', + platform: 'Windows 8', + version: '10', + tunnelIdentifier, + }, + { + browserName: 'internet explorer', + platform: 'Windows 8.1', + version: '11', + tunnelIdentifier, + }, + { + browserName: "MicrosoftEdge", + platform: 'Windows 10', + version: "14.14393", + tunnelIdentifier, + }, + { + browserName: "MicrosoftEdge", + platform: 'Windows 10', + version: "17.17134", + tunnelIdentifier, + }, + { + browserName: "MicrosoftEdge", + platform: 'Windows 10', + version: "18.17763", + tunnelIdentifier, + }, + ], + + baseUrl: 'http://localhost:2000/', + framework: 'jasmine', + jasmineNodeOpts: { + showColors: true, + defaultTimeoutInterval: 30000, + print: function() {}, + }, + + onPrepare() { + // Fix for Safari 12 -- https://github.com/angular/protractor/issues/4964 + browser.resetUrl = 'about:blank'; + + require('ts-node').register({ + project: require('path').join(__dirname, './tsconfig.json'), + }); + jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } })); + }, +}; diff --git a/tests/legacy-cli/e2e/tests/misc/browsers.ts b/tests/legacy-cli/e2e/tests/misc/browsers.ts new file mode 100644 index 000000000000..8b1c617d8b19 --- /dev/null +++ b/tests/legacy-cli/e2e/tests/misc/browsers.ts @@ -0,0 +1,63 @@ +import * as express from 'express'; +import * as path from 'path'; +import { copyProjectAsset } from '../../utils/assets'; +import { replaceInFile } from '../../utils/fs'; +import { ng } from '../../utils/process'; + +export default async function () { + if (!process.env['E2E_BROWSERS']) { + return; + } + + // Ensure SauceLabs configuration + if (!process.env['SAUCE_USERNAME'] || !process.env['SAUCE_ACCESS_KEY']) { + throw new Error('SauceLabs is not configured.'); + } + + await ng('build', '--prod'); + + // Add Protractor configuration + await copyProjectAsset('protractor-saucelabs.conf.js', 'e2e/protractor-saucelabs.conf.js'); + + // Remove browser log checks as they are only supported with the chrome webdriver + await replaceInFile( + 'e2e/src/app.e2e-spec.ts', + 'await browser.manage().logs().get(logging.Type.BROWSER)', + '[]', + ); + + // Workaround defect in getText WebDriver implementation for Safari/Edge + // Leading and trailing space is not removed + await replaceInFile( + 'e2e/src/app.e2e-spec.ts', + '\'should display welcome message\',', + '\'should display welcome message\', async', + ); + await replaceInFile( + 'e2e/src/app.e2e-spec.ts', + 'page.navigateTo();', + 'await page.navigateTo();', + ); + await replaceInFile( + 'e2e/src/app.e2e-spec.ts', + 'page.getTitleText()', + '(await page.getTitleText()).trim()', + ); + + // Setup server + const app = express(); + app.use(express.static(path.resolve('dist/test-project'))); + const server = app.listen(2000, 'localhost'); + + try { + // Execute application's E2E tests with SauceLabs + await ng( + 'e2e', + 'test-project', + '--protractorConfig=e2e/protractor-saucelabs.conf.js', + '--devServerTarget=', + ); + } finally { + server.close(); + } +} diff --git a/yarn.lock b/yarn.lock index 06bd6c064d00..4846aabd996a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9096,6 +9096,10 @@ sauce-connect-launcher@^1.2.4: lodash "^4.16.6" rimraf "^2.5.4" +"sauce-connect@https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz": + version "0.0.0" + resolved "https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz#dc5efcd2be24ddb099a85b923d6e754754651fa8" + saucelabs@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/saucelabs/-/saucelabs-1.5.0.tgz#9405a73c360d449b232839919a86c396d379fd9d"