run e2e tests under bazel (#24338)

* test: run legacy-cli e2e tests via bazel

* fixup! test: run legacy-cli e2e tests via bazel

* fixup! test: run legacy-cli e2e tests via bazel

* fixup! test: run legacy-cli e2e tests via bazel

Author: jbedard

.bazelrc

@@ -90,6 +90,10 @@ build:snapshot --workspace_status_command="yarn -s ng-dev release build-env-stam
 build:snapshot --stamp
 build:snapshot --//:enable_snapshot_repo_deps
 
+build:e2e --workspace_status_command="yarn -s ng-dev release build-env-stamp --mode=release"
+build:e2e --stamp
+test:e2e  --test_timeout=3600
+
 build:local --//:enable_package_json_tar_deps
 
 ###############################

.circleci/dynamic_config.yml

@@ -35,19 +35,27 @@ var_3_all_major: &all_nodeversion_major ['14', '16']
 # https://circleci.com/blog/deep-diving-into-circleci-workspaces/
 var_4: &workspace_location .
 # Filter to only release branches on a given job.
-var_5: &only_release_branches
+var_5_only_releases: &only_release_branches
   filters:
     branches:
       only:
         - main
         - /\d+\.\d+\.x/
+var_5_only_snapshots: &only_snapshot_branches
+  filters:
+    branches:
+      only:
+        - main
+        # This is needed to run this steps on Renovate PRs that amend the snapshots package.json
+        - /^pull\/.*/
 
 var_6: &only_pull_requests
   filters:
     branches:
       only:
         - /pull\/\d+/
 
+# All e2e test suites
 var_7: &all_e2e_subsets ['npm', 'esbuild', 'yarn']
 
 # Executor Definitions
@@ -63,10 +71,20 @@ executors:
     working_directory: ~/ng
     resource_class: small
 
+  bazel-executor:
+    parameters:
+      nodeversion:
+        type: string
+        default: *default_nodeversion
+    docker:
+      - image: cimg/node:<< parameters.nodeversion >>-browsers
+    working_directory: ~/ng
+    resource_class: xlarge
+
   windows-executor:
     # Same as https://circleci.com/orbs/registry/orb/circleci/windows, but named.
     working_directory: ~/ng
-    resource_class: windows.medium
+    resource_class: windows.large
     shell: powershell.exe -ExecutionPolicy Bypass
     machine:
       # Contents of this image:
@@ -116,7 +134,7 @@ commands:
       - initialize_env
       - run: nvm install 16.13
       - run: nvm use 16.13
-      - run: npm install -g yarn@1.22.10
+      - run: npm install -g yarn@1.22.10 @bazel/bazelisk@${BAZELISK_VERSION}
       - run: node --version
       - run: yarn --version
 
@@ -126,6 +144,7 @@ commands:
         type: env_var_name
         default: CIRCLE_PROJECT_REPONAME
     steps:
+      - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
       - devinfra/setup-bazel-remote-exec:
           bazelrc: ./.bazelrc.user
 
@@ -269,23 +288,24 @@ jobs:
           paths:
             - dist/_*.tgz
 
-  build-bazel-e2e:
-    executor: action-executor
-    resource_class: medium
+  bazel-build:
+    executor: bazel-executor
     steps:
       - custom_attach_workspace
-      - run: yarn bazel build //tests/legacy-cli/...
+      - setup_bazel_rbe
+      - run:
+          name: Bazel Build Packages
+          command: yarn bazel build //...
+      - fail_fast
 
-  unit-test:
-    executor: action-executor
-    resource_class: xlarge
+  bazel-test:
+    executor: bazel-executor
     parameters:
       nodeversion:
         type: string
         default: *default_nodeversion_major
     steps:
       - custom_attach_workspace
-      - browser-tools/install-chrome
       - setup_bazel_rbe
       - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
       - when:
@@ -311,6 +331,58 @@ jobs:
                 no_output_timeout: 40m
       - fail_fast
 
+  bazel-e2e-tests:
+    executor: bazel-executor
+    parallelism: 8
+    parameters:
+      snapshots:
+        type: boolean
+        default: false
+      subset:
+        type: enum
+        enum: *all_e2e_subsets
+        default: 'npm'
+    steps:
+      - custom_attach_workspace
+      - initialize_env
+      - setup_bazel_rbe
+      - run: mkdir /mnt/ramdisk/e2e
+      - run:
+          name: Execute CLI E2E Tests with << parameters.subset >>
+          command: yarn bazel test --define=E2E_TEMP=/mnt/ramdisk/e2e --define=E2E_SHARD_TOTAL=${CIRCLE_NODE_TOTAL} --define=E2E_SHARD_INDEX=${CIRCLE_NODE_INDEX} --config=e2e //tests/legacy-cli:e2e<<# parameters.snapshots >>.snapshots<</ parameters.snapshots >>.<< parameters.subset >>
+          no_output_timeout: 40m
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+      - fail_fast
+
+  bazel-test-browsers:
+    executor: bazel-executor
+    steps:
+      - custom_attach_workspace
+      - initialize_env
+      - setup_bazel_rbe
+      - run:
+          name: Initialize Saucelabs
+          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - run:
+          name: Start Saucelabs Tunnel
+          command: ./scripts/saucelabs/start-tunnel.sh
+          background: true
+        # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
+        # too early without Saucelabs not being ready.
+      - run: ./scripts/saucelabs/wait-for-tunnel.sh
+      - run:
+          name: E2E Saucelabs Tests
+          command: yarn bazel test --config=saucelabs //tests/legacy-cli:e2e.saucelabs
+      - run: ./scripts/saucelabs/stop-tunnel.sh
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.saucelabs
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.saucelabs
+      - fail_fast
+
   snapshot_publish:
     executor: action-executor
     resource_class: medium
@@ -375,6 +447,48 @@ jobs:
             node tests\legacy-cli\run_e2e.js --nb-shards=$env:CIRCLE_NODE_TOTAL --shard=$env:CIRCLE_NODE_INDEX --tmpdir=X:/ramdisk/e2e-main --ignore="tests/misc/browsers.ts"
       - fail_fast
 
+  bazel-e2e-cli-win:
+    executor: windows-executor
+    parallelism: 12
+    steps:
+      - checkout
+      - rebase_pr_win
+      - setup_windows
+      - restore_cache:
+          keys:
+            - *cache_key_win
+      - run:
+          # We use Arsenal Image Mounter (AIM) instead of ImDisk because of: https://github.com/nodejs/node/issues/6861
+          # Useful resources for AIM: http://reboot.pro/index.php?showtopic=22068
+          name: 'Arsenal Image Mounter (RAM Disk)'
+          command: |
+            pwsh ./.circleci/win-ram-disk.ps1
+      - run: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
+      - save_cache:
+          key: *cache_key_win
+          paths:
+            - ~/.cache/yarn
+            # Path where Arsenal Image Mounter files are downloaded.
+            # Must match path in .circleci/win-ram-disk.ps1
+            - ./aim
+      - run:
+          name: Execute E2E Tests
+          environment:
+            # Required by `yarn ng-dev`
+            # See https://github.com/angular/angular/issues/46858
+            PWD: .
+          command: |
+            mkdir X:/ramdisk/e2e
+            bazel test --define=E2E_TEMP=X:/ramdisk/e2e --define=E2E_SHARD_TOTAL=$env:CIRCLE_NODE_TOTAL --define=E2E_SHARD_INDEX=$env:CIRCLE_NODE_INDEX --config=e2e //tests/legacy-cli:e2e.npm
+          # This timeout provides time for the actual tests to timeout and report status
+          # instead of CircleCI stopping the job without test failure information.
+          no_output_timeout: 40m
+      - fail_fast
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.npm
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.npm
+
 workflows:
   version: 2
   default_workflow:
@@ -429,44 +543,73 @@ workflows:
           snapshots: true
           pre-steps:
             - when:
+                # Don't run snapshot E2E's unless it's on the main branch or the snapshots file has been updated.
                 condition:
                   and:
                     - not:
                         equal: [main, << pipeline.git.branch >>]
                     - not: << pipeline.parameters.snapshot_changed >>
                 steps:
-                  # Don't run snapshot E2E's unless it's on the main branch or the snapshots file has been updated.
                   - run: circleci-agent step halt
           requires:
             - build
-          filters:
-            branches:
-              only:
-                - main
-                # This is needed to run this steps on Renovate PRs that amend the snapshots package.json
-                - /^pull\/.*/
+          <<: *only_snapshot_branches
 
       # Bazel jobs
       # These jobs only really depend on Setup, but the build job is very quick to run (~35s) and
       # will catch any build errors before proceeding to the more lengthy and resource intensive
       # Bazel jobs.
-      - unit-test:
+      - bazel-test:
           name: test-node<< matrix.nodeversion >>
           matrix:
             parameters:
               nodeversion: *all_nodeversion_major
           requires:
-            - build
-
-      # Compile the e2e tests with bazel to ensure the non-runtime typescript
-      # compilation completes succesfully.
-      - build-bazel-e2e:
-          requires:
-            - build
+            - bazel-build
 
       # Windows jobs
       - e2e-cli-win
 
+      - bazel-e2e-cli-win
+
+      # Bazel jobs
+      - bazel-build:
+          requires:
+            - setup
+
+      - bazel-e2e-tests:
+          name: bazel-e2e-cli-<< matrix.subset >>
+          matrix:
+            parameters:
+              subset: *all_e2e_subsets
+          snapshots: false
+          requires:
+            - bazel-build
+
+      - bazel-e2e-tests:
+          name: bazel-e2e-snapshots-<< matrix.subset >>
+          matrix:
+            parameters:
+              subset: *all_e2e_subsets
+          snapshots: true
+          pre-steps:
+            - when:
+                # Don't run snapshot E2E's unless it's on the main branch or the snapshots file has been updated.
+                condition:
+                  and:
+                    - not:
+                        equal: [main, << pipeline.git.branch >>]
+                    - not: << pipeline.parameters.snapshot_changed >>
+                steps:
+                  - run: circleci-agent step halt
+          requires:
+            - bazel-build
+          <<: *only_snapshot_branches
+
+      - bazel-test-browsers:
+          requires:
+            - bazel-build
+
       # Publish jobs
       - snapshot_publish:
           <<: *only_release_branches

.circleci/env.sh

@@ -36,3 +36,8 @@ source $BASH_ENV;
 
 # Disable husky.
 setPublicVar HUSKY 0
+
+# Expose the Bazelisk version. We need to run Bazelisk globally since Windows has problems launching
+# Bazel from a node modules directoy that might be modified by the Bazel Yarn install then.
+setPublicVar BAZELISK_VERSION \
+    "$(cd ${PROJECT_ROOT}; node -p 'require("./package.json").devDependencies["@bazel/bazelisk"]')"

.circleci/win-ram-disk.ps1

@@ -26,5 +26,6 @@ if (-not (Test-Path -Path $aimContents)) {
 ./aim/cli/x64/aim_ll.exe --install ./aim/drivers
 
 # Setup RAM disk mount. Same parameters as ImDisk
+# Ensure size is large enough to support the bazel 'shard_count's such as for e2e tests.
 # See: https://support.circleci.com/hc/en-us/articles/4411520952091-Create-a-windows-RAM-disk
-./aim/cli/x64/aim_ll.exe -a -s 5G -m X: -p "/fs:ntfs /q /y"
+./aim/cli/x64/aim_ll.exe -a -s 12G -m X: -p "/fs:ntfs /q /y"

BUILD.bazel

@@ -3,6 +3,7 @@
 # Use of this source code is governed by an MIT-style license that can be
 # found in the LICENSE file at https://angular.io/license
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -16,6 +17,14 @@ exports_files([
     "package.json",
 ])
 
+# Files required by e2e tests
+copy_to_bin(
+    name = "config-files",
+    srcs = [
+        "package.json",
+    ],
+)
+
 # Detect if the build is running under --stamp
 config_setting(
     name = "stamp",

WORKSPACE

@@ -98,3 +98,14 @@ nodejs_register_toolchains(
     name = "node16",
     node_version = "16.13.1",
 )
+
+register_toolchains(
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_linux_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_x86_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_arm64_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_windows_toolchain",
+)
+
+load("@npm//@angular/build-tooling/bazel/browsers:browser_repositories.bzl", "browser_repositories")
+
+browser_repositories()

docs/DEVELOPER.md

@@ -85,7 +85,7 @@ You can find more info about debugging [tests with Bazel in the docs.](https://g
 - Compile the packages being tested: `yarn build`
 - Run all tests: `node tests/legacy-cli/run_e2e.js`
 - Run a subset of the tests: `node tests/legacy-cli/run_e2e.js tests/legacy-cli/e2e/tests/i18n/ivy-localize-*`
-- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz ...`
+- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz tests/legacy-cli/e2e/tests/i18n/ivy-localize-*`
 
 When running the debug commands, Node will stop and wait for a debugger to attach.
 You can attach your IDE to the debugger to stop on breakpoints and step through the code. Also, see [IDE Specific Usage](#ide-specific-usage) for a

packages/angular/cli/src/utilities/version.ts

@@ -23,7 +23,7 @@ class Version {
   }
 }
 
-// TODO: Convert this to use build-time version stamping after flipping the build script to use bazel
+// TODO(bazel): Convert this to use build-time version stamping after flipping the build script to use bazel
 // export const VERSION = new Version('0.0.0-PLACEHOLDER');
 export const VERSION = new Version(
   (