From 8ea84def8cc2196b27c05fce2e1bd687dd70b36e Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Sat, 19 Feb 2022 14:25:46 +0100 Subject: [PATCH 1/2] ci: setup automatic deployment for the docs app See the following document for the overall plan that this commit implements: https://docs.google.com/document/d/1xkrSOFa6WeFqyg1cTwMhl_wB8ygbVwdSxr3K2-cps14/edit --- .circleci/config.yml | 46 ++++- package.json | 9 +- scripts/build-docs-content.js | 52 ----- scripts/build-docs-content.ts | 69 +++++++ scripts/build-packages-dist.ts | 14 +- scripts/deploy/publish-docs-content.sh | 2 +- scripts/docs-deploy/README.md | 3 + scripts/docs-deploy/clone-docs-repo.ts | 63 ++++++ scripts/docs-deploy/deploy-ci-push.ts | 117 +++++++++++ scripts/docs-deploy/deploy-to-site.ts | 75 +++++++ scripts/docs-deploy/docs-deps-install.ts | 20 ++ scripts/docs-deploy/github-versioning.ts | 23 +++ scripts/docs-deploy/install-built-packages.ts | 35 ++++ scripts/docs-deploy/monitoring/ci-test.ts | 8 + scripts/docs-deploy/monitoring/index.ts | 37 ++++ scripts/docs-deploy/snapshot-deploy.ts | 73 +++++++ scripts/docs-deploy/update-versions-file.ts | 64 ++++++ scripts/docs-deploy/utils.ts | 52 +++++ yarn.lock | 191 ++++++------------ 19 files changed, 757 insertions(+), 196 deletions(-) delete mode 100644 scripts/build-docs-content.js create mode 100644 scripts/build-docs-content.ts create mode 100644 scripts/docs-deploy/README.md create mode 100644 scripts/docs-deploy/clone-docs-repo.ts create mode 100644 scripts/docs-deploy/deploy-ci-push.ts create mode 100644 scripts/docs-deploy/deploy-to-site.ts create mode 100644 scripts/docs-deploy/docs-deps-install.ts create mode 100644 scripts/docs-deploy/github-versioning.ts create mode 100644 scripts/docs-deploy/install-built-packages.ts create mode 100644 scripts/docs-deploy/monitoring/ci-test.ts create mode 100644 scripts/docs-deploy/monitoring/index.ts create mode 100644 scripts/docs-deploy/snapshot-deploy.ts create mode 100644 scripts/docs-deploy/update-versions-file.ts create mode 100644 scripts/docs-deploy/utils.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 24dd23d38532..671844ae5011 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,8 +86,6 @@ var_14: &publish_branches_filter - master # 6.0.x, 7.1.x, etc. - /\d+\.\d+\.x/ - # 6.x, 7.x, 8.x etc - - /\d+\.x/ # Branch filter that is usually applied to all jobs. Since there is no way within CircleCI to # exclude a branch for all defined jobs, we need to manually specify the filters for each job. @@ -395,6 +393,42 @@ jobs: - store_artifacts: path: dist/release-archives + # ---------------------------------------- + # Job that publishes the docs site + # ---------------------------------------- + deploy_docs_site: + docker: + - image: *docker-browser-image + resource_class: xlarge + environment: + GCP_DECRYPT_TOKEN: *gcp_decrypt_token + steps: + - checkout_and_rebase + - *restore_cache + - *setup_bazel_ci_config + - *setup_bazel_remote_execution + - *yarn_install + - *setup_bazel_binary + + - run: yarn ci-push-deploy-docs-app + - *slack_notify_on_failure + + # ---------------------------------------- + # Job that monitors the docs site, ensuring + # the docs site is online and works as expected. + # ---------------------------------------- + monitor_docs_site: + docker: + - image: *docker-browser-image + resource_class: xlarge + steps: + - checkout_and_rebase + - *restore_cache + - *yarn_install + + - run: yarn ci-docs-monitor-test + - *slack_notify_on_failure + # ---------------------------------------- # Job that publishes the build snapshots # ---------------------------------------- @@ -609,6 +643,12 @@ workflows: filters: *publish_branches_filter requires: - build_release_packages + - deploy_docs_site: + filters: *publish_branches_filter + requires: + - lint + - build_release_packages + - tests_local_browsers # Snapshot tests workflow that is scheduled to run all specified jobs every hour. # This workflow runs various jobs against the Angular snapshot builds from Github. @@ -623,6 +663,8 @@ workflows: filters: *only_main_branch_filter - snapshot_linker_tests: filters: *only_main_branch_filter + - monitor_docs_site: + filters: *only_main_branch_filter triggers: - schedule: diff --git a/package.json b/package.json index 3e706811d8c1..84d869feb051 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "postinstall": "node tools/postinstall/apply-patches.js", "build": "ts-node --project scripts/tsconfig.json ./scripts/build-packages-dist.ts", "build-and-check-release-output": "ts-node --project scripts/tsconfig.json scripts/build-and-check-release-output.ts", - "build-docs-content": "node ./scripts/build-docs-content.js", + "build-docs-content": "ts-node --project scripts/tsconfig.json ./scripts/build-docs-content.ts", "dev-app": "ibazel run //src/dev-app:devserver", "test": "node ./scripts/run-component-tests.js", "test-local": "yarn -s test --local", @@ -48,6 +48,8 @@ "check-mdc-exports": "ts-node --project scripts/tsconfig.json scripts/check-mdc-exports.ts", "check-tooling-setup": "yarn tsc --project tools/tsconfig.json && yarn tsc --project .ng-dev/tsconfig.json", "tsc": "node ./node_modules/typescript/bin/tsc", + "ci-push-deploy-docs-app": "ts-node --project scripts/tsconfig.json scripts/docs-deploy/deploy-ci-push.ts", + "ci-docs-monitor-test": "ts-node --project scripts/tsconfig.json scripts/docs-deploy/monitoring/ci-test.ts", "prepare": "husky install" }, "version": "14.0.0-next.6", @@ -73,7 +75,7 @@ "@angular/bazel": "14.0.0-next.6", "@angular/cli": "14.0.0-next.5", "@angular/compiler-cli": "14.0.0-next.6", - "@angular/dev-infra-private": "https://github.com/angular/dev-infra-private-builds.git#b6656cffbd46bb3637b09b08561e5f1a147603c9", + "@angular/dev-infra-private": "https://github.com/angular/dev-infra-private-builds.git#7544378ad9aa94cdfe256aaaa923c107f45175a2", "@angular/localize": "14.0.0-next.6", "@angular/platform-browser-dynamic": "14.0.0-next.6", "@angular/platform-server": "14.0.0-next.6", @@ -218,7 +220,8 @@ "typescript-4.5": "npm:typescript@4.5.5", "vrsource-tslint-rules": "6.0.0", "yaml": "^1.10.2", - "yargs": "^17.3.1" + "yargs": "^17.3.1", + "zx": "^4.3.0" }, "resolutions": { "@angular/dev-infra-private/typescript": "~4.6.2", diff --git a/scripts/build-docs-content.js b/scripts/build-docs-content.js deleted file mode 100644 index 81df15bac87f..000000000000 --- a/scripts/build-docs-content.js +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env node - -/** - * Script that builds the docs content NPM package and moves it into an conveniently - * accessible distribution directory (the project `dist/` directory). - */ - -const {join} = require('path'); -const {chmod, cd, cp, mkdir, rm, set, exec} = require('shelljs'); - -/** Path to the project directory. */ -const projectDir = join(__dirname, '../'); - -/** Path to the distribution directory. */ -const distDir = join(projectDir, 'dist/'); - -/** - * Path to the directory where the docs-content package is copied to. Note: When - * changing the path, also change the path in the docs-content deploy script. - */ -const outputDir = join(distDir, 'docs-content-pkg'); - -/** Command that runs Bazel. */ -const bazelCmd = process.env.BAZEL_COMMAND || `yarn -s bazel`; - -// ShellJS should exit if a command fails. -set('-e'); - -// Go to project directory. -cd(projectDir); - -/** Path to the bazel bin output directory. */ -const bazelBinPath = exec(`${bazelCmd} info bazel-bin`).stdout.trim(); - -/** Path where the NPM package is built into by Bazel. */ -const bazelBinOutDir = join(bazelBinPath, 'src/components-examples/npm_package'); - -// Build the docs-content package with the snapshot-build mode. That will help -// determining which commit is associated with the built docs-content. -exec(`${bazelCmd} build src/components-examples:npm_package --config=snapshot-build`); - -// Clean the output directory to ensure that the docs-content package -// will not contain outdated files from previous builds. -rm('-rf', outputDir); -mkdir('-p', distDir); - -// Copy the package output into the dist path. Also update the permissions -// as Bazel by default marks files in the bazel-out as readonly. -cp('-R', bazelBinOutDir, outputDir); -chmod('-R', 'u+w', outputDir); - -console.info(`Built docs-content into: ${outputDir}`); diff --git a/scripts/build-docs-content.ts b/scripts/build-docs-content.ts new file mode 100644 index 000000000000..f4dc7e50940a --- /dev/null +++ b/scripts/build-docs-content.ts @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +/** + * Script that builds the docs content NPM package and moves it into a conveniently + * accessible distribution directory (the project `dist/` directory). + */ + +import {cd, chmod, cp, exec, mkdir, rm, set} from 'shelljs'; + +import {BuiltPackage} from '@angular/dev-infra-private/ng-dev'; +import {join} from 'path'; + +/** Path to the project directory. */ +const projectDir = join(__dirname, '../'); + +/** Path to the distribution directory. */ +const distDir = join(projectDir, 'dist/'); + +/** + * Path to the directory where the docs-content package is copied to. Note: When + * changing the path, also change the path in the docs-content deploy script. + */ +const outputDir = join(distDir, 'docs-content-pkg'); + +/** Command that runs Bazel. */ +const bazelCmd = process.env.BAZEL || `yarn -s bazel`; + +/** + * Builds the docs content NPM package in snapshot mode. + * + * @returns an object describing the built package and its output path. + */ +export function buildDocsContentPackage(): BuiltPackage { + // ShellJS should exit if a command fails. + set('-e'); + + // Go to project directory. + cd(projectDir); + + /** Path to the bazel bin output directory. */ + const bazelBinPath = exec(`${bazelCmd} info bazel-bin`).stdout.trim(); + + /** Path where the NPM package is built into by Bazel. */ + const bazelBinOutDir = join(bazelBinPath, 'src/components-examples/npm_package'); + + // Clean the output directory to ensure that the docs-content package + // will not contain outdated files from previous builds. + rm('-rf', outputDir); + mkdir('-p', distDir); + + // Build the docs-content package with the snapshot-build mode. That will help + // determining which commit is associated with the built docs-content. + exec(`${bazelCmd} build src/components-examples:npm_package --config=snapshot-build`); + + // Copy the package output into the dist path. Also update the permissions + // as Bazel by default marks files in the bazel-out as readonly. + cp('-R', bazelBinOutDir, outputDir); + chmod('-R', 'u+w', outputDir); + + return { + name: '@angular/components-examples', + outputPath: outputDir, + }; +} + +if (require.main === module) { + const builtPackage = buildDocsContentPackage(); + console.info(`Built docs-content into: ${builtPackage.outputPath}`); +} diff --git a/scripts/build-packages-dist.ts b/scripts/build-packages-dist.ts index da0b0b589080..24f3cb135de0 100755 --- a/scripts/build-packages-dist.ts +++ b/scripts/build-packages-dist.ts @@ -21,7 +21,7 @@ const releaseTargetTag = 'release-package'; const projectDir = join(__dirname, '../'); /** Command that runs Bazel. */ -const bazelCmd = process.env.BAZEL_COMMAND || `bazel`; +const bazelCmd = process.env.BAZEL || `yarn -s bazel`; /** Command that queries Bazel for all release package targets. */ const queryPackagesCmd = @@ -69,7 +69,8 @@ function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): Built const targets = exec(queryPackagesCmd, true).split(/\r?\n/); const packageNames = getPackageNamesOfTargets(targets); const bazelBinPath = exec(`${bazelCmd} info bazel-bin`, true); - const getOutputPath = (pkgName: string) => join(bazelBinPath, 'src', pkgName, 'npm_package'); + const getBazelOutputPath = (pkgName: string) => join(bazelBinPath, 'src', pkgName, 'npm_package'); + const getDistPath = (pkgName: string) => join(distPath, pkgName); // Build with "--config=release" or `--config=snapshot-build` so that Bazel // runs the workspace stamping script. The stamping script ensures that the @@ -80,7 +81,7 @@ function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): Built // a workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1219. We need to // do this to ensure that the version placeholders are properly populated. packageNames.forEach(pkgName => { - const outputPath = getOutputPath(pkgName); + const outputPath = getBazelOutputPath(pkgName); if (test('-d', outputPath)) { chmod('-R', 'u+w', outputPath); rm('-rf', outputPath); @@ -96,18 +97,17 @@ function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): Built // Copy the package output into the specified distribution folder. packageNames.forEach(pkgName => { - const outputPath = getOutputPath(pkgName); - const targetFolder = join(distPath, pkgName); + const outputPath = getBazelOutputPath(pkgName); + const targetFolder = getDistPath(pkgName); console.log(`> Copying package output to "${targetFolder}"`); cp('-R', outputPath, targetFolder); chmod('-R', 'u+w', targetFolder); }); return packageNames.map(pkg => { - const outputPath = getOutputPath(pkg); return { name: `@angular/${pkg}`, - outputPath, + outputPath: getDistPath(pkg), }; }); } diff --git a/scripts/deploy/publish-docs-content.sh b/scripts/deploy/publish-docs-content.sh index 1a7de44bd287..dd68f75a6f9a 100755 --- a/scripts/deploy/publish-docs-content.sh +++ b/scripts/deploy/publish-docs-content.sh @@ -24,7 +24,7 @@ docsDistPath="${projectPath}/dist/docs" docsContentPath="${projectPath}/tmp/material2-docs-content" # Path to the build output of the Bazel "@angular/components-examples" NPM package. -# Note: When changing this, also change the path in `scripts/build-docs-content.js`. +# Note: When changing this, also change the path in `scripts/build-docs-content.ts`. examplesPackagePath="${projectPath}/dist/docs-content-pkg/" # Git clone URL for the material2-docs-content repository. diff --git a/scripts/docs-deploy/README.md b/scripts/docs-deploy/README.md new file mode 100644 index 000000000000..6d46476c6f93 --- /dev/null +++ b/scripts/docs-deploy/README.md @@ -0,0 +1,3 @@ +### Docs app deployment + +https://docs.google.com/document/d/1xkrSOFa6WeFqyg1cTwMhl_wB8ygbVwdSxr3K2-cps14/edit?usp=sharing diff --git a/scripts/docs-deploy/clone-docs-repo.ts b/scripts/docs-deploy/clone-docs-repo.ts new file mode 100644 index 000000000000..5989fc038a1b --- /dev/null +++ b/scripts/docs-deploy/clone-docs-repo.ts @@ -0,0 +1,63 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import {$} from 'zx'; +import {projectDir} from './utils'; + +/** Git repository HTTP url pointing to the docs repository. */ +export const docsRepoUrl = 'https://github.com/angular/material.angular.io.git'; + +/** + * Clones the docs repository for the given major into a + * temporary directory. + * + * @returns An absolute path to the temporary directory. + */ +export async function cloneDocsRepositoryForMajor(major: number): Promise { + const repoTmpDir = path.join(projectDir, 'tmp/docs-repo'); + const baseCloneArgs = [docsRepoUrl, repoTmpDir, '--single-branch', '--depth=1']; + const majorDocsBranchName = getDocsBranchNameForMajor(major); + + // Cleanup the temporary repository directory if it exists. + try { + await fs.promises.rm(repoTmpDir, {recursive: true}); + } catch {} + + // Clone the docs app (either the main branch, or a dedicated major branch if available). + if (await hasUpstreamDocsBranch(majorDocsBranchName)) { + console.log(`Cloning docs app with dedicated branch: ${majorDocsBranchName}`); + await $`git clone ${baseCloneArgs} --branch=${majorDocsBranchName}`; + } else { + console.log(`Cloning docs app with default branch (no dedicated branch for major).`); + await $`git clone ${baseCloneArgs}`; + } + + return repoTmpDir; +} + +/** + * Gets whether the specified branch exists in the specified remote URL. + */ +async function hasUpstreamDocsBranch(branchName: string): Promise { + try { + const proc = await $`git ls-remote ${docsRepoUrl} refs/heads/${branchName}`; + return proc.stdout.trim() !== ''; + } catch { + return false; + } +} + +/** + * Gets the name of a potential dedicated branch for this major in the + * docs repository. + * + * e.g. if a branch like `13.x` exists and we intend to deploy v13, then + * this branch can be used as revision for the docs-app. + * + * More details on why this is preferred: + * https://docs.google.com/document/d/1xkrSOFa6WeFqyg1cTwMhl_wB8ygbVwdSxr3K2-cps14/edit#heading=h.nsf3ag63jpwu. + */ +function getDocsBranchNameForMajor(major: number): string { + return 'firebase-target'; + // TODO return `${major}.x`; +} diff --git a/scripts/docs-deploy/deploy-ci-push.ts b/scripts/docs-deploy/deploy-ci-push.ts new file mode 100644 index 000000000000..f38d486ace30 --- /dev/null +++ b/scripts/docs-deploy/deploy-ci-push.ts @@ -0,0 +1,117 @@ +#!/usr/bin/env node + +import { + fetchActiveReleaseTrains, + getBranchesForMajorVersions, + getVersionForVersionBranch, + isVersionBranch, +} from '@angular/dev-infra-private/ng-dev'; +import {firebaseConfig, sites} from './utils'; + +import {buildAndDeployWithSnapshots} from './snapshot-deploy'; +import {getReleaseRepoWithApi} from './github-versioning'; +import {updateVersionsFile} from './update-versions-file'; + +async function main() { + if (process.env.CIRCLE_PR_NUMBER !== undefined) { + console.log('Skipping deployment for pull request build.'); + return; + } + + const branchName = process.env.CIRCLE_BRANCH; + if (branchName === undefined) { + throw new Error('Deployment script is unable to determine CI branch.'); + } + + const repo = getReleaseRepoWithApi(); + const active = await fetchActiveReleaseTrains(repo); + const description = `${branchName} - ${process.env.CIRCLE_SHA1!}`; + const {projectId, serviceKey} = firebaseConfig; + + if (branchName === active.next.branchName) { + const major = active.next.version.major; + const targets = [{projectId, description, site: sites.next}]; + + // If the next release train is for a new major that is not published as part of the + // other active release trains, we also publish to e.g. `v14.material.angular.io`. + // Note: If both `rc` and `next` have the same major, we want the `rc` release-train + // to take precedence and be responsible for the `.material.angular.io` deploy. + if (major > (active.releaseCandidate ?? active.latest).version.major) { + targets.push({projectId, description, site: sites.forMajor(major)}); + } + + await buildAndDeployWithSnapshots(serviceKey, major, targets); + return; + } + + if (branchName === active.latest.branchName) { + const major = active.latest.version.major; + const targets = [ + {projectId, description, site: sites.stable}, + {projectId, description, site: sites.forMajor(major)}, + ]; + + // If there is no active RC train, we also push the current stable to the `rc` site. + // TODO: This can be improved by using redirects from rc -> stable. + if (active.releaseCandidate === null) { + targets.push({projectId, description, site: sites.rc}); + } + + await buildAndDeployWithSnapshots(serviceKey, major, targets, { + // For the stable deployment, we want to update the versions file which is loaded + // by all docs sites (archives, rc, next etc.). The source of truth for all versions + // shown in the "docs version picker" is the versions file deployed in stable. + prebuild: docsRepoDir => updateVersionsFile(docsRepoDir, active), + }); + return; + } + + if (branchName === active.releaseCandidate?.branchName) { + const major = active.releaseCandidate.version.major; + const targets = [{projectId, description, site: sites.rc}]; + + // If the RC is for a new major that `latest` does not publish yet, we will deploy + // the dedicated major site like `v13.material.angular.io` using the `rc` branch. + // Note: If both `rc` and `next` have the same major, we want the `rc` release-train + // to take precedence and be responsible for the `.material.angular.io` deploy. + if (major > active.latest.version.major) { + targets.push({projectId, description, site: sites.forMajor(major)}); + } + + await buildAndDeployWithSnapshots(serviceKey, major, targets); + return; + } + + // In other cases, we are potentially deploying an archived major, regardless + // of LTS being active or not. + if (!isVersionBranch(branchName)) { + console.log('Skipping deployment as the current branch is not a version branch.'); + return; + } + + const branchVersion = getVersionForVersionBranch(branchName)!; + const branchMajor = branchVersion.major; + const branchesForMajor = await getBranchesForMajorVersions(repo, [branchMajor]); + + // The `branchesForMajor` array will hold the most recent version branch for current major. + // If the latest branch does not match the current one, we know that this is not the + // most recent minor and should not be re-deployed to e.g. `.material.angular.io`. + if (branchesForMajor[0].name !== branchName) { + console.log( + `Skipping deployment as version branch is not the most recent ` + + `one for v${branchMajor}. Expected: ${branchesForMajor[0].name}`, + ); + return; + } + + await buildAndDeployWithSnapshots(serviceKey, branchMajor, [ + {projectId, description, site: sites.forMajor(branchMajor)}, + ]); +} + +if (require.main === module) { + main().catch(e => { + console.error(e); + process.exitCode = 1; + }); +} diff --git a/scripts/docs-deploy/deploy-to-site.ts b/scripts/docs-deploy/deploy-to-site.ts new file mode 100644 index 000000000000..ed91f08912e6 --- /dev/null +++ b/scripts/docs-deploy/deploy-to-site.ts @@ -0,0 +1,75 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import {$} from 'zx'; +import {SiteTarget} from './utils'; + +interface Deployment { + projectId: string; + site: SiteTarget; +} + +/** Interface describing a production deployment. */ +export interface ProductionDeployment extends Deployment { + description: string; +} + +/** Interface describing a temporary preview deployment. */ +export interface PreviewDeployment extends Deployment { + channelId: string; + expires: string; +} + +/** Type describing a Firebase deployment. */ +export type DeploymentInfo = ProductionDeployment | PreviewDeployment; + +/** Path to a temporary file for the GCP service key credentials file. */ +const gcpServiceKeyTmpFile = path.join(os.tmpdir(), 'mat-docs-deploy-gcp-key.json'); + +/** + * Deploys the docs site at the specified directory to Firebase with respect + * to the deployment information provided. + * + * The deployment info either describes the production deployment information, + * or a preview temporary deployment that will expire automatically. + */ +export async function deployToSite( + projectPath: string, + firebaseServiceKey: string, + info: DeploymentInfo, +) { + const firebase = async (...cmd: string[]) => + $`yarn --cwd ${projectPath} firebase --non-interactive ${cmd}`; + + // Setup GCP service key for the docs-app deployment. + // https://firebase.google.com/docs/admin/setup. + await fs.promises.writeFile(gcpServiceKeyTmpFile, firebaseServiceKey); + process.env.GOOGLE_APPLICATION_CREDENTIALS = gcpServiceKeyTmpFile; + + await firebase('use', info.projectId); + await firebase('target:clear', 'hosting', 'mat-aio'); + await firebase('target:apply', 'hosting', 'mat-aio', info.site.firebaseSiteId); + + if (isPreviewDeployment(info)) { + const channelId = info.channelId; + const expires = info.expires; + + await firebase('hosting:channel:deploy', channelId, '--only', 'mat-aio', '--expires', expires); + } else { + await firebase('deploy', '--only', 'hosting:mat-aio', '--message', info.description); + } + + // Remove the temporary service key file (this is an optional step and just for sanity). + await fs.promises.rm(gcpServiceKeyTmpFile, {force: true}); +} + +/** Whether the given deployment info corresponds to a preview deployment. */ +export function isPreviewDeployment(info: DeploymentInfo): info is PreviewDeployment { + return (info as Partial).channelId !== undefined; +} + +/** Whether the given deployment info corresponds to a production deployment. */ +export function isProductionDeployment(info: DeploymentInfo): info is ProductionDeployment { + return !isPreviewDeployment(info); +} diff --git a/scripts/docs-deploy/docs-deps-install.ts b/scripts/docs-deploy/docs-deps-install.ts new file mode 100644 index 000000000000..ba6984fc1545 --- /dev/null +++ b/scripts/docs-deploy/docs-deps-install.ts @@ -0,0 +1,20 @@ +import {$} from 'zx'; + +export interface InstallOptions { + /** Whether dependencies should be installed with the lockfile being frozen. */ + frozenLockfile: boolean; +} + +/** Installs dependencies in the specified docs repository. */ +export async function installDepsForDocsSite( + repoDirPath: string, + options: InstallOptions = {frozenLockfile: true}, +) { + const additionalArgs = ['--non-interactive']; + + if (options.frozenLockfile) { + additionalArgs.push('--frozen-lockfile'); + } + + await $`yarn --cwd ${repoDirPath} install ${additionalArgs}`; +} diff --git a/scripts/docs-deploy/github-versioning.ts b/scripts/docs-deploy/github-versioning.ts new file mode 100644 index 000000000000..c9cf59e992c4 --- /dev/null +++ b/scripts/docs-deploy/github-versioning.ts @@ -0,0 +1,23 @@ +import { + AuthenticatedGithubClient, + GithubClient, + ReleaseRepoWithApi, + assertValidGithubConfig, + getConfig, + getNextBranchName, +} from '@angular/dev-infra-private/ng-dev'; + +export function getReleaseRepoWithApi(): ReleaseRepoWithApi { + const githubClient = + process.env.GITHUB_TOKEN !== undefined + ? new AuthenticatedGithubClient(process.env.GITHUB_TOKEN) + : new GithubClient(); + const {github} = getConfig([assertValidGithubConfig]); + + return { + api: githubClient, + name: github.name, + owner: github.owner, + nextBranchName: getNextBranchName(github), + }; +} diff --git a/scripts/docs-deploy/install-built-packages.ts b/scripts/docs-deploy/install-built-packages.ts new file mode 100644 index 000000000000..c8c4a340c61d --- /dev/null +++ b/scripts/docs-deploy/install-built-packages.ts @@ -0,0 +1,35 @@ +import * as fs from 'fs'; +import * as url from 'url'; + +import {BuiltPackage} from '@angular/dev-infra-private/ng-dev'; +import {getPackageJsonOfProject} from './utils'; + +export async function installBuiltPackagesInRepo(repoPath: string, builtPackages: BuiltPackage[]) { + const {parsed: packageJson, path: packageJsonPath} = await getPackageJsonOfProject(repoPath); + + // We will use Yarn resolutions to install the built packages. + if (packageJson.resolutions === undefined) { + packageJson.resolutions = {}; + } + + for (const builtPackage of builtPackages) { + const pkgName = builtPackage.name; + const destinationUrl = url.pathToFileURL(builtPackage.outputPath); + + // Add resolutions for each package in the format "**/{PACKAGE}" so that all + // nested versions of that specific package will have the same version. + packageJson.resolutions[`**/${pkgName}`] = destinationUrl; + + // Since the resolutions only control the version of all nested installs, we also need + // to explicitly set the version for the package listed in the project "package.json". + // e.g. the resolution above ensures transitive installs of `@angular/cdk` are updated, + // while the `dependencies` field overrides the workspace install of `@angular/cdk`. + packageJson.dependencies[pkgName] = destinationUrl; + + // In case this dependency was previously a dev dependency, just remove it because we + // re-added it as a normal dependency for simplicity. + delete packageJson.devDependencies[pkgName]; + } + + await fs.promises.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); +} diff --git a/scripts/docs-deploy/monitoring/ci-test.ts b/scripts/docs-deploy/monitoring/ci-test.ts new file mode 100644 index 000000000000..1bdc63140c27 --- /dev/null +++ b/scripts/docs-deploy/monitoring/ci-test.ts @@ -0,0 +1,8 @@ +import {runMonitorTestsForStable} from './index'; + +if (require.main === module) { + runMonitorTestsForStable().catch(e => { + console.error(e); + process.exitCode = 1; + }); +} diff --git a/scripts/docs-deploy/monitoring/index.ts b/scripts/docs-deploy/monitoring/index.ts new file mode 100644 index 000000000000..8598114eb3ad --- /dev/null +++ b/scripts/docs-deploy/monitoring/index.ts @@ -0,0 +1,37 @@ +import {$, cd} from 'zx'; + +import {ProductionDeployment} from '../deploy-to-site'; +import {cloneDocsRepositoryForMajor} from '../clone-docs-repo'; +import {fetchActiveReleaseTrains} from '@angular/dev-infra-private/ng-dev'; +import {getReleaseRepoWithApi} from '../github-versioning'; +import {installDepsForDocsSite} from '../docs-deps-install'; +import {sites} from '../utils'; + +/** + * Runs all audit tests from the given docs repository, ensuring that the + * specified remote URL is properly functioning. + */ +export async function runMonitorTests(docsRepoDir: string, remoteUrl: string) { + cd(docsRepoDir); + + await $`node ./tools/audit-docs-a11y.js ${remoteUrl}`; + await $`node ./tools/audit-docs.js ${remoteUrl}`; +} + +/** Runs the monitoring tests for the stable release train. */ +export async function runMonitorTestsForStable() { + const repo = getReleaseRepoWithApi(); + const active = await fetchActiveReleaseTrains(repo); + const stableMajor = active.latest.version.major; + const docsRepoDir = await cloneDocsRepositoryForMajor(stableMajor); + + await installDepsForDocsSite(docsRepoDir); + await runMonitorTests(docsRepoDir, sites.stable.remoteUrl); +} + +/** Runs monitoring tests for all specified production deployments. */ +export async function runMonitoringTests(docsRepoDir: string, targets: ProductionDeployment[]) { + for (const target of targets) { + await runMonitorTests(docsRepoDir, target.site.remoteUrl); + } +} diff --git a/scripts/docs-deploy/snapshot-deploy.ts b/scripts/docs-deploy/snapshot-deploy.ts new file mode 100644 index 000000000000..36dd6729b7bb --- /dev/null +++ b/scripts/docs-deploy/snapshot-deploy.ts @@ -0,0 +1,73 @@ +import {DeploymentInfo, deployToSite, isProductionDeployment} from './deploy-to-site'; + +import {buildDocsContentPackage} from '../build-docs-content'; +import {cloneDocsRepositoryForMajor} from './clone-docs-repo'; +import {installBuiltPackagesInRepo} from './install-built-packages'; +import {installDepsAndBuildDocsSite} from './utils'; +import {performDefaultSnapshotBuild} from '../build-packages-dist'; +import {runMonitoringTests} from './monitoring/index'; + +export type DeploymentConfig = { + /** + * Optional hook running before building the docs-app for deployment. + * + * Runs after the docs-content and local packages have been installed + * in the docs repository. + */ + prebuild?: (docsRepoDir: string) => Promise | void; +}; + +/** + * Builds and deploys the docs app with snapshot artifacts built + * from the currently checked-out `angular/components` `HEAD`. + * + * This allows us to build the library artifacts and docs-content + * directly without needing to wait for e.g. NPM to finish. + * + * One downside is that the docs site might be ahead of what is available + * on NPM, potentially causing examples to not work in StackBlitz. These should + * never break though since the stable release train is supposed to never receive + * breaking changes (i.e. `material.angular.io` Stackblitz should work as expected). + * + * Building based on the NPM packages would result in some inevitable complexity. + * For example: we would know that `13.2.0` is our latest release. We would need + * to wait for the NPM packages to be available (in case CI runs for the bump commit). + * Since packages will not be available and CI cannot wait for this to happen, the + * caretaker would need to manually re-run the job. Additionally it would require some + * additional rather complicated logic to find the proper matching docs-content revision. + */ +export async function buildAndDeployWithSnapshots( + firebaseServiceKey: string, + major: number, + targets: DeploymentInfo[], + options: DeploymentConfig = {}, +) { + // Clone the docs repo. + const docsRepoDir = await cloneDocsRepositoryForMajor(major); + + // Build the release output. + const builtPackages = performDefaultSnapshotBuild(); + + // Build the docs-content NPM package (not included in the default snapshot build) + builtPackages.push(buildDocsContentPackage()); + + // Install the release output, together with the examples into the + // the docs repository. + await installBuiltPackagesInRepo(docsRepoDir, builtPackages); + + // Run the prebuild hook if available. + await options.prebuild?.(docsRepoDir); + + // Install yarn dependencies and build the production output. + // Lockfile freezing needs to be disabled since we updated the `package.json` + // to point to our locally-built package artifacts. + await installDepsAndBuildDocsSite(docsRepoDir, {frozenLockfile: false}); + + // Deploy all targets to Firebase. + for (const target of targets) { + await deployToSite(docsRepoDir, firebaseServiceKey, target); + } + + // Run post monitoring tests for production deployments. + await runMonitoringTests(docsRepoDir, targets.filter(isProductionDeployment)); +} diff --git a/scripts/docs-deploy/update-versions-file.ts b/scripts/docs-deploy/update-versions-file.ts new file mode 100644 index 000000000000..6790c58fb6e8 --- /dev/null +++ b/scripts/docs-deploy/update-versions-file.ts @@ -0,0 +1,64 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import { + ActiveReleaseTrains, + assertValidReleaseConfig, + fetchLongTermSupportBranchesFromNpm, + getConfig, +} from '@angular/dev-infra-private/ng-dev'; + +import {sites} from './utils'; + +interface VersionEntry { + url: string; + title: string; +} + +type VersionList = VersionEntry[]; + +/** Path to the `versions.json` file in the docs-app. */ +const relativeVersionFilePath = 'src/assets/versions.json'; + +/** + * List of hard-coded major versions which have an archived docs-site, + * but their major is not detectable as active/inactive LTS due to these + * majors being released with the old release tool (not tagging LTS majors). + */ +// TODO: This can be removed at some point in the future if we feel like nobody +// needing these doc-site archives. +const hardcodedOldMajorsWithoutLtsTag = [5, 6, 7, 8, 9, 10, 11]; + +/** + * Updates the versions list file in the specified docs repo to reflect + * the current active release trains and archived LTS versions. + */ +export async function updateVersionsFile(docsRepoDir: string, active: ActiveReleaseTrains) { + const versionFilePath = path.join(docsRepoDir, relativeVersionFilePath); + const {release} = getConfig([assertValidReleaseConfig]); + const ltsBranches = await fetchLongTermSupportBranchesFromNpm(release); + + const versions: VersionList = [ + {url: sites.next.remoteUrl, title: 'Next'}, + {url: sites.rc.remoteUrl, title: 'Release Candidate'}, + {url: sites.stable.remoteUrl, title: `${active.latest.version.format()} (latest)`}, + ]; + + const archivedMajors = new Set( + [...ltsBranches.active, ...ltsBranches.inactive] + .map(e => e.version.major) + .concat(hardcodedOldMajorsWithoutLtsTag), + ); + + // Insert archived majors in descending order (i.e. starting with newer ones). + Array.from(archivedMajors) + .sort((a, b) => b - a) + .forEach(major => + versions.push({ + title: `v${major}`, + url: sites.forMajor(major).remoteUrl, + }), + ); + + await fs.promises.writeFile(versionFilePath, JSON.stringify(versions, null, 2)); +} diff --git a/scripts/docs-deploy/utils.ts b/scripts/docs-deploy/utils.ts new file mode 100644 index 000000000000..666df9fb6ccb --- /dev/null +++ b/scripts/docs-deploy/utils.ts @@ -0,0 +1,52 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import {InstallOptions, installDepsForDocsSite} from './docs-deps-install'; + +import {$} from 'zx'; + +/** Absolute path to the `angular/components` project root. */ +export const projectDir = path.join(__dirname, '../..'); + +/** Interface describing a site target for the docs-app. */ +export class SiteTarget { + constructor(public firebaseSiteId: string, public remoteUrl: string) {} +} + +/** Object capturing all site targets for the docs-app. */ +export const sites = { + stable: new SiteTarget('ng-comp-test', 'https://ng-comp-test.firebaseapp.com'), + next: new SiteTarget('next-ng-comp-test', 'https://next-ng-comp-test.firebaseapp.com'), + rc: new SiteTarget('rc-ng-comp-test', 'https://rc-ng-comp-test.firebaseapp.com'), + + forMajor: (major: number) => + new SiteTarget(`v${major}-ng-comp-test`, `https://v${major}-ng-comp-test.firebaseapp.com`), +}; + +/** Configuration describing the Firebase project that we deploy to. */ +export const firebaseConfig = { + projectId: 'angular-components-test', + serviceKey: process.env.DOCS_SITE_FIREBASE_SERVICE_KEY!, +}; + +/** Finds and parsed the `package.json` of the specified project directory. */ +export async function getPackageJsonOfProject( + projectPath: string, +): Promise<{path: string; parsed: any}> { + const packageJsonPath = path.join(projectPath, 'package.json'); + const packageJsonContent = await fs.promises.readFile(packageJsonPath, 'utf8'); + + return { + path: packageJsonPath, + parsed: JSON.parse(packageJsonContent), + }; +} + +/** + * Installs dependencies in the specified docs repository and builds the + * production site output. + */ +export async function installDepsAndBuildDocsSite(repoPath: string, options: InstallOptions) { + await installDepsForDocsSite(repoPath, options); + await $`yarn --cwd ${repoPath} prod-build`; +} diff --git a/yarn.lock b/yarn.lock index 3ca88e933fba..7a25d0f7514e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -323,20 +323,20 @@ dependencies: tslib "^2.3.0" -"@angular/dev-infra-private@https://github.com/angular/dev-infra-private-builds.git#b6656cffbd46bb3637b09b08561e5f1a147603c9": - version "0.0.0-138ec743c342cd2a4a75443d19e0ccd47244ee07" - resolved "https://github.com/angular/dev-infra-private-builds.git#b6656cffbd46bb3637b09b08561e5f1a147603c9" +"@angular/dev-infra-private@https://github.com/angular/dev-infra-private-builds.git#7544378ad9aa94cdfe256aaaa923c107f45175a2": + version "0.0.0-8feef7a4ad17f94a98b55e3fa7ce4697bfaef895" + resolved "https://github.com/angular/dev-infra-private-builds.git#7544378ad9aa94cdfe256aaaa923c107f45175a2" dependencies: "@angular-devkit/build-angular" "14.0.0-next.3" "@angular/benchpress" "0.3.0" "@babel/core" "^7.16.0" + "@bazel/buildifier" "^5.0.1" "@bazel/concatjs" "4.6.0" "@bazel/esbuild" "4.6.0" "@bazel/protractor" "4.6.0" "@bazel/runfiles" "4.6.0" "@bazel/terser" "4.6.0" "@bazel/typescript" "4.6.0" - "@google-cloud/storage" "^5.18.1" "@microsoft/api-extractor" "7.19.4" "@types/browser-sync" "^2.26.3" "@types/node" "16.10.9" @@ -348,9 +348,9 @@ "@types/yargs" "^17.0.0" browser-sync "^2.27.7" chalk "^4.1.0" + clang-format "^1.6.0" node-fetch "^2.6.1" prettier "^2.3.2" - protobufjs "^6.11.2" protractor "^7.0.0" selenium-webdriver "4.1.1" send "^0.17.2" @@ -1345,6 +1345,11 @@ resolved "https://registry.yarnpkg.com/@bazel/buildifier/-/buildifier-4.2.5.tgz#de51339613e2ca1f39dda6ade8b0062e6575b3e1" integrity sha512-lTg3YEuCO1e7dCc/fLUWu5R7KOHmpPDSkoSeT+rPrszFfxqrWGJTSxZbtk/4AJZnuC3p704po8RUJinPtmxTmQ== +"@bazel/buildifier@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@bazel/buildifier/-/buildifier-5.0.1.tgz#8946848cf2c28717ec8fb4ff46e424aeba82be74" + integrity sha512-3eMWxdFtcQf+Jw55PZqD/I9N785wp6QQ2k/SZst7R64KAGrS8Ke1EhPXaZHZBkXao5GXrm6SNLDV287xg2kguA== + "@bazel/concatjs@4.4.5": version "4.4.5" resolved "https://registry.yarnpkg.com/@bazel/concatjs/-/concatjs-4.4.5.tgz#105a156f412da5a43b37724b50d29ee5ee4ff678" @@ -1579,22 +1584,7 @@ resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== -"@google-cloud/common@^3.8.1": - version "3.10.0" - resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.10.0.tgz#454d1155bb512109cd83c6183aabbd39f9aabda7" - integrity sha512-XMbJYMh/ZSaZnbnrrOFfR/oQrb0SxG4qh6hDisWCoEbFcBHV0qHQo4uXfeMCzolx2Mfkh6VDaOGg+hyJsmxrlw== - dependencies: - "@google-cloud/projectify" "^2.0.0" - "@google-cloud/promisify" "^2.0.0" - arrify "^2.0.1" - duplexify "^4.1.1" - ent "^2.2.0" - extend "^3.0.2" - google-auth-library "^7.14.0" - retry-request "^4.2.2" - teeny-request "^7.0.0" - -"@google-cloud/paginator@^3.0.6", "@google-cloud/paginator@^3.0.7": +"@google-cloud/paginator@^3.0.6": version "3.0.7" resolved "https://registry.yarnpkg.com/@google-cloud/paginator/-/paginator-3.0.7.tgz#fb6f8e24ec841f99defaebf62c75c2e744dd419b" integrity sha512-jJNutk0arIQhmpUUQJPJErsojqo834KcyB6X7a1mxuic8i1tKXxde8E69IZxNZawRIlZdIK2QY4WALvlK5MzYQ== @@ -1638,34 +1628,6 @@ lodash.snakecase "^4.1.1" p-defer "^3.0.0" -"@google-cloud/storage@^5.18.1": - version "5.18.2" - resolved "https://registry.yarnpkg.com/@google-cloud/storage/-/storage-5.18.2.tgz#0ded98a69323d253e6dd986650edc89b5c504bf9" - integrity sha512-hL/6epBF2uPt7YtJoOKI6mVxe6RsKBs7S8o2grE0bFGdQKSOngVHBcstH8jDw7aN2rXGouA2TfVTxH+VapY5cg== - dependencies: - "@google-cloud/common" "^3.8.1" - "@google-cloud/paginator" "^3.0.7" - "@google-cloud/promisify" "^2.0.0" - abort-controller "^3.0.0" - arrify "^2.0.0" - async-retry "^1.3.3" - compressible "^2.0.12" - configstore "^5.0.0" - date-and-time "^2.0.0" - duplexify "^4.0.0" - extend "^3.0.2" - gaxios "^4.0.0" - get-stream "^6.0.0" - google-auth-library "^7.0.0" - hash-stream-validation "^0.2.2" - mime "^3.0.0" - mime-types "^2.0.8" - p-limit "^3.0.1" - pumpify "^2.0.0" - snakeize "^0.1.0" - stream-events "^1.0.4" - xdg-basedir "^4.0.0" - "@grpc/grpc-js@~1.5.0": version "1.5.7" resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.5.7.tgz#c83a5dc1d0cf7b8aa82371cfa7125955d1f25a96" @@ -3024,7 +2986,7 @@ "@types/qs" "*" "@types/serve-static" "*" -"@types/fs-extra@^9.0.13": +"@types/fs-extra@^9.0.12", "@types/fs-extra@^9.0.13": version "9.0.13" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-9.0.13.tgz#7594fbae04fe7f1918ce8b3d213f74ff44ac1f45" integrity sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA== @@ -3115,12 +3077,12 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== -"@types/minimist@^1.2.0": +"@types/minimist@^1.2.0", "@types/minimist@^1.2.2": version "1.2.2" resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== -"@types/node-fetch@^2.5.10", "@types/node-fetch@^2.5.5": +"@types/node-fetch@^2.5.10", "@types/node-fetch@^2.5.12", "@types/node-fetch@^2.5.5": version "2.6.1" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975" integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA== @@ -3148,7 +3110,7 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b" integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw== -"@types/node@^16.10.9": +"@types/node@^16.10.9", "@types/node@^16.6": version "16.11.26" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.26.tgz#63d204d136c9916fb4dcd1b50f9740fe86884e47" integrity sha512-GZ7bu5A6+4DtG7q9GsoHXy3ALcgeIHP4NnL0Vv2wu0uUB/yQex26v0tf6/na1mm0+bS9Uw+0DFex7aaKr2qawQ== @@ -3931,7 +3893,7 @@ arrify@^1.0.0, arrify@^1.0.1: resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0= -arrify@^2.0.0, arrify@^2.0.1: +arrify@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== @@ -4004,14 +3966,7 @@ async-limiter@~1.0.0: resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== -async-retry@^1.3.3: - version "1.3.3" - resolved "https://registry.yarnpkg.com/async-retry/-/async-retry-1.3.3.tgz#0e7f36c04d8478e7a58bdbed80cedf977785f280" - integrity sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw== - dependencies: - retry "0.13.1" - -async@1.5.2, async@1.x, async@^1.3.0: +async@1.5.2, async@1.x, async@^1.3.0, async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" integrity sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo= @@ -4887,7 +4842,7 @@ chalk@^3.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.1, chalk@~4.1.0: +chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.1, chalk@^4.1.2, chalk@~4.1.0: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -5006,6 +4961,15 @@ cjson@^0.3.1: dependencies: json-parse-helpfulerror "^1.0.3" +clang-format@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/clang-format/-/clang-format-1.6.0.tgz#48fac4387712aeeae0f47b5d72f639f3fd95f4b6" + integrity sha512-W3/L7fWkA8DoLkz9UGjrRnNi+J5a5TuS2HDLqk6WsicpOzb66MBu4eY/EcXhicHriVnAXWQVyk5/VeHWY6w4ow== + dependencies: + async "^1.5.2" + glob "^7.0.0" + resolve "^1.1.6" + clean-stack@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" @@ -5282,7 +5246,7 @@ compress-commons@^4.1.0: normalize-path "^3.0.0" readable-stream "^3.6.0" -compressible@^2.0.12, compressible@~2.0.16: +compressible@~2.0.16: version "2.0.18" resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" integrity sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg== @@ -5315,7 +5279,7 @@ config-chain@^1.1.11: ini "^1.3.4" proto-list "~1.2.1" -configstore@^5.0.0, configstore@^5.0.1: +configstore@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96" integrity sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA== @@ -5697,11 +5661,6 @@ data-uri-to-buffer@3: resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz#594b8973938c5bc2c33046535785341abc4f3636" integrity sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og== -date-and-time@^2.0.0: - version "2.2.1" - resolved "https://registry.yarnpkg.com/date-and-time/-/date-and-time-2.2.1.tgz#70691b7d8a17633f3abe32aab148e194f9f2e215" - integrity sha512-i5lrLPmRbdtb9/Tcu+NWWvff29ejNZtnL/6TI5VbhuPtqgea7bovY0F2AtFSX9QDQEQLUtqJ3R8ypM2c6alGoA== - date-fns@^2.28.0: version "2.28.0" resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.28.0.tgz#9570d656f5fc13143e50c975a3b6bbeb46cd08b2" @@ -6354,7 +6313,7 @@ duplexer@~0.1.1: resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== -duplexify@^4.0.0, duplexify@^4.1.1: +duplexify@^4.0.0: version "4.1.2" resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0" integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw== @@ -6563,7 +6522,7 @@ enquirer@^2.3.5: dependencies: ansi-colors "^4.1.1" -ent@^2.2.0, ent@~2.2.0: +ent@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d" integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0= @@ -7754,7 +7713,7 @@ fs-extra@3.0.1: jsonfile "^3.0.0" universalify "^0.1.0" -fs-extra@^10.0.1: +fs-extra@^10.0.0, fs-extra@^10.0.1: version "10.0.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.0.1.tgz#27de43b4320e833f6867cc044bfce29fdf0ef3b8" integrity sha512-NbdoVMZso2Lsrn/QwLXOy6rm0ufY2zEOKCDzJR/0kBsb0E6qed0P3iYK+Ath3BfvXEeu4JhEtXLgILx5psUfag== @@ -8134,7 +8093,7 @@ globby@^11.0.1, globby@^11.0.3, globby@^11.1.0: merge2 "^1.4.1" slash "^3.0.0" -globby@^12.0.2: +globby@^12.0.1, globby@^12.0.2: version "12.2.0" resolved "https://registry.yarnpkg.com/globby/-/globby-12.2.0.tgz#2ab8046b4fba4ff6eede835b29f678f90e3d3c22" integrity sha512-wiSuFQLZ+urS9x2gGPl1H5drc5twabmm4m2gTR27XDFyjUHJUNsS8o/2aKyIF6IoBaR630atdher0XJ5g6OMmA== @@ -8185,7 +8144,7 @@ google-auth-library@^6.1.3: jws "^4.0.0" lru-cache "^6.0.0" -google-auth-library@^7.0.0, google-auth-library@^7.14.0, google-auth-library@^7.6.1: +google-auth-library@^7.0.0, google-auth-library@^7.6.1: version "7.14.0" resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-7.14.0.tgz#9d6a20592f7b4d4c463cd3e93934c4b1711d5dc6" integrity sha512-or8r7qUqGVI3W8lVSdPh0ZpeFyQHeE73g5c0p+bLNTTUFXJ+GSeDQmZRZ2p4H8cF/RJYa4PNvi/A1ar1uVNLFA== @@ -8429,11 +8388,6 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -hash-stream-validation@^0.2.2: - version "0.2.4" - resolved "https://registry.yarnpkg.com/hash-stream-validation/-/hash-stream-validation-0.2.4.tgz#ee68b41bf822f7f44db1142ec28ba9ee7ccb7512" - integrity sha512-Gjzu0Xn7IagXVkSu9cSFuK1fqzwtLwFhNhVL8IFJijRNMgUttFbBSIAzKuSIrsFMO1+g1RlsoN49zPIbwPDMGQ== - hash.js@^1.1.7: version "1.1.7" resolved "https://registry.yarnpkg.com/hash.js/-/hash.js-1.1.7.tgz#0babca538e8d4ee4a0f8988d68866537a003cf42" @@ -10608,7 +10562,7 @@ mime-db@1.51.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.0.8, mime-types@^2.1.12, mime-types@^2.1.16, mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.12, mime-types@^2.1.16, mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.34" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24" integrity sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A== @@ -10630,11 +10584,6 @@ mime@^2.5.2: resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367" integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg== -mime@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7" - integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== - mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" @@ -11476,7 +11425,7 @@ p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.0.1, p-limit@^3.0.2: +p-limit@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== @@ -12476,7 +12425,7 @@ prr@~1.0.1: resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= -ps-tree@=1.2.0: +ps-tree@=1.2.0, ps-tree@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/ps-tree/-/ps-tree-1.2.0.tgz#5e7425b89508736cdd4f2224d028f7bb3f722ebd" integrity sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA== @@ -12501,15 +12450,6 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" -pumpify@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pumpify/-/pumpify-2.0.1.tgz#abfc7b5a621307c728b551decbbefb51f0e4aa1e" - integrity sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw== - dependencies: - duplexify "^4.1.1" - inherits "^2.0.3" - pump "^3.0.0" - punycode@^1.3.2: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" @@ -13061,7 +13001,7 @@ restore-cursor@^3.1.0: onetime "^5.1.0" signal-exit "^3.0.2" -retry-request@^4.0.0, retry-request@^4.2.2: +retry-request@^4.0.0: version "4.2.2" resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-4.2.2.tgz#b7d82210b6d2651ed249ba3497f07ea602f1a903" integrity sha512-xA93uxUD/rogV7BV59agW/JHPGXeREMWiZc9jhcwY4YdZ7QOtC7qbomYg0n4wyk2lJhggjvKvhNX8wln/Aldhg== @@ -13069,16 +13009,16 @@ retry-request@^4.0.0, retry-request@^4.2.2: debug "^4.1.1" extend "^3.0.2" -retry@0.13.1, retry@^0.13.1: - version "0.13.1" - resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" - integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== - retry@^0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs= +retry@^0.13.1: + version "0.13.1" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" + integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== + reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" @@ -13678,11 +13618,6 @@ snake-case@^3.0.4: dot-case "^3.0.4" tslib "^2.0.3" -snakeize@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/snakeize/-/snakeize-0.1.0.tgz#10c088d8b58eb076b3229bb5a04e232ce126422d" - integrity sha1-EMCI2LWOsHazIpu1oE4jLOEmQi0= - socket.io-adapter@~1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-1.1.2.tgz#ab3f0d6f66b8fc7fca3959ab5991f82221789be9" @@ -14065,13 +14000,6 @@ stream-combiner@~0.0.4: dependencies: duplexer "~0.1.1" -stream-events@^1.0.4, stream-events@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/stream-events/-/stream-events-1.0.5.tgz#bbc898ec4df33a4902d892333d47da9bf1c406d5" - integrity sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg== - dependencies: - stubs "^3.0.0" - stream-shift@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" @@ -14243,11 +14171,6 @@ strip-outer@^1.0.0: dependencies: escape-string-regexp "^1.0.2" -stubs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/stubs/-/stubs-3.0.0.tgz#e8d2ba1fa9c90570303c030b6900f7d5f89abe5b" - integrity sha1-6NK6H6nJBXAwPAMLaQD31fiavls= - style-search@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/style-search/-/style-search-0.1.0.tgz#7958c793e47e32e07d2b5cafe5c0bf8e12e77902" @@ -14521,17 +14444,6 @@ tcp-port-used@^1.0.1: debug "4.3.1" is2 "^2.0.6" -teeny-request@^7.0.0: - version "7.1.3" - resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-7.1.3.tgz#5a3d90c559a6c664a993477b138e331a518765ba" - integrity sha512-Ew3aoFzgQEatLA5OBIjdr1DWJUaC1xardG+qbPPo5k/y/3fMwXLxpjh5UB5dVfElktLaQbbMs80chkz53ByvSg== - dependencies: - http-proxy-agent "^5.0.0" - https-proxy-agent "^5.0.0" - node-fetch "^2.6.1" - stream-events "^1.0.5" - uuid "^8.0.0" - temp-fs@^0.9.9: version "0.9.9" resolved "https://registry.yarnpkg.com/temp-fs/-/temp-fs-0.9.9.tgz#8071730437870720e9431532fe2814364f8803d7" @@ -15966,3 +15878,20 @@ zone.js@~0.10.3: version "0.10.3" resolved "https://registry.yarnpkg.com/zone.js/-/zone.js-0.10.3.tgz#3e5e4da03c607c9dcd92e37dd35687a14a140c16" integrity sha512-LXVLVEq0NNOqK/fLJo3d0kfzd4sxwn2/h67/02pjCjfKDxgx1i9QqpvtHD8CrBnSSwMw5+dy11O7FRX5mkO7Cg== + +zx@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/zx/-/zx-4.3.0.tgz#6275115c7cc914461a7d77e52abb60c15cb62d9e" + integrity sha512-KuEjpu5QFIMx0wWfzknDRhY98s7a3tWNRmYt19XNmB7AfOmz5zISA4+3Q8vlJc2qguxMn89uSxhPDCldPa3YLA== + dependencies: + "@types/fs-extra" "^9.0.12" + "@types/minimist" "^1.2.2" + "@types/node" "^16.6" + "@types/node-fetch" "^2.5.12" + chalk "^4.1.2" + fs-extra "^10.0.0" + globby "^12.0.1" + minimist "^1.2.5" + node-fetch "^2.6.1" + ps-tree "^1.2.0" + which "^2.0.2" From a59977e282916ebd7fa42a0765c45df2a22c98c6 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Wed, 9 Mar 2022 22:01:29 +0100 Subject: [PATCH 2/2] build: do not reserve `GOOGLE_APPLICATION_CREDENTIALS` for RBE auth We currently configure RBE by setting `GOOGLE_APPLICATION_CREDENTIALS` into the `$BASH_ENV` variable, ensuring RBE is configured everywhere on CI. This worked nicely but now with automatic docs deployment turned out to be problematic since it prevents scripts from defining `GOOGLE_APPLICATION_CREDENTIALS` themselves/overriding it. the reason is that `$BASH_ENV` always runs in new child processes (like when firebase is initialized) and then overrides the credentials back to the RBE service key. We can simplify this code by using a dedicated Bazel flag. --- scripts/bazel/setup-remote-execution.sh | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/scripts/bazel/setup-remote-execution.sh b/scripts/bazel/setup-remote-execution.sh index 87068416c12b..ce46e2ad3587 100755 --- a/scripts/bazel/setup-remote-execution.sh +++ b/scripts/bazel/setup-remote-execution.sh @@ -11,21 +11,10 @@ fi # Decode the GCP token that is needed to authenticate the Bazel remote execution. openssl aes-256-cbc -d -in scripts/bazel/gcp_token -md md5 -k ${GCP_DECRYPT_TOKEN} \ - -out $HOME/.gcp_credentials - -# Set the "GOOGLE_APPLICATION_CREDENTIALS" environment variable. It should point to the GCP credentials -# file. Bazel will then automatically picks up the credentials from that variable. -# https://docs.bazel.build/versions/main/command-line-reference.html#flag--google_default_credentials -# https://cloud.google.com/docs/authentication/production. -if [[ ! -z "${BASH_ENV}" ]]; then - # CircleCI uses the `BASH_ENV` variable for environment variables. - echo "export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.gcp_credentials" >> ${BASH_ENV} -elif [[ ! -z "${GITHUB_ENV}" ]]; then - # Github actions use the `GITHUB_ENV` variable for environment variables. - echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.gcp_credentials" >> ${GITHUB_ENV} -fi + -out $HOME/.gcp_rbe_credentials # Update the project Bazel configuration to always use remote execution. # Note: We add the remote config flag to the user bazelrc file that is not tracked # by Git. This is necessary to avoid stamping builds with `.with-local-changes`. echo "build --config=remote" >> .bazelrc.user +echo "build:remote --google_credentials=\"$HOME/.gcp_rbe_credentials\"" >> .bazelrc.user