From 02a8a9f3e2eb0826f881d21b3ff3b7c50821a522 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Tue, 12 Nov 2019 15:19:18 +0100 Subject: [PATCH] build: enable bazel node modules symlinking Enables bazel managed node modules symlinking (similar to what framework does). We benefit from: * Easier way to make manual changes to debug issues * Less disk size since we do not need to maintain two instances of `node_modules` * Caching of node modules in CI. Bazel previously discarded the whole node modules folder if the lock file changed. --- .circleci/config.yml | 6 ++-- WORKSPACE | 21 ++++-------- package.json | 2 +- tools/bazel/postinstall-patches.js | 53 +++++++++++++++++++++++++----- 4 files changed, 56 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b505f6db3176..e40d3cfd4d52 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,8 +13,10 @@ var_2: &docker-firefox-image circleci/node:12.9.1-browsers # **Note**: When updating the beginning of the cache key, also update the cache key to match # the new cache key prefix. This allows us to take advantage of CircleCI's fallback caching. # Read more here: https://circleci.com/docs/2.0/caching/#restoring-cache. -var_3: &cache_key v4-ng-mat-{{ checksum "WORKSPACE" }}-{{ checksum "yarn.lock" }} -var_4: &cache_fallback_key v4-ng-mat- +var_3: &cache_key v4-ng-mat-{{ checksum "tools/bazel/postinstall-patches.js" }}-{{ checksum "WORKSPACE" }}-{{ checksum "yarn.lock" }} +# We want to invalidate the cache if the postinstall patches change. In order to apply new +# patches, a clean version of the node modules is needed. +var_4: &cache_fallback_key v4-ng-mat-{{ checksum "tools/bazel/postinstall-patches.js" }}- # Settings common to each job var_5: &job_defaults diff --git a/WORKSPACE b/WORKSPACE index 1d28521082d5..8f542359ee8a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,4 +1,7 @@ -workspace(name = "angular_material") +workspace( + name = "angular_material", + managed_directories = {"@npm": ["node_modules"]}, +) load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") @@ -42,20 +45,10 @@ node_repositories( yarn_install( name = "npm", - # Ensure that all resources are available when the "postinstall" or "preinstall" scripts - # are executed in the Bazel sandbox. - data = [ - "//:angular-tsconfig.json", - "//:tools/bazel/flat_module_factory_resolution.patch", - "//:tools/bazel/manifest_externs_hermeticity.patch", - "//:tools/bazel/postinstall-patches.js", - "//:tools/npm/check-npm.js", - ], + # We add the postinstall patches file here so that Yarn will rerun whenever + # the patches script changes. + data = ["//:tools/bazel/postinstall-patches.js"], package_json = "//:package.json", - # Temporarily disable node_modules symlinking until the fix for - # https://github.com/bazelbuild/bazel/issues/8487 makes it into a - # future Bazel release - symlink_node_modules = False, yarn_lock = "//:yarn.lock", ) diff --git a/package.json b/package.json index 7a6cc15b4fc7..ea2ae949367b 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "yarn": ">= 1.19.1" }, "scripts": { - "postinstall": "node --preserve-symlinks --preserve-symlinks-main tools/bazel/postinstall-patches.js && ngcc --properties main --create-ivy-entry-points", + "postinstall": "node tools/bazel/postinstall-patches.js && ngcc --properties main --create-ivy-entry-points", "build": "bash ./scripts/build-packages-dist.sh", "bazel:buildifier": "find . -type f \\( -name \"*.bzl\" -or -name WORKSPACE -or -name BUILD -or -name BUILD.bazel \\) ! -path \"*/node_modules/*\" | xargs buildifier -v --warnings=attr-cfg,attr-license,attr-non-empty,attr-output-default,attr-single-file,constant-glob,ctx-args,depset-iteration,depset-union,dict-concatenation,duplicated-name,filetype,git-repository,http-archive,integer-division,load,load-on-top,native-build,native-package,output-group,package-name,package-on-top,redefined-variable,repository-name,same-origin-load,string-iteration,unused-variable,unsorted-dict-items,out-of-order-load", "bazel:format-lint": "yarn -s bazel:buildifier --lint=warn --mode=check", diff --git a/tools/bazel/postinstall-patches.js b/tools/bazel/postinstall-patches.js index 71ac0ea18124..c5e1bd27d641 100644 --- a/tools/bazel/postinstall-patches.js +++ b/tools/bazel/postinstall-patches.js @@ -8,18 +8,18 @@ const shelljs = require('shelljs'); const path = require('path'); const fs = require('fs'); +/** + * Version of the post install patch. Needs to be incremented when patches + * have been added or removed. + */ +const PATCH_VERSION = 1; + /** Path to the project directory. */ const projectDir = path.join(__dirname, '../..'); shelljs.set('-e'); shelljs.cd(projectDir); -// Do not apply postinstall patches when running "postinstall" outside. The -// "generate_build_file.js" file indicates that we run in Bazel managed node modules. -if (!shelljs.test('-e', 'generate_build_file.js')) { - return; -} - // Workaround for https://github.com/angular/angular/issues/18810. shelljs.exec('ngc -p angular-tsconfig.json'); @@ -69,7 +69,7 @@ searchAndReplace( const hasFlatModuleBundle = fs.existsSync(filePath.replace('.d.ts', '.metadata.json')); if ((filePath.includes('node_modules/') || !hasFlatModuleBundle) && $1`, 'node_modules/@angular/compiler-cli/src/transformers/compiler_host.js'); -shelljs.cat(path.join(__dirname, './flat_module_factory_resolution.patch')).exec('patch -p0'); +applyPatch(path.join(__dirname, './flat_module_factory_resolution.patch')); // The three replacements below ensure that metadata files can be read by NGC and // that metadata files are collected as Bazel action inputs. searchAndReplace( @@ -90,7 +90,7 @@ searchAndReplace( 'node_modules/@angular/bazel/src/ng_module.bzl'); // Workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1208. -shelljs.cat(path.join(__dirname, './manifest_externs_hermeticity.patch')).exec('patch -p0'); +applyPatch(path.join(__dirname, './manifest_externs_hermeticity.patch')); // Workaround for using Ngcc with "--create-ivy-entry-points". This is a special // issue for our repository since we want to run Ivy by default in the module resolution, @@ -105,12 +105,34 @@ searchAndReplace(/angular_compiler_options = {/, `$& "strictAttributeTypes": False, "strictDomEventTypes": False,`, 'node_modules/@angular/bazel/src/ng_module.bzl'); +/** + * Applies the given patch if not done already. Throws if the patch does + * not apply cleanly. + */ +function applyPatch(patchFile) { + const patchMarkerFileName = `${path.basename(patchFile)}.patch_marker`; + const patchMarkerPath = path.join(projectDir, 'node_modules/', patchMarkerFileName); + + if (hasFileBeenPatched(patchMarkerPath)) { + return; + } + + writePatchMarker(patchMarkerPath); + shelljs.cat(patchFile).exec('patch -p0'); +} + /** * Reads the specified file and replaces matches of the search expression - * with the given replacement. Throws if no changes were made. + * with the given replacement. Throws if no changes were made and the + * patch has not been applied yet. */ function searchAndReplace(search, replacement, relativeFilePath) { const filePath = path.join(projectDir, relativeFilePath); + + if (hasFileBeenPatched(filePath)) { + return; + } + const originalContent = fs.readFileSync(filePath, 'utf8'); const newFileContent = originalContent.replace(search, replacement); @@ -118,5 +140,18 @@ function searchAndReplace(search, replacement, relativeFilePath) { throw Error(`Could not perform replacement in: ${filePath}.`); } + writePatchMarker(filePath); fs.writeFileSync(filePath, newFileContent, 'utf8'); } + +/** Marks the specified file as patched. */ +function writePatchMarker(filePath) { + shelljs.echo(PATCH_VERSION).to(`${filePath}.patch_marker`); +} + +/** Checks if the given file has been patched. */ +function hasFileBeenPatched(filePath) { + const markerFilePath = `${filePath}.patch_marker`; + return shelljs.test('-e', markerFilePath) && + shelljs.cat(markerFilePath).toString().trim() === `${PATCH_VERSION}`; +}