From 4bccfb76eb61f56119f784a3e08b6e746a357097 Mon Sep 17 00:00:00 2001 From: per1234 Date: Tue, 21 Nov 2023 23:44:25 -0800 Subject: [PATCH 1/4] Control scope of recursion when validating Dependabot configuration files The project infrastructure validates the Dependabot configuration files against their JSON schema. In addition to the repository's own Dependabot configuration file at the standard location, a reusable standardized "asset" configuration file for GitHub Actions workflows is hosted in the repository. There is a need to eventually add additional "asset" files to configure Dependabot for other dependency management frameworks. Previously, in order to provide validation coverage for all the dependabot.yml files in the repository, a "globstar" was used to cause the validator to recursively search the entire file tree under the repository. That approach is problematic because the repository contains externally maintained files (e.g., the npm packages under the node_modules folder). Searching and validating these files is inefficient at best and the cause of spurious failures at worst. This is avoided by targeting the search. In order to support the addition of more "asset" Dependabot configuration files in the future, a globstar is still used, but the recursion is limited to the folder dedicated as a container for those files, which will never lead to unintended files being validated. --- Taskfile.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index ea23d623..d707eba9 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -359,7 +359,10 @@ tasks: SCHEMA_URL: https://json.schemastore.org/dependabot-2.0 SCHEMA_PATH: sh: task utility:mktemp-file TEMPLATE="dependabot-schema-XXXXXXXXXX.json" - DATA_PATH: "**/dependabot.yml" + # The Dependabot configuration file for the repository. + DATA_PATH: ".github/dependabot.yml" + # The asset Dependabot configuration files. + ASSETS_DATA_PATH: "workflow-templates/assets/dependabot/**/dependabot.yml" PROJECT_FOLDER: sh: pwd WORKING_FOLDER: @@ -372,6 +375,12 @@ tasks: --all-errors \ -s "{{.SCHEMA_PATH}}" \ -d "{{.PROJECT_FOLDER}}/{{.DATA_PATH}}" + - | + cd "{{.WORKING_FOLDER}}" # Workaround for https://github.com/npm/cli/issues/3210 + npx ajv-cli@{{.SCHEMA_DRAFT_4_AJV_CLI_VERSION}} validate \ + --all-errors \ + -s "{{.SCHEMA_PATH}}" \ + -d "{{.PROJECT_FOLDER}}/{{.ASSETS_DATA_PATH}}" docs:generate: desc: Create all generated documentation content From 36fc56516cb2db9c6008333f7e9bd440d7783006 Mon Sep 17 00:00:00 2001 From: per1234 Date: Tue, 21 Nov 2023 23:56:10 -0800 Subject: [PATCH 2/4] Avoid uncontrolled recursion when validating markdownlint configuration files The project infrastructure validates the markdownlint configuration files against their JSON schema. In addition to the repository's own markdownlint configuration file at the standard location, a reusable standardized "asset" configuration file is hosted in the repository. Previously, in order to provide validation coverage for all the .markdownlint.yml files in the repository, a "globstar" was used to cause the validator to recursively search the entire file tree under the repository. That approach is problematic because the repository contains externally maintained files (e.g., the npm packages under the node_modules folder). Searching and validating these files is inefficient at best and the cause of spurious failures at worst. That worst case scenario has now occurred as a file from a npm package dependency is being discovered by the validation system and failing validation. This is avoided by targeting the search. The chosen approach is to validate only the "asset" file. Even though this causes the file in the root of the repository to no longer be validated directly, it continues to be indirectly validated because the file in the root is an exact copy of the "asset" (and the repository infrastructure enforces the sync). --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index d707eba9..6cf7f0b2 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -705,7 +705,7 @@ tasks: SCHEMA_URL: https://raw.githubusercontent.com/DavidAnson/markdownlint/main/schema/markdownlint-config-schema.json SCHEMA_PATH: sh: task utility:mktemp-file TEMPLATE="markdownlint-schema-XXXXXXXXXX.json" - DATA_PATH: "**/.markdownlint.{yml,yaml}" + DATA_PATH: "workflow-templates/assets/check-markdown/.markdownlint.yml" deps: - task: npm:install-deps cmds: From c7987fb9a0dc24f38a44dc86c6d92ec9d296e249 Mon Sep 17 00:00:00 2001 From: per1234 Date: Tue, 21 Nov 2023 23:37:51 -0800 Subject: [PATCH 3/4] Add support to "Check npm" workflow for projects in subfolders The "Check npm" workflow checks for problems with the npm configuration files of a repository. Previously this workflow assumed that a repository would only ever have these files in the root folder. However, a repository might also contain multiple separate npm-managed projects in arbitrary subfolders. Support for any repository structure is added to the workflow by using a job matrix to check the projects under an array of arbitrary paths that can be configured for the specific repository the template is installed in. --- .github/workflows/check-npm-task.yml | 12 ++++++++++-- Taskfile.yml | 4 ++++ workflow-templates/assets/npm-task/Taskfile.yml | 4 ++++ workflow-templates/check-npm-task.md | 5 +++++ workflow-templates/check-npm-task.yml | 12 ++++++++++-- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-npm-task.yml b/.github/workflows/check-npm-task.yml index 592b1611..5ba8a177 100644 --- a/.github/workflows/check-npm-task.yml +++ b/.github/workflows/check-npm-task.yml @@ -77,12 +77,20 @@ jobs: run: task --silent npm:validate check-sync: + name: check-sync (${{ matrix.project.path }}) needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest permissions: contents: read + strategy: + fail-fast: false + matrix: + project: + # TODO: add paths of all npm-managed projects in the repository here. + - path: . + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -99,7 +107,7 @@ jobs: version: 3.x - name: Install npm dependencies - run: task npm:install-deps + run: task npm:install-deps PROJECT_PATH="${{ matrix.project.path }}" - name: Check package-lock.json - run: git diff --color --exit-code package-lock.json + run: git diff --color --exit-code "${{ matrix.project.path }}/package-lock.json" diff --git a/Taskfile.yml b/Taskfile.yml index 6cf7f0b2..23f1f2b3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -722,9 +722,13 @@ tasks: -s "{{.SCHEMA_PATH}}" \ -d "{{.DATA_PATH}}" + # Parameter variables: + # - PROJECT_PATH: path of the npm-managed project. Default value: "./" # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/npm-task/Taskfile.yml npm:install-deps: desc: Install dependencies managed by npm + dir: | + "{{default "./" .PROJECT_PATH}}" cmds: - npm install diff --git a/workflow-templates/assets/npm-task/Taskfile.yml b/workflow-templates/assets/npm-task/Taskfile.yml index f569eded..0d28ec97 100644 --- a/workflow-templates/assets/npm-task/Taskfile.yml +++ b/workflow-templates/assets/npm-task/Taskfile.yml @@ -2,8 +2,12 @@ version: "3" tasks: + # Parameter variables: + # - PROJECT_PATH: path of the npm-managed project. Default value: "./" # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/npm-task/Taskfile.yml npm:install-deps: desc: Install dependencies managed by npm + dir: | + "{{default "./" .PROJECT_PATH}}" cmds: - npm install diff --git a/workflow-templates/check-npm-task.md b/workflow-templates/check-npm-task.md index a05ee1d3..3095e690 100644 --- a/workflow-templates/check-npm-task.md +++ b/workflow-templates/check-npm-task.md @@ -23,6 +23,11 @@ Install the [check-npm-task.yml](check-npm-task.yml) GitHub Actions workflow to Configure the version of Node.js used for development of the project in the `env.NODE_VERSION` field of `check-npm-task.yml`. +If the project contains **npm**-managed projects (i.e., a folder containing a `package.json` file) in paths other than the root of the repository, add their paths to the [job matrices](https://docs.github.com/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrix) of `check-npm-task.yml` at: + +- `jobs.validate.strategy.matrix.project[].path` +- `jobs.check-sync.strategy.matrix.project[].path` + ## Readme badge Markdown badge: diff --git a/workflow-templates/check-npm-task.yml b/workflow-templates/check-npm-task.yml index 592b1611..5ba8a177 100644 --- a/workflow-templates/check-npm-task.yml +++ b/workflow-templates/check-npm-task.yml @@ -77,12 +77,20 @@ jobs: run: task --silent npm:validate check-sync: + name: check-sync (${{ matrix.project.path }}) needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest permissions: contents: read + strategy: + fail-fast: false + matrix: + project: + # TODO: add paths of all npm-managed projects in the repository here. + - path: . + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -99,7 +107,7 @@ jobs: version: 3.x - name: Install npm dependencies - run: task npm:install-deps + run: task npm:install-deps PROJECT_PATH="${{ matrix.project.path }}" - name: Check package-lock.json - run: git diff --color --exit-code package-lock.json + run: git diff --color --exit-code "${{ matrix.project.path }}/package-lock.json" From b7d77f123beed4200f6c32bb9b8b9cc4147b3dfa Mon Sep 17 00:00:00 2001 From: per1234 Date: Tue, 21 Nov 2023 23:43:34 -0800 Subject: [PATCH 4/4] Only validate package.json files in specified paths The project infrastructure validates the package.json npm configuration files against their JSON schema. Previously, in order to provide validation coverage for all package.json files in any locations in the repository, a "globstar" was used to cause the validator to recursively search the entire file tree under the repository. That approach is problematic because the repository contains externally maintained files (e.g., the npm packages under the node_modules folder). Searching and validating these files is inefficient at best and the cause of spurious failures at worst. This is avoided by targeting the search. Support for a repository maintainer to configure any number of specific locations of npm-managed projects in the "Check npm" workflow has been added, so this system is used to target the validations. When the `npm:validate` task is ran by a contributor on their local clone, it defaults to the root of the repository, but the path can be configured by setting the PROJECT_PATH taskfile variable via an argument to the task invocation command. --- .github/workflows/check-npm-task.yml | 10 +++++++++- Taskfile.yml | 5 ++++- workflow-templates/assets/check-npm-task/Taskfile.yml | 5 ++++- workflow-templates/check-npm-task.yml | 10 +++++++++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-npm-task.yml b/.github/workflows/check-npm-task.yml index 5ba8a177..a397ee7b 100644 --- a/.github/workflows/check-npm-task.yml +++ b/.github/workflows/check-npm-task.yml @@ -52,12 +52,20 @@ jobs: echo "result=$RESULT" >> $GITHUB_OUTPUT validate: + name: validate (${{ matrix.project.path }}) needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest permissions: contents: read + strategy: + fail-fast: false + matrix: + project: + # TODO: add paths of all npm-managed projects in the repository here. + - path: . + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -74,7 +82,7 @@ jobs: version: 3.x - name: Validate package.json - run: task --silent npm:validate + run: task --silent npm:validate PROJECT_PATH="${{ matrix.project.path }}" check-sync: name: check-sync (${{ matrix.project.path }}) diff --git a/Taskfile.yml b/Taskfile.yml index 23f1f2b3..8c07b7ea 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -732,6 +732,8 @@ tasks: cmds: - npm install + # Parameter variables: + # - PROJECT_PATH: path of the npm-managed project. Default value: "./" # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-npm-task/Taskfile.yml npm:validate: desc: Validate npm configuration files against their JSON schema @@ -768,7 +770,8 @@ tasks: STYLELINTRC_SCHEMA_URL: https://json.schemastore.org/stylelintrc.json STYLELINTRC_SCHEMA_PATH: sh: task utility:mktemp-file TEMPLATE="stylelintrc-schema-XXXXXXXXXX.json" - INSTANCE_PATH: "**/package.json" + INSTANCE_PATH: >- + {{default "." .PROJECT_PATH}}/package.json PROJECT_FOLDER: sh: pwd WORKING_FOLDER: diff --git a/workflow-templates/assets/check-npm-task/Taskfile.yml b/workflow-templates/assets/check-npm-task/Taskfile.yml index 0aeb6975..a44582ce 100644 --- a/workflow-templates/assets/check-npm-task/Taskfile.yml +++ b/workflow-templates/assets/check-npm-task/Taskfile.yml @@ -6,6 +6,8 @@ vars: SCHEMA_DRAFT_4_AJV_CLI_VERSION: 3.3.0 tasks: + # Parameter variables: + # - PROJECT_PATH: path of the npm-managed project. Default value: "./" # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-npm-task/Taskfile.yml npm:validate: desc: Validate npm configuration files against their JSON schema @@ -42,7 +44,8 @@ tasks: STYLELINTRC_SCHEMA_URL: https://json.schemastore.org/stylelintrc.json STYLELINTRC_SCHEMA_PATH: sh: task utility:mktemp-file TEMPLATE="stylelintrc-schema-XXXXXXXXXX.json" - INSTANCE_PATH: "**/package.json" + INSTANCE_PATH: >- + {{default "." .PROJECT_PATH}}/package.json PROJECT_FOLDER: sh: pwd WORKING_FOLDER: diff --git a/workflow-templates/check-npm-task.yml b/workflow-templates/check-npm-task.yml index 5ba8a177..a397ee7b 100644 --- a/workflow-templates/check-npm-task.yml +++ b/workflow-templates/check-npm-task.yml @@ -52,12 +52,20 @@ jobs: echo "result=$RESULT" >> $GITHUB_OUTPUT validate: + name: validate (${{ matrix.project.path }}) needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest permissions: contents: read + strategy: + fail-fast: false + matrix: + project: + # TODO: add paths of all npm-managed projects in the repository here. + - path: . + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -74,7 +82,7 @@ jobs: version: 3.x - name: Validate package.json - run: task --silent npm:validate + run: task --silent npm:validate PROJECT_PATH="${{ matrix.project.path }}" check-sync: name: check-sync (${{ matrix.project.path }})