From 53be2724157ca6fe9412e52dea56d4b29759ac11 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 23 Oct 2019 15:56:21 +0200 Subject: [PATCH 1/3] ci: enable "run when submodule changes" with environment variables We have a job in our CI (PR's x86_64-gnu-tools) that's supposed to run only when a submodule is changed in the PR, and it works by having a task at the start of the build that skips all the following tasks if the condition isn't met. Before this commit that task was gated with template parameters, which is a unique feature of Azure Pipelines. To make our CI more generic this commit switches the gate to use a simple environment variable plus a condition, which should be supported on more CI providers. --- src/ci/azure-pipelines/pr.yml | 14 +++----------- src/ci/azure-pipelines/steps/run.yml | 29 ++++++++++++---------------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/ci/azure-pipelines/pr.yml b/src/ci/azure-pipelines/pr.yml index 62e23efe1ef16..566e654fdb3f0 100644 --- a/src/ci/azure-pipelines/pr.yml +++ b/src/ci/azure-pipelines/pr.yml @@ -22,14 +22,6 @@ jobs: IMAGE: x86_64-gnu-llvm-6.0 mingw-check: IMAGE: mingw-check - -- job: LinuxTools - timeoutInMinutes: 600 - pool: - vmImage: ubuntu-16.04 - steps: - - template: steps/run.yml - parameters: - only_on_updated_submodules: 'yes' - variables: - IMAGE: x86_64-gnu-tools + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 01858cca50b1c..f3339808567e7 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -6,11 +6,6 @@ # # Check travis config for `gdb --batch` command to print all crash logs -parameters: - # When this parameter is set to anything other than an empty string the tests - # will only be executed when the commit updates submodules - only_on_updated_submodules: '' - steps: # Disable automatic line ending conversion, which is enabled by default on @@ -29,18 +24,18 @@ steps: # Set the SKIP_JOB environment variable if this job is supposed to only run # when submodules are updated and they were not. The following time consuming # tasks will be skipped when the environment variable is present. -- ${{ if parameters.only_on_updated_submodules }}: - - bash: | - set -e - # Submodules pseudo-files inside git have the 160000 permissions, so when - # those files are present in the diff a submodule was updated. - if git diff HEAD^ | grep "^index .* 160000" >/dev/null 2>&1; then - echo "Executing the job since submodules are updated" - else - echo "Not executing this job since no submodules were updated" - echo "##vso[task.setvariable variable=SKIP_JOB;]1" - fi - displayName: Decide whether to run this job +- bash: | + set -e + # Submodules pseudo-files inside git have the 160000 permissions, so when + # those files are present in the diff a submodule was updated. + if git diff HEAD^ | grep "^index .* 160000" >/dev/null 2>&1; then + echo "Executing the job since submodules are updated" + else + echo "Not executing this job since no submodules were updated" + echo "##vso[task.setvariable variable=SKIP_JOB;]1" + fi + displayName: Decide whether to run this job + condition: and(succeeded(), variables.CI_ONLY_WHEN_SUBMODULES_CHANGED) # Spawn a background process to collect CPU usage statistics which we'll upload # at the end of the build. See the comments in the script here for more From 4fb8a9a73cb5c42543e02ce1d797a6d028f6068f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 23 Oct 2019 16:07:13 +0200 Subject: [PATCH 2/3] ci: extract job skipping logic into a script --- src/ci/azure-pipelines/steps/run.yml | 15 +-------------- src/ci/scripts/should-skip-this.sh | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 14 deletions(-) create mode 100755 src/ci/scripts/should-skip-this.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index f3339808567e7..b8e32cf2cdfe3 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -21,21 +21,8 @@ steps: - checkout: self fetchDepth: 2 -# Set the SKIP_JOB environment variable if this job is supposed to only run -# when submodules are updated and they were not. The following time consuming -# tasks will be skipped when the environment variable is present. -- bash: | - set -e - # Submodules pseudo-files inside git have the 160000 permissions, so when - # those files are present in the diff a submodule was updated. - if git diff HEAD^ | grep "^index .* 160000" >/dev/null 2>&1; then - echo "Executing the job since submodules are updated" - else - echo "Not executing this job since no submodules were updated" - echo "##vso[task.setvariable variable=SKIP_JOB;]1" - fi +- bash: src/ci/scripts/should-skip-this.sh displayName: Decide whether to run this job - condition: and(succeeded(), variables.CI_ONLY_WHEN_SUBMODULES_CHANGED) # Spawn a background process to collect CPU usage statistics which we'll upload # at the end of the build. See the comments in the script here for more diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh new file mode 100755 index 0000000000000..a38099a2db7ff --- /dev/null +++ b/src/ci/scripts/should-skip-this.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Set the SKIP_JOB environment variable if this job is supposed to only run +# when submodules are updated and they were not. The following time consuming +# tasks will be skipped when the environment variable is present. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then + echo "Executing the job since there is no skip rule in effect" +elif git diff HEAD^ | grep "^index .* 160000" >/dev/null 2>&1; then + # Submodules pseudo-files inside git have the 160000 permissions, so when + # those files are present in the diff a submodule was updated. + echo "Executing the job since submodules are updated" +else + echo "Not executing this job since no submodules were updated" + ciCommandSetEnv SKIP_JOB 1 +fi From 95ad6c33c7744da263c6cf05840cbfdfd790bb89 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 29 Oct 2019 10:32:51 +0100 Subject: [PATCH 3/3] Apply suggestions from lzutao Co-Authored-By: lzutao --- src/ci/scripts/should-skip-this.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index a38099a2db7ff..f945db0ada27d 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -10,7 +10,7 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then echo "Executing the job since there is no skip rule in effect" -elif git diff HEAD^ | grep "^index .* 160000" >/dev/null 2>&1; then +elif git diff HEAD^ | grep --quiet "^index .* 160000"; then # Submodules pseudo-files inside git have the 160000 permissions, so when # those files are present in the diff a submodule was updated. echo "Executing the job since submodules are updated"