From ac45403bb27bd78b2d5924d84889b5dec16debf9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 28 Apr 2022 08:46:45 -0400 Subject: [PATCH 1/3] CI: Cache test data --- .circleci/config.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a33b9df25..8fedd5e52c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,6 +134,23 @@ jobs: - docker/Dockerfile.base-pruned - docker/get_base_image.sh + get_test_data: + machine: *machine_kwds + working_directory: /home/circleci/nipype + steps: + - restore_cache: + keys: + - data-v0-{{ .Branch }}-{{ .Revision }} + - data-v0--{{ .Revision }} + - data-v0-{{ .Branch }}- + - data-v0-master- + - data-v0- + - run: *_download_test_data + - save_cache: + key: data-v0-{{ .Branch }}-{{ .Revision }} + paths: + - /home/circleci/examples + test_pytest: machine: *machine_kwds working_directory: /home/circleci/nipype @@ -142,13 +159,15 @@ jobs: path: /home/circleci/nipype - attach_workspace: at: /tmp + - restore_cache: + keys: + - data-v0-{{ .Branch }}-{{ .Revision }} - run: *set_pr_number - run: *generate_dockerfiles - run: *modify_nipype_version - run: *get_base_image - run: *build_main_image_py38 - run: *_get_codecov - - run: *_download_test_data - run: *prepare_working_directory - run: name: Run pytests @@ -248,6 +267,13 @@ workflows: - /docs?\/.*/ tags: only: /.*/ + - get_test_data: + filters: + branches: + ignore: + - /docs?\/.*/ + tags: + only: /.*/ - test_pytest: filters: branches: @@ -257,6 +283,7 @@ workflows: only: /.*/ requires: - compare_base_dockerfiles + - get_test_data - deploy_dockerhub: filters: branches: From b4466b5625c00953a71fa0e6845a01fbfad23914 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 28 Apr 2022 08:59:43 -0400 Subject: [PATCH 2/3] CI: Increase verbosity --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8fedd5e52c..8c2701be1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,14 +64,15 @@ _download_test_data: &_download_test_data environment: OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" command: | + set -x export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xjv export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xzv export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xzv _prepare_working_directory: &prepare_working_directory name: Prepare working directory From 8dcd04bfc8daf0777339eef4d4846075a8ab830b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 28 Apr 2022 09:08:45 -0400 Subject: [PATCH 3/3] CI: Check for pre-existing directories --- .circleci/config.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c2701be1f..5930e43ab3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,13 +66,19 @@ _download_test_data: &_download_test_data command: | set -x export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xjv + if [[ ! -d nipype-tutorial ]]; then + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xjv + fi export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xzv + if [[ ! -d nipype-fsl_course_data ]]; then + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xzv + fi export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xzv + if [[ ! -d feeds ]]; then + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xzv + fi _prepare_working_directory: &prepare_working_directory name: Prepare working directory