From 02607b87e542bb361795da0b1b3ce6ba5f57a833 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 12:21:13 -0600 Subject: [PATCH 01/28] TEST: Split Circle workflow --- .circleci/config.yml | 159 +++++++++++++++--- .circleci/pytests.sh | 12 ++ .circleci/test_complete.sh | 12 ++ .circleci/test_init.sh | 12 ++ .../test_py2_fmri_spm_nested_multiproc_l2.sh | 8 + .../test_py3_fmri_fsl_feeds_linear_l1.sh | 8 + .../test_py3_fmri_fsl_reuse_linear_l1.sh | 8 + 7 files changed, 192 insertions(+), 27 deletions(-) create mode 100644 .circleci/pytests.sh create mode 100644 .circleci/test_complete.sh create mode 100644 .circleci/test_init.sh create mode 100644 .circleci/test_py2_fmri_spm_nested_multiproc_l2.sh create mode 100644 .circleci/test_py3_fmri_fsl_feeds_linear_l1.sh create mode 100644 .circleci/test_py3_fmri_fsl_reuse_linear_l1.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index e922b37520..e1cdd705e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,9 +35,7 @@ jobs: - docker/Dockerfile.base-pruned - docker/get_base_image.sh - - build_and_test: - parallelism: 4 + fetch_base: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -45,6 +43,7 @@ jobs: steps: - checkout: path: /home/circleci/nipype + - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -75,34 +74,141 @@ jobs: exit 1 fi - run: - name: Build main image (py36) + name: Save Docker images to workspace + no_output_timeout: 60m + command: | + docker save nipype/nipype:base | gzip -1 > /tmp/docker/nipype-base.tar.gz \ + && du -h /tmp/docker/nipype-base.tar.gz + - persist_to_workspace: + root: /tmp + paths: + - docker/* + + build_py2: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - checkout: + path: /home/circleci/nipype + - setup_remote_docker + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base.tar.gz + - run: + name: Get test dependencies and generate Dockerfiles + command: | + pip install --no-cache-dir codecov + make gen-dockerfiles + - run: + name: Modify Nipype version if necessary + command: | + if [ "$CIRCLE_TAG" != "" ]; then + sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + fi + - run: + name: Build main image (py27) no_output_timeout: 60m command: | e=1 && for i in {1..5}; do docker build \ --rm=false \ - --tag nipype/nipype:latest \ - --tag nipype/nipype:py36 \ + --tag nipype/nipype:py27 \ + --build-arg PYTHON_VERSION_MAJOR=2 \ + --build-arg PYTHON_VERSION_MINOR=7 \ --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ + --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ && e=0 && break || sleep 15 done && [ "$e" -eq "0" ] - run: - name: Build main image (py27) + name: Save Docker images to workspace + no_output_timeout: 60m + command: | + docker save nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-py27.tar.gz \ + && du -h /tmp/docker/nipype-py27.tar.gz + - persist_to_workspace: + root: /tmp + paths: + - docker/* + + + build_py3: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - checkout: + path: /home/circleci/nipype + - setup_remote_docker + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base.tar.gz + - run: + name: Get test dependencies and generate Dockerfiles + command: | + pip install --no-cache-dir codecov + make gen-dockerfiles + - run: + name: Modify Nipype version if necessary + command: | + if [ "$CIRCLE_TAG" != "" ]; then + sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + fi + - run: + name: Build main image (py36) no_output_timeout: 60m command: | e=1 && for i in {1..5}; do docker build \ --rm=false \ - --tag nipype/nipype:py27 \ - --build-arg PYTHON_VERSION_MAJOR=2 \ - --build-arg PYTHON_VERSION_MINOR=7 \ + --tag nipype/nipype:latest \ + --tag nipype/nipype:py36 \ --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ + --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ && e=0 && break || sleep 15 done && [ "$e" -eq "0" ] + - run: + name: Save Docker images to workspace + no_output_timeout: 60m + command: | + docker save nipype/nipype:latest \ + nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-latest-py36.tar.gz \ + && du -h /tmp/docker/nipype-latest-py36.tar.gz + - persist_to_workspace: + root: /tmp + paths: + - docker/* + + + build_and_test: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base.tar.gz \ + && docker load < /tmp/docker/nipype-py27.tar.gz \ + && docker load < /tmp/docker/nipype-latest-py36.tar.gz + - checkout: + path: /home/circleci/nipype - run: name: Download test data no_output_timeout: 20m @@ -120,26 +226,15 @@ jobs: curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - run: name: Run tests - no_output_timeout: 4h + no_output_timeout: 2h environment: WORKDIR: /home/circleci/work command: | mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/tests.sh + bash /home/circleci/nipype/.circleci/pytests.sh - store_artifacts: path: /home/circleci/work/tests - - run: - name: Save Docker images to workspace - no_output_timeout: 60m - command: | - if [ "$CIRCLE_NODE_INDEX" -eq "0" ] && [ "$CIRCLE_BRANCH" == "master" ]; then - docker save nipype/nipype:base \ - nipype/nipype:latest \ - nipype/nipype:py36 \ - nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz - du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz - fi - persist_to_workspace: root: /tmp paths: @@ -180,12 +275,22 @@ jobs: workflows: version: 2 - build_test_deply: + build_test_deploy: jobs: - compare_base_dockerfiles - - build_and_test: + - fetch_base: requires: - compare_base_dockerfiles + - build_py2: + requires: + - fetch_base + - build_py3: + requires: + - fetch_base + - build_and_test: + requires: + - build_py2 + - build_py3 - deploy: filters: branches: diff --git a/.circleci/pytests.sh b/.circleci/pytests.sh new file mode 100644 index 0000000000..c92de4a582 --- /dev/null +++ b/.circleci/pytests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +. test_init.sh + +# These tests are manually balanced based on previous build timings. +# They may need to be rebalanced in the future. +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \ +&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \ +&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh +exitcode=$? + +. test_complete.sh diff --git a/.circleci/test_complete.sh b/.circleci/test_complete.sh new file mode 100644 index 0000000000..5c138a2d82 --- /dev/null +++ b/.circleci/test_complete.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Source after running test and storing relevant return code in $exitcode + +# Exit with error if any of the tests failed +if [ "$exitcode" != "0" ]; then exit 1; fi + +codecov --file "${WORKDIR}/tests/coverage*.xml" \ + --root "${HOME}/nipype/" --flags unittests -e SCRIPT_NAME + +codecov --file "${WORKDIR}/tests/smoketest*.xml" \ + --root "${HOME}/nipype/" --flags smoketests -e SCRIPT_NAME diff --git a/.circleci/test_init.sh b/.circleci/test_init.sh new file mode 100644 index 0000000000..373d5efe7a --- /dev/null +++ b/.circleci/test_init.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Source before running tests + +# Setting # $ help set +set -e # Exit immediately if a command exits with a non-zero status. +set -u # Treat unset variables as an error when substituting. +set -x # Print command traces before executing command. + +export DOCKER_IMAGE="nipype/nipype" + +export SCRIPT_NAME=`basename $0` diff --git a/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh new file mode 100644 index 0000000000..276511542a --- /dev/null +++ b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline +exitcode=$? + +. test_complete.sh diff --git a/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh new file mode 100644 index 0000000000..95534d127b --- /dev/null +++ b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline +exitcode=$? + +. test_complete.sh diff --git a/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh new file mode 100644 index 0000000000..df16db713a --- /dev/null +++ b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow +exitcode=$? + +. test_complete.sh From 700ba293066df9dbc0e1d2be04adb28527a011d5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 13:09:40 -0600 Subject: [PATCH 02/28] FIX: Remove setup_remote_docker from fetch/build --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1cdd705e1..139dabe516 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,6 @@ jobs: steps: - checkout: path: /home/circleci/nipype - - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -92,7 +91,6 @@ jobs: steps: - checkout: path: /home/circleci/nipype - - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -146,7 +144,6 @@ jobs: steps: - checkout: path: /home/circleci/nipype - - setup_remote_docker - attach_workspace: at: /tmp - run: From c2b0ebc2c6cfb96cd3be135477f557a85311ab67 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 13:37:24 -0600 Subject: [PATCH 03/28] CI: Build in serial, to reduce save/load steps --- .circleci/config.yml | 114 ++++++++----------------------------------- 1 file changed, 21 insertions(+), 93 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 139dabe516..6193922e40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,7 +35,7 @@ jobs: - docker/Dockerfile.base-pruned - docker/get_base_image.sh - fetch_base: + build_images: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -72,32 +72,6 @@ jobs: echo "Error: method to get base image not understood" exit 1 fi - - run: - name: Save Docker images to workspace - no_output_timeout: 60m - command: | - docker save nipype/nipype:base | gzip -1 > /tmp/docker/nipype-base.tar.gz \ - && du -h /tmp/docker/nipype-base.tar.gz - - persist_to_workspace: - root: /tmp - paths: - - docker/* - - build_py2: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - checkout: - path: /home/circleci/nipype - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base.tar.gz - run: name: Get test dependencies and generate Dockerfiles command: | @@ -110,86 +84,49 @@ jobs: sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py fi - run: - name: Build main image (py27) + name: Build main image (py36) no_output_timeout: 60m command: | e=1 && for i in {1..5}; do docker build \ --rm=false \ - --tag nipype/nipype:py27 \ - --build-arg PYTHON_VERSION_MAJOR=2 \ - --build-arg PYTHON_VERSION_MINOR=7 \ + --tag nipype/nipype:latest \ + --tag nipype/nipype:py36 \ --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ + --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ && e=0 && break || sleep 15 done && [ "$e" -eq "0" ] - run: - name: Save Docker images to workspace - no_output_timeout: 60m - command: | - docker save nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-py27.tar.gz \ - && du -h /tmp/docker/nipype-py27.tar.gz - - persist_to_workspace: - root: /tmp - paths: - - docker/* - - - build_py3: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - checkout: - path: /home/circleci/nipype - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base.tar.gz - - run: - name: Get test dependencies and generate Dockerfiles - command: | - pip install --no-cache-dir codecov - make gen-dockerfiles - - run: - name: Modify Nipype version if necessary - command: | - if [ "$CIRCLE_TAG" != "" ]; then - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py - fi - - run: - name: Build main image (py36) + name: Build main image (py27) no_output_timeout: 60m command: | e=1 && for i in {1..5}; do docker build \ --rm=false \ - --tag nipype/nipype:latest \ - --tag nipype/nipype:py36 \ + --tag nipype/nipype:py27 \ + --build-arg PYTHON_VERSION_MAJOR=2 \ + --build-arg PYTHON_VERSION_MINOR=7 \ --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ + --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ && e=0 && break || sleep 15 done && [ "$e" -eq "0" ] - run: name: Save Docker images to workspace no_output_timeout: 60m command: | - docker save nipype/nipype:latest \ - nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-latest-py36.tar.gz \ - && du -h /tmp/docker/nipype-latest-py36.tar.gz + docker save nipype/nipype:base \ + nipype/nipype:latest \ + nipype/nipype:py27 \ + nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz \ + && du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz - persist_to_workspace: root: /tmp paths: - docker/* - - build_and_test: + test: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -201,9 +138,7 @@ jobs: name: Load saved Docker images. no_output_timeout: 60m command: | - docker load < /tmp/docker/nipype-base.tar.gz \ - && docker load < /tmp/docker/nipype-py27.tar.gz \ - && docker load < /tmp/docker/nipype-latest-py36.tar.gz + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype - run: @@ -275,22 +210,15 @@ workflows: build_test_deploy: jobs: - compare_base_dockerfiles - - fetch_base: + - build_images: requires: - compare_base_dockerfiles - - build_py2: - requires: - - fetch_base - - build_py3: - requires: - - fetch_base - - build_and_test: + - test: requires: - - build_py2 - - build_py3 + - build_images - deploy: filters: branches: only: master requires: - - build_and_test + - test From ab1e6b55deb90564cdd6b6964163457f77e820a5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 13:57:18 -0600 Subject: [PATCH 04/28] CI: Add two tests in parallel --- .circleci/config.yml | 55 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6193922e40..afd5f26871 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,7 +126,7 @@ jobs: paths: - docker/* - test: + test_pytest: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -167,11 +167,48 @@ jobs: bash /home/circleci/nipype/.circleci/pytests.sh - store_artifacts: path: /home/circleci/work/tests - - persist_to_workspace: - root: /tmp - paths: - - docker/* + test_py2_fmri_spm_nested_multiproc_l2: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh + - store_artifacts: + path: /home/circleci/work/tests deploy: docker: @@ -213,7 +250,10 @@ workflows: - build_images: requires: - compare_base_dockerfiles - - test: + - test_pytest: + requires: + - build_images + - test_py2_fmri_spm_nested_multiproc_l2: requires: - build_images - deploy: @@ -221,4 +261,5 @@ workflows: branches: only: master requires: - - test + - test_pytest + - test_py2_fmri_spm_nested_multiproc_l2 From 34ea71832cc9c068812a04c6611fd6dff2c58ef5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 14:05:01 -0600 Subject: [PATCH 05/28] CI: Remove duplicate steps --- .circleci/config.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index afd5f26871..b2b98a9958 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,17 +72,6 @@ jobs: echo "Error: method to get base image not understood" exit 1 fi - - run: - name: Get test dependencies and generate Dockerfiles - command: | - pip install --no-cache-dir codecov - make gen-dockerfiles - - run: - name: Modify Nipype version if necessary - command: | - if [ "$CIRCLE_TAG" != "" ]; then - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py - fi - run: name: Build main image (py36) no_output_timeout: 60m From 2c6627834020ce4dd2559ad85ba85044bfc4ca39 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 14:10:26 -0600 Subject: [PATCH 06/28] CI: Full test suite --- .circleci/config.yml | 322 ++++++++++++++++++ .../test_py3_fmri_spm_dartel_linear_l1.sh | 8 + .../test_py3_fmri_spm_dartel_linear_l2.sh | 8 + .circleci/test_py3_fmri_spm_linear_3d.sh | 8 + .circleci/test_py3_fmri_spm_linear_4d.sh | 8 + .../test_py3_fmri_spm_nested_multiproc_l1.sh | 8 + 6 files changed, 362 insertions(+) create mode 100644 .circleci/test_py3_fmri_spm_dartel_linear_l1.sh create mode 100644 .circleci/test_py3_fmri_spm_dartel_linear_l2.sh create mode 100644 .circleci/test_py3_fmri_spm_linear_3d.sh create mode 100644 .circleci/test_py3_fmri_spm_linear_4d.sh create mode 100644 .circleci/test_py3_fmri_spm_nested_multiproc_l1.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index b2b98a9958..9ee472b712 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -199,6 +199,300 @@ jobs: - store_artifacts: path: /home/circleci/work/tests + test_py3_fmri_fsl_feeds_linear_l1: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh + - store_artifacts: + path: /home/circleci/work/tests + + test_py3_fmri_fsl_reuse_linear_l1: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh + - store_artifacts: + path: /home/circleci/work/tests + + test_py3_fmri_spm_dartel_linear_l1: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh + - store_artifacts: + path: /home/circleci/work/tests + + test_py3_fmri_spm_dartel_linear_l2: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh + - store_artifacts: + path: /home/circleci/work/tests + + test_py3_fmri_spm_linear_3d: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh + - store_artifacts: + path: /home/circleci/work/tests + + test_py3_fmri_spm_linear_4d: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh + - store_artifacts: + path: /home/circleci/work/tests + + test_py3_fmri_spm_nested_multiproc_l1: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 + working_directory: /home/circleci/nipype + steps: + - attach_workspace: + at: /tmp + - run: + name: Load saved Docker images. + no_output_timeout: 60m + command: | + docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - checkout: + path: /home/circleci/nipype + - run: + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + - run: + name: Run tests + no_output_timeout: 2h + environment: + WORKDIR: /home/circleci/work + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh + - store_artifacts: + path: /home/circleci/work/tests + deploy: docker: - image: docker:17.10.0-ce-git @@ -245,6 +539,27 @@ workflows: - test_py2_fmri_spm_nested_multiproc_l2: requires: - build_images + - test_py3_fmri_fsl_feeds_linear_l1: + requires: + - build_images + - test_py3_fmri_fsl_reuse_linear_l1: + requires: + - build_images + - test_py3_fmri_spm_dartel_linear_l1: + requires: + - build_images + - test_py3_fmri_spm_dartel_linear_l2: + requires: + - build_images + - test_py3_fmri_spm_linear_3d: + requires: + - build_images + - test_py3_fmri_spm_linear_4d: + requires: + - build_images + - test_py3_fmri_spm_nested_multiproc_l1: + requires: + - build_images - deploy: filters: branches: @@ -252,3 +567,10 @@ workflows: requires: - test_pytest - test_py2_fmri_spm_nested_multiproc_l2 + - test_py3_fmri_fsl_feeds_linear_l1 + - test_py3_fmri_fsl_reuse_linear_l1 + - test_py3_fmri_spm_dartel_linear_l1 + - test_py3_fmri_spm_dartel_linear_l2 + - test_py3_fmri_spm_linear_3d + - test_py3_fmri_spm_linear_4d + - test_py3_fmri_spm_nested_multiproc_l1 diff --git a/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh b/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh new file mode 100644 index 0000000000..d9ca4e9927 --- /dev/null +++ b/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 +exitcode=$? + +. test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh b/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh new file mode 100644 index 0000000000..9122860ab7 --- /dev/null +++ b/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline +exitcode=$? + +. test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_linear_3d.sh b/.circleci/test_py3_fmri_spm_linear_3d.sh new file mode 100644 index 0000000000..8ab987f113 --- /dev/null +++ b/.circleci/test_py3_fmri_spm_linear_3d.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d +exitcode=$? + +. test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_linear_4d.sh b/.circleci/test_py3_fmri_spm_linear_4d.sh new file mode 100644 index 0000000000..0025971b75 --- /dev/null +++ b/.circleci/test_py3_fmri_spm_linear_4d.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d +exitcode=$? + +. test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh new file mode 100644 index 0000000000..2a4a4916c3 --- /dev/null +++ b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 +exitcode=$? + +. test_complete.sh From 7d7ff1932044095223b9f893e38bf770b915a872 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 14:56:02 -0600 Subject: [PATCH 07/28] CI: Add explicit directory to source lines --- .circleci/pytests.sh | 6 ++++-- .circleci/test_py2_fmri_spm_nested_multiproc_l2.sh | 6 ++++-- .circleci/test_py3_fmri_fsl_feeds_linear_l1.sh | 6 ++++-- .circleci/test_py3_fmri_fsl_reuse_linear_l1.sh | 6 ++++-- .circleci/test_py3_fmri_spm_dartel_linear_l1.sh | 6 ++++-- .circleci/test_py3_fmri_spm_dartel_linear_l2.sh | 6 ++++-- .circleci/test_py3_fmri_spm_linear_3d.sh | 6 ++++-- .circleci/test_py3_fmri_spm_linear_4d.sh | 6 ++++-- .circleci/test_py3_fmri_spm_nested_multiproc_l1.sh | 6 ++++-- 9 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.circleci/pytests.sh b/.circleci/pytests.sh index c92de4a582..7d8bb27553 100644 --- a/.circleci/pytests.sh +++ b/.circleci/pytests.sh @@ -1,6 +1,8 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh # These tests are manually balanced based on previous build timings. # They may need to be rebalanced in the future. @@ -9,4 +11,4 @@ docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh index 276511542a..f575e40e0d 100644 --- a/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh +++ b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh index 95534d127b..435d08738a 100644 --- a/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh +++ b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh index df16db713a..9ac6d77d91 100644 --- a/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh +++ b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh b/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh index d9ca4e9927..d10550f2ed 100644 --- a/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh +++ b/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh b/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh index 9122860ab7..398a308df6 100644 --- a/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh +++ b/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_linear_3d.sh b/.circleci/test_py3_fmri_spm_linear_3d.sh index 8ab987f113..dd3542b542 100644 --- a/.circleci/test_py3_fmri_spm_linear_3d.sh +++ b/.circleci/test_py3_fmri_spm_linear_3d.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_linear_4d.sh b/.circleci/test_py3_fmri_spm_linear_4d.sh index 0025971b75..05a21fc786 100644 --- a/.circleci/test_py3_fmri_spm_linear_4d.sh +++ b/.circleci/test_py3_fmri_spm_linear_4d.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh index 2a4a4916c3..de1dcd77fb 100644 --- a/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh +++ b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh @@ -1,8 +1,10 @@ #!/bin/bash -. test_init.sh +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 exitcode=$? -. test_complete.sh +. $DIR/test_complete.sh From 0dba5427110725d197bd3072b34dd68013b28623 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 20 Jan 2018 16:57:54 -0600 Subject: [PATCH 08/28] CI: Install codecov on test jobs --- .circleci/config.yml | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ee472b712..fb4ca226f7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,9 +46,8 @@ jobs: - attach_workspace: at: /tmp - run: - name: Get test dependencies and generate Dockerfiles + name: Generate Dockerfiles command: | - pip install --no-cache-dir codecov make gen-dockerfiles - run: name: Modify Nipype version if necessary @@ -130,6 +129,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -172,6 +175,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -214,6 +221,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -256,6 +267,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -298,6 +313,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -340,6 +359,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -382,6 +405,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -424,6 +451,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m @@ -466,6 +497,10 @@ jobs: docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - checkout: path: /home/circleci/nipype + - run: + name: Get codecov + command: | + pip install --no-cache-dir codecov - run: name: Download test data no_output_timeout: 20m From 83a15306d3a999639bd6e8528d03f0dab8b0a1b8 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sat, 20 Jan 2018 20:57:04 -0600 Subject: [PATCH 09/28] CI: Use runtimes to regroup tests Runtimes aggregated from https://circleci.com/workflow-run/721d8f15-dac6-4b7d-87c9-7dbeb1709f1a Two workflows require pre-run of other workflows. --- .circleci/config.yml | 269 ++----------------------------------------- 1 file changed, 12 insertions(+), 257 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb4ca226f7..b934029823 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,53 +160,7 @@ jobs: - store_artifacts: path: /home/circleci/work/tests - test_py2_fmri_spm_nested_multiproc_l2: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Run tests - no_output_timeout: 2h - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - - store_artifacts: - path: /home/circleci/work/tests - - test_py3_fmri_fsl_feeds_linear_l1: + test_py3_fmri_fsl_spm: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -249,56 +203,13 @@ jobs: mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh - - store_artifacts: - path: /home/circleci/work/tests - - test_py3_fmri_fsl_reuse_linear_l1: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Run tests - no_output_timeout: 2h - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh - store_artifacts: path: /home/circleci/work/tests - test_py3_fmri_spm_dartel_linear_l1: + test_py3_fmri_spm_dartel_linear: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -341,148 +252,11 @@ jobs: mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh - - store_artifacts: - path: /home/circleci/work/tests - - test_py3_fmri_spm_dartel_linear_l2: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Run tests - no_output_timeout: 2h - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh - store_artifacts: path: /home/circleci/work/tests - test_py3_fmri_spm_linear_3d: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Run tests - no_output_timeout: 2h - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh - - store_artifacts: - path: /home/circleci/work/tests - - test_py3_fmri_spm_linear_4d: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Run tests - no_output_timeout: 2h - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh - - store_artifacts: - path: /home/circleci/work/tests - - test_py3_fmri_spm_nested_multiproc_l1: + test_fmri_spm_nested_multiproc: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -525,6 +299,7 @@ jobs: mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh + bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - store_artifacts: path: /home/circleci/work/tests @@ -571,28 +346,13 @@ workflows: - test_pytest: requires: - build_images - - test_py2_fmri_spm_nested_multiproc_l2: - requires: - - build_images - - test_py3_fmri_fsl_feeds_linear_l1: - requires: - - build_images - - test_py3_fmri_fsl_reuse_linear_l1: - requires: - - build_images - - test_py3_fmri_spm_dartel_linear_l1: - requires: - - build_images - - test_py3_fmri_spm_dartel_linear_l2: - requires: - - build_images - - test_py3_fmri_spm_linear_3d: + - test_py3_fmri_fsl_spm: requires: - build_images - - test_py3_fmri_spm_linear_4d: + - test_py3_fmri_spm_dartel_linear: requires: - build_images - - test_py3_fmri_spm_nested_multiproc_l1: + - test_fmri_spm_nested_multiproc: requires: - build_images - deploy: @@ -601,11 +361,6 @@ workflows: only: master requires: - test_pytest - - test_py2_fmri_spm_nested_multiproc_l2 - - test_py3_fmri_fsl_feeds_linear_l1 - - test_py3_fmri_fsl_reuse_linear_l1 - - test_py3_fmri_spm_dartel_linear_l1 - - test_py3_fmri_spm_dartel_linear_l2 - - test_py3_fmri_spm_linear_3d - - test_py3_fmri_spm_linear_4d - - test_py3_fmri_spm_nested_multiproc_l1 + - test_fmri_spm_nested_multiproc + - test_py3_fmri_fsl_spm + - test_py3_fmri_spm_dartel_linear From a5c8228d7a9f9e1cc39c8d3d5e5172ff17c5cf69 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 24 Jan 2018 11:20:47 -0500 Subject: [PATCH 10/28] CI: Move to docker, attempt to use cache-from --- .circleci/config.yml | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b934029823..60801dbc80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,25 +36,25 @@ jobs: - docker/get_base_image.sh build_images: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + docker: + - image: docker:17.10.0-ce-git working_directory: /home/circleci/nipype steps: + - setup_remote_docker - checkout: path: /home/circleci/nipype - attach_workspace: at: /tmp - - run: - name: Generate Dockerfiles - command: | - make gen-dockerfiles - run: name: Modify Nipype version if necessary command: | if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py fi + - run: + name: Generate Dockerfiles + command: | + make gen-dockerfiles - run: name: Get base image (pull or build) no_output_timeout: 60m @@ -65,7 +65,7 @@ jobs: docker pull nipype/nipype:base elif [ "$GET_BASE" == "BUILD" ]; then e=1 && for i in {1..5}; do - docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 + docker build --cache-from=nipype/nipype -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 done && [ "$e" -eq "0" ] else echo "Error: method to get base image not understood" @@ -115,11 +115,11 @@ jobs: - docker/* test_pytest: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + docker: + - image: docker:17.10.0-ce-git working_directory: /home/circleci/nipype steps: + - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -166,6 +166,7 @@ jobs: image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: + - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -210,11 +211,11 @@ jobs: path: /home/circleci/work/tests test_py3_fmri_spm_dartel_linear: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + docker: + - image: docker:17.10.0-ce-git working_directory: /home/circleci/nipype steps: + - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -257,11 +258,11 @@ jobs: path: /home/circleci/work/tests test_fmri_spm_nested_multiproc: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + docker: + - image: docker:17.10.0-ce-git working_directory: /home/circleci/nipype steps: + - setup_remote_docker - attach_workspace: at: /tmp - run: From 36e8f69549379eb8675fa41091eee5650a39918a Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 24 Jan 2018 11:23:43 -0500 Subject: [PATCH 11/28] CI: Install dependencies --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60801dbc80..a717335db3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,6 +51,9 @@ jobs: if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py fi + - run: + name: Install build dependencies + command: apk update && apk add make bash - run: name: Generate Dockerfiles command: | From 49f7c690ed15e42237bfab5bee521de5b82f7ff8 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 24 Jan 2018 21:37:57 -0500 Subject: [PATCH 12/28] REV: Restore 610e571 --- .circleci/config.yml | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a717335db3..b934029823 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,28 +36,25 @@ jobs: - docker/get_base_image.sh build_images: - docker: - - image: docker:17.10.0-ce-git + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: - - setup_remote_docker - checkout: path: /home/circleci/nipype - attach_workspace: at: /tmp + - run: + name: Generate Dockerfiles + command: | + make gen-dockerfiles - run: name: Modify Nipype version if necessary command: | if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py fi - - run: - name: Install build dependencies - command: apk update && apk add make bash - - run: - name: Generate Dockerfiles - command: | - make gen-dockerfiles - run: name: Get base image (pull or build) no_output_timeout: 60m @@ -68,7 +65,7 @@ jobs: docker pull nipype/nipype:base elif [ "$GET_BASE" == "BUILD" ]; then e=1 && for i in {1..5}; do - docker build --cache-from=nipype/nipype -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 + docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 done && [ "$e" -eq "0" ] else echo "Error: method to get base image not understood" @@ -118,11 +115,11 @@ jobs: - docker/* test_pytest: - docker: - - image: docker:17.10.0-ce-git + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: - - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -169,7 +166,6 @@ jobs: image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: - - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -214,11 +210,11 @@ jobs: path: /home/circleci/work/tests test_py3_fmri_spm_dartel_linear: - docker: - - image: docker:17.10.0-ce-git + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: - - setup_remote_docker - attach_workspace: at: /tmp - run: @@ -261,11 +257,11 @@ jobs: path: /home/circleci/work/tests test_fmri_spm_nested_multiproc: - docker: - - image: docker:17.10.0-ce-git + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: - - setup_remote_docker - attach_workspace: at: /tmp - run: From b6932b36b05fa4fffa147c93c370d219a1fcd2d9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 24 Jan 2018 21:42:44 -0500 Subject: [PATCH 13/28] CI: Build images in each job --- .circleci/config.yml | 199 ++++++++++++++++++++++++++++++++----------- 1 file changed, 150 insertions(+), 49 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b934029823..87052480c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,7 +35,7 @@ jobs: - docker/Dockerfile.base-pruned - docker/get_base_image.sh - build_images: + test_pytest: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -100,35 +100,6 @@ jobs: --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ && e=0 && break || sleep 15 done && [ "$e" -eq "0" ] - - run: - name: Save Docker images to workspace - no_output_timeout: 60m - command: | - docker save nipype/nipype:base \ - nipype/nipype:latest \ - nipype/nipype:py27 \ - nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz \ - && du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - persist_to_workspace: - root: /tmp - paths: - - docker/* - - test_pytest: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 - working_directory: /home/circleci/nipype - steps: - - attach_workspace: - at: /tmp - - run: - name: Load saved Docker images. - no_output_timeout: 60m - command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype - run: name: Get codecov command: | @@ -166,15 +137,50 @@ jobs: image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: + - checkout: + path: /home/circleci/nipype - attach_workspace: at: /tmp - run: - name: Load saved Docker images. + name: Generate Dockerfiles + command: | + make gen-dockerfiles + - run: + name: Modify Nipype version if necessary + command: | + if [ "$CIRCLE_TAG" != "" ]; then + sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + fi + - run: + name: Get base image (pull or build) no_output_timeout: 60m command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype + source /tmp/docker/get_base_image.sh + if [ "$GET_BASE" == "PULL" ]; then + echo "Pulling base image ..." + docker pull nipype/nipype:base + elif [ "$GET_BASE" == "BUILD" ]; then + e=1 && for i in {1..5}; do + docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + else + echo "Error: method to get base image not understood" + exit 1 + fi + - run: + name: Build main image (py36) + no_output_timeout: 60m + command: | + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag nipype/nipype:latest \ + --tag nipype/nipype:py36 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] - run: name: Get codecov command: | @@ -215,15 +221,50 @@ jobs: image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: + - checkout: + path: /home/circleci/nipype - attach_workspace: at: /tmp - run: - name: Load saved Docker images. + name: Generate Dockerfiles + command: | + make gen-dockerfiles + - run: + name: Modify Nipype version if necessary + command: | + if [ "$CIRCLE_TAG" != "" ]; then + sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + fi + - run: + name: Get base image (pull or build) no_output_timeout: 60m command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype + source /tmp/docker/get_base_image.sh + if [ "$GET_BASE" == "PULL" ]; then + echo "Pulling base image ..." + docker pull nipype/nipype:base + elif [ "$GET_BASE" == "BUILD" ]; then + e=1 && for i in {1..5}; do + docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + else + echo "Error: method to get base image not understood" + exit 1 + fi + - run: + name: Build main image (py36) + no_output_timeout: 60m + command: | + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag nipype/nipype:latest \ + --tag nipype/nipype:py36 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] - run: name: Get codecov command: | @@ -262,15 +303,65 @@ jobs: image: circleci/classic:201710-02 working_directory: /home/circleci/nipype steps: + - checkout: + path: /home/circleci/nipype - attach_workspace: at: /tmp - run: - name: Load saved Docker images. + name: Generate Dockerfiles + command: | + make gen-dockerfiles + - run: + name: Modify Nipype version if necessary + command: | + if [ "$CIRCLE_TAG" != "" ]; then + sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + fi + - run: + name: Get base image (pull or build) no_output_timeout: 60m command: | - docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz - - checkout: - path: /home/circleci/nipype + source /tmp/docker/get_base_image.sh + if [ "$GET_BASE" == "PULL" ]; then + echo "Pulling base image ..." + docker pull nipype/nipype:base + elif [ "$GET_BASE" == "BUILD" ]; then + e=1 && for i in {1..5}; do + docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + else + echo "Error: method to get base image not understood" + exit 1 + fi + - run: + name: Build main image (py36) + no_output_timeout: 60m + command: | + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag nipype/nipype:latest \ + --tag nipype/nipype:py36 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + - run: + name: Build main image (py27) + no_output_timeout: 60m + command: | + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag nipype/nipype:py27 \ + --build-arg PYTHON_VERSION_MAJOR=2 \ + --build-arg PYTHON_VERSION_MINOR=7 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] - run: name: Get codecov command: | @@ -302,6 +393,19 @@ jobs: bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - store_artifacts: path: /home/circleci/work/tests + - run: + name: Save Docker images to workspace + no_output_timeout: 60m + command: | + docker save nipype/nipype:base \ + nipype/nipype:latest \ + nipype/nipype:py27 \ + nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz \ + && du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz + - persist_to_workspace: + root: /tmp + paths: + - docker/* deploy: docker: @@ -340,21 +444,18 @@ workflows: build_test_deploy: jobs: - compare_base_dockerfiles - - build_images: - requires: - - compare_base_dockerfiles - test_pytest: requires: - - build_images + - compare_base_dockerfiles - test_py3_fmri_fsl_spm: requires: - - build_images + - compare_base_dockerfiles - test_py3_fmri_spm_dartel_linear: requires: - - build_images + - compare_base_dockerfiles - test_fmri_spm_nested_multiproc: requires: - - build_images + - compare_base_dockerfiles - deploy: filters: branches: From 7e6760b64eb797dd8ce6c6fbe95fb665a09f5ade Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 24 Jan 2018 21:50:35 -0500 Subject: [PATCH 14/28] TEST: Move DARTEL to MultiProc --- .circleci/config.yml | 10 +++++----- ..._l1.sh => test_py3_fmri_spm_dartel_multiproc_l1.sh} | 2 +- ..._l2.sh => test_py3_fmri_spm_dartel_multiproc_l2.sh} | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename .circleci/{test_py3_fmri_spm_dartel_linear_l1.sh => test_py3_fmri_spm_dartel_multiproc_l1.sh} (81%) rename .circleci/{test_py3_fmri_spm_dartel_linear_l2.sh => test_py3_fmri_spm_dartel_multiproc_l2.sh} (80%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 87052480c4..a4245c7959 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,7 +215,7 @@ jobs: - store_artifacts: path: /home/circleci/work/tests - test_py3_fmri_spm_dartel_linear: + test_py3_fmri_spm_dartel_multiproc: machine: # Ubuntu 14.04 with Docker 17.10.0-ce image: circleci/classic:201710-02 @@ -292,8 +292,8 @@ jobs: command: | mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh + bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh - store_artifacts: path: /home/circleci/work/tests @@ -450,7 +450,7 @@ workflows: - test_py3_fmri_fsl_spm: requires: - compare_base_dockerfiles - - test_py3_fmri_spm_dartel_linear: + - test_py3_fmri_spm_dartel_multiproc: requires: - compare_base_dockerfiles - test_fmri_spm_nested_multiproc: @@ -464,4 +464,4 @@ workflows: - test_pytest - test_fmri_spm_nested_multiproc - test_py3_fmri_fsl_spm - - test_py3_fmri_spm_dartel_linear + - test_py3_fmri_spm_dartel_multiproc diff --git a/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh b/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh similarity index 81% rename from .circleci/test_py3_fmri_spm_dartel_linear_l1.sh rename to .circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh index d10550f2ed..ac17336b4b 100644 --- a/.circleci/test_py3_fmri_spm_dartel_linear_l1.sh +++ b/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh @@ -4,7 +4,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh -docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ level1 exitcode=$? . $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh b/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh similarity index 80% rename from .circleci/test_py3_fmri_spm_dartel_linear_l2.sh rename to .circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh index 398a308df6..ad4326eb47 100644 --- a/.circleci/test_py3_fmri_spm_dartel_linear_l2.sh +++ b/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh @@ -4,7 +4,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh -docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ l2pipeline exitcode=$? . $DIR/test_complete.sh From 1e9e54182276fe8ccb8e9e09cde2cd977732a212 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 25 Jan 2018 11:03:01 -0500 Subject: [PATCH 15/28] CI: Split tests in config, for timing steps --- .circleci/config.yml | 78 ++++++++++++++++++++++++++++-------- .circleci/pytests.sh | 14 ------- .circleci/test_py2_pytest.sh | 10 +++++ .circleci/test_py3_docs.sh | 10 +++++ .circleci/test_py3_pytest.sh | 10 +++++ 5 files changed, 91 insertions(+), 31 deletions(-) delete mode 100644 .circleci/pytests.sh create mode 100644 .circleci/test_py2_pytest.sh create mode 100644 .circleci/test_py3_docs.sh create mode 100644 .circleci/test_py3_pytest.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a4245c7959..c8de3f66ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,14 +120,21 @@ jobs: export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - run: - name: Run tests - no_output_timeout: 2h + name: Prepare working directory environment: WORKDIR: /home/circleci/work command: | mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/pytests.sh + - run: + name: Run pytests (py36) + command: bash /home/circleci/nipype/.circleci/test_py3_pytest.sh + - run: + name: Run pytests (py27) + command: bash /home/circleci/nipype/.circleci/test_py2_pytest.sh + - run: + name: Build docs (py36) + command: bash /home/circleci/nipype/.circleci/test_py3_docs.sh - store_artifacts: path: /home/circleci/work/tests @@ -201,17 +208,36 @@ jobs: export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - run: - name: Run tests - no_output_timeout: 2h + name: Prepare working directory environment: WORKDIR: /home/circleci/work command: | mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh - bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh + - run: + name: Run FSL FEEDS pipeline (py36) + no_output_timeout: 40m + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh + - run: + name: Run FSL reuse pipeline (py36) + no_output_timeout: 40m + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh + - run: + name: Run SPM test workflow - 3D inputs (py36) + no_output_timeout: 40m + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh + - run: + name: Run SPM test workflow - 4D inputs (py36) + no_output_timeout: 40m + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh - store_artifacts: path: /home/circleci/work/tests @@ -285,15 +311,24 @@ jobs: export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - run: - name: Run tests - no_output_timeout: 2h + name: Prepare working directory environment: WORKDIR: /home/circleci/work command: | mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh + - run: + name: Run SPM DARTEL Level 1 pipeline (py36) + no_output_timeout: 1h + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh + - run: + name: Run SPM DARTEL Level 2 pipeline (py36) + no_output_timeout: 30m + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh - store_artifacts: path: /home/circleci/work/tests @@ -382,15 +417,24 @@ jobs: export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - run: - name: Run tests - no_output_timeout: 2h + name: Prepare working directory environment: WORKDIR: /home/circleci/work command: | mkdir -p "$WORKDIR" chmod -R 777 "$WORKDIR" - bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh - bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh + - run: + name: Run SPM Nested Level 1 pipeline (py36) + no_output_timeout: 1h + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh + - run: + name: Run SPM Nested Level 2 pipeline (py27) + no_output_timeout: 30m + environment: + WORKDIR: /home/circleci/work + command: bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - store_artifacts: path: /home/circleci/work/tests - run: diff --git a/.circleci/pytests.sh b/.circleci/pytests.sh deleted file mode 100644 index 7d8bb27553..0000000000 --- a/.circleci/pytests.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - -# These tests are manually balanced based on previous build timings. -# They may need to be rebalanced in the future. -docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \ -&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \ -&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py2_pytest.sh b/.circleci/test_py2_pytest.sh new file mode 100644 index 0000000000..b89f44b046 --- /dev/null +++ b/.circleci/test_py2_pytest.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh +exitcode=$? + +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_docs.sh b/.circleci/test_py3_docs.sh new file mode 100644 index 0000000000..951a096f22 --- /dev/null +++ b/.circleci/test_py3_docs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh +exitcode=$? + +. $DIR/test_complete.sh diff --git a/.circleci/test_py3_pytest.sh b/.circleci/test_py3_pytest.sh new file mode 100644 index 0000000000..dcb91209a3 --- /dev/null +++ b/.circleci/test_py3_pytest.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. $DIR/test_init.sh + +docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh +exitcode=$? + +. $DIR/test_complete.sh From d0848ad0c1d629cd4e964c852c719cd0f33d7c1b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 25 Jan 2018 11:03:55 -0500 Subject: [PATCH 16/28] Remove old test script --- .circleci/tests.sh | 54 ---------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 .circleci/tests.sh diff --git a/.circleci/tests.sh b/.circleci/tests.sh deleted file mode 100644 index f55a3249d7..0000000000 --- a/.circleci/tests.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -# Balance nipype testing workflows across CircleCI build nodes -# - -# Setting # $ help set -set -e # Exit immediately if a command exits with a non-zero status. -set -u # Treat unset variables as an error when substituting. -set -x # Print command traces before executing command. - -if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then - echo "These tests were designed to be run at 4x parallelism." - exit 1 -fi - -DOCKER_IMAGE="nipype/nipype" - -# These tests are manually balanced based on previous build timings. -# They may need to be rebalanced in the future. -case ${CIRCLE_NODE_INDEX} in - 0) - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d - exitcode=$? - ;; - 1) - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline - exitcode=$? - ;; - 2) - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline - exitcode=$? - ;; - 3) - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline \ - && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow - exitcode=$? - ;; -esac - -# Exit with error if any of the tests failed -if [ "$exitcode" != "0" ]; then exit 1; fi - -codecov --file "${WORKDIR}/tests/coverage*.xml" \ - --root "${HOME}/nipype/" --flags unittests -e CIRCLE_NODE_INDEX - -codecov --file "${WORKDIR}/tests/smoketest*.xml" \ - --root "${HOME}/nipype/" --flags smoketests -e CIRCLE_NODE_INDEX From 3cf0d768cc41cfaf349dacc2b8a013d94e2f0302 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 25 Jan 2018 11:39:29 -0500 Subject: [PATCH 17/28] CI: Re-add WORKDIR variable, timeouts --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8de3f66ca..0f0113786b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -128,12 +128,21 @@ jobs: chmod -R 777 "$WORKDIR" - run: name: Run pytests (py36) + no_output_timeout: 30m + environment: + WORKDIR: /home/circleci/work command: bash /home/circleci/nipype/.circleci/test_py3_pytest.sh - run: name: Run pytests (py27) + no_output_timeout: 30m + environment: + WORKDIR: /home/circleci/work command: bash /home/circleci/nipype/.circleci/test_py2_pytest.sh - run: name: Build docs (py36) + no_output_timeout: 30m + environment: + WORKDIR: /home/circleci/work command: bash /home/circleci/nipype/.circleci/test_py3_docs.sh - store_artifacts: path: /home/circleci/work/tests From 98fd2c17f92fba0f808e02fe17f468bad87b7804 Mon Sep 17 00:00:00 2001 From: Jakub Kaczmarzyk Date: Sat, 10 Feb 2018 13:36:15 -0500 Subject: [PATCH 18/28] enh: use yaml pointers to reduce duplication --- .circleci/config.yml | 483 +++++++++++++------------------------------ 1 file changed, 147 insertions(+), 336 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f0113786b..ee5683f1bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,101 @@ +_machine_kwds: &machine_kwds + image: circleci/classic:201710-02 + +_store_artifacts_kwds: &store_artifacts_kwds + path: /home/circleci/work/tests + +_test_environment: &test_environment + WORKDIR: /home/circleci/work + +_generate_dockerfiles: &generate_dockerfiles + name: Generate Dockerfiles + command: | + make gen-dockerfiles + +_modify_nipype_version: &modify_nipype_version + name: Modify Nipype version if necessary + command: | + if [ "$CIRCLE_TAG" != "" ]; then + sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py + fi + +_get_base_image: &get_base_image + name: Get base image (pull or build) + no_output_timeout: 60m + command: | + source /tmp/docker/get_base_image.sh + if [ "$GET_BASE" == "PULL" ]; then + echo "Pulling base image ..." + docker pull nipype/nipype:base + elif [ "$GET_BASE" == "BUILD" ]; then + e=1 && for i in {1..5}; do + docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + else + echo "Error: method to get base image not understood" + exit 1 + fi + +_build_main_image_py36: &build_main_image_py36 + name: Build main image (py36) + no_output_timeout: 60m + command: | + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag nipype/nipype:latest \ + --tag nipype/nipype:py36 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + +_build_main_image_py27: &build_main_image_py27 + name: Build main image (py27) + no_output_timeout: 60m + command: | + e=1 && for i in {1..5}; do + docker build \ + --rm=false \ + --tag nipype/nipype:py27 \ + --build-arg PYTHON_VERSION_MAJOR=2 \ + --build-arg PYTHON_VERSION_MINOR=7 \ + --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ + --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + +_download_test_data: &_download_test_data + name: Download test data + no_output_timeout: 20m + working_directory: /home/circleci/examples + environment: + OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" + command: | + export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj + + export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz + + export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" + curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz + +_prepare_working_directory: &prepare_working_directory + name: Prepare working directory + environment: *test_environment + command: | + mkdir -p "$WORKDIR" + chmod -R 777 "$WORKDIR" + +_get_codecov: &_get_codecov + name: Get codecov + command: | + pip install --no-cache-dir codecov + + version: 2 jobs: @@ -36,416 +134,129 @@ jobs: - docker/get_base_image.sh test_pytest: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + machine: *machine_kwds working_directory: /home/circleci/nipype steps: - checkout: path: /home/circleci/nipype - attach_workspace: at: /tmp - - run: - name: Generate Dockerfiles - command: | - make gen-dockerfiles - - run: - name: Modify Nipype version if necessary - command: | - if [ "$CIRCLE_TAG" != "" ]; then - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py - fi - - run: - name: Get base image (pull or build) - no_output_timeout: 60m - command: | - source /tmp/docker/get_base_image.sh - if [ "$GET_BASE" == "PULL" ]; then - echo "Pulling base image ..." - docker pull nipype/nipype:base - elif [ "$GET_BASE" == "BUILD" ]; then - e=1 && for i in {1..5}; do - docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - else - echo "Error: method to get base image not understood" - exit 1 - fi - - run: - name: Build main image (py36) - no_output_timeout: 60m - command: | - e=1 && for i in {1..5}; do - docker build \ - --rm=false \ - --tag nipype/nipype:latest \ - --tag nipype/nipype:py36 \ - --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ - && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - - run: - name: Build main image (py27) - no_output_timeout: 60m - command: | - e=1 && for i in {1..5}; do - docker build \ - --rm=false \ - --tag nipype/nipype:py27 \ - --build-arg PYTHON_VERSION_MAJOR=2 \ - --build-arg PYTHON_VERSION_MINOR=7 \ - --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ - && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Prepare working directory - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" + - run: *generate_dockerfiles + - run: *modify_nipype_version + - run: *get_base_image + - run: *build_main_image_py36 + - run: *build_main_image_py27 + - run: *_get_codecov + - run: *_download_test_data + - run: *prepare_working_directory - run: name: Run pytests (py36) no_output_timeout: 30m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_pytest.sh - run: name: Run pytests (py27) no_output_timeout: 30m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py2_pytest.sh - run: name: Build docs (py36) no_output_timeout: 30m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_docs.sh - - store_artifacts: - path: /home/circleci/work/tests + - store_artifacts: *store_artifacts_kwds test_py3_fmri_fsl_spm: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + machine: *machine_kwds working_directory: /home/circleci/nipype steps: - checkout: path: /home/circleci/nipype - attach_workspace: at: /tmp - - run: - name: Generate Dockerfiles - command: | - make gen-dockerfiles - - run: - name: Modify Nipype version if necessary - command: | - if [ "$CIRCLE_TAG" != "" ]; then - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py - fi - - run: - name: Get base image (pull or build) - no_output_timeout: 60m - command: | - source /tmp/docker/get_base_image.sh - if [ "$GET_BASE" == "PULL" ]; then - echo "Pulling base image ..." - docker pull nipype/nipype:base - elif [ "$GET_BASE" == "BUILD" ]; then - e=1 && for i in {1..5}; do - docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - else - echo "Error: method to get base image not understood" - exit 1 - fi - - run: - name: Build main image (py36) - no_output_timeout: 60m - command: | - e=1 && for i in {1..5}; do - docker build \ - --rm=false \ - --tag nipype/nipype:latest \ - --tag nipype/nipype:py36 \ - --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ - && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Prepare working directory - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" + - run: *generate_dockerfiles + - run: *modify_nipype_version + - run: *get_base_image + - run: *build_main_image_py36 + - run: *_get_codecov + - run: *_download_test_data + - run: *prepare_working_directory - run: name: Run FSL FEEDS pipeline (py36) no_output_timeout: 40m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh - run: name: Run FSL reuse pipeline (py36) no_output_timeout: 40m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh - run: name: Run SPM test workflow - 3D inputs (py36) no_output_timeout: 40m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh - run: name: Run SPM test workflow - 4D inputs (py36) no_output_timeout: 40m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh - - store_artifacts: - path: /home/circleci/work/tests + - store_artifacts: *store_artifacts_kwds test_py3_fmri_spm_dartel_multiproc: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + machine: *machine_kwds working_directory: /home/circleci/nipype steps: - checkout: path: /home/circleci/nipype - attach_workspace: at: /tmp - - run: - name: Generate Dockerfiles - command: | - make gen-dockerfiles - - run: - name: Modify Nipype version if necessary - command: | - if [ "$CIRCLE_TAG" != "" ]; then - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py - fi - - run: - name: Get base image (pull or build) - no_output_timeout: 60m - command: | - source /tmp/docker/get_base_image.sh - if [ "$GET_BASE" == "PULL" ]; then - echo "Pulling base image ..." - docker pull nipype/nipype:base - elif [ "$GET_BASE" == "BUILD" ]; then - e=1 && for i in {1..5}; do - docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - else - echo "Error: method to get base image not understood" - exit 1 - fi - - run: - name: Build main image (py36) - no_output_timeout: 60m - command: | - e=1 && for i in {1..5}; do - docker build \ - --rm=false \ - --tag nipype/nipype:latest \ - --tag nipype/nipype:py36 \ - --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ - && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Prepare working directory - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" + - run: *generate_dockerfiles + - run: *modify_nipype_version + - run: *get_base_image + - run: *build_main_image_py36 + - run: *_get_codecov + - run: *_download_test_data + - run: *prepare_working_directory - run: name: Run SPM DARTEL Level 1 pipeline (py36) no_output_timeout: 1h - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh - run: name: Run SPM DARTEL Level 2 pipeline (py36) no_output_timeout: 30m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh - - store_artifacts: - path: /home/circleci/work/tests + - store_artifacts: *store_artifacts_kwds test_fmri_spm_nested_multiproc: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201710-02 + machine: *machine_kwds working_directory: /home/circleci/nipype steps: - checkout: path: /home/circleci/nipype - attach_workspace: at: /tmp - - run: - name: Generate Dockerfiles - command: | - make gen-dockerfiles - - run: - name: Modify Nipype version if necessary - command: | - if [ "$CIRCLE_TAG" != "" ]; then - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py - fi - - run: - name: Get base image (pull or build) - no_output_timeout: 60m - command: | - source /tmp/docker/get_base_image.sh - if [ "$GET_BASE" == "PULL" ]; then - echo "Pulling base image ..." - docker pull nipype/nipype:base - elif [ "$GET_BASE" == "BUILD" ]; then - e=1 && for i in {1..5}; do - docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - else - echo "Error: method to get base image not understood" - exit 1 - fi - - run: - name: Build main image (py36) - no_output_timeout: 60m - command: | - e=1 && for i in {1..5}; do - docker build \ - --rm=false \ - --tag nipype/nipype:latest \ - --tag nipype/nipype:py36 \ - --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \ - && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - - run: - name: Build main image (py27) - no_output_timeout: 60m - command: | - e=1 && for i in {1..5}; do - docker build \ - --rm=false \ - --tag nipype/nipype:py27 \ - --build-arg PYTHON_VERSION_MAJOR=2 \ - --build-arg PYTHON_VERSION_MINOR=7 \ - --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \ - && e=0 && break || sleep 15 - done && [ "$e" -eq "0" ] - - run: - name: Get codecov - command: | - pip install --no-cache-dir codecov - - run: - name: Download test data - no_output_timeout: 20m - working_directory: /home/circleci/examples - environment: - OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage" - command: | - export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj - - export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz - - export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af" - curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz - - run: - name: Prepare working directory - environment: - WORKDIR: /home/circleci/work - command: | - mkdir -p "$WORKDIR" - chmod -R 777 "$WORKDIR" + - run: *generate_dockerfiles + - run: *modify_nipype_version + - run: *get_base_image + - run: *build_main_image_py36 + - run: *build_main_image_py27 + - run: *_get_codecov + - run: *_download_test_data + - run: *prepare_working_directory - run: name: Run SPM Nested Level 1 pipeline (py36) no_output_timeout: 1h - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh - run: name: Run SPM Nested Level 2 pipeline (py27) no_output_timeout: 30m - environment: - WORKDIR: /home/circleci/work + environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - - store_artifacts: - path: /home/circleci/work/tests + - store_artifacts: *store_artifacts_kwds - run: name: Save Docker images to workspace no_output_timeout: 60m From c1487e7e6b9d8142e49503d4ea2200daab606eae Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 13 Feb 2018 21:16:35 -0500 Subject: [PATCH 19/28] TEST: Set WORKDIR if WORKDIR unset --- docker/files/run_examples.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/files/run_examples.sh b/docker/files/run_examples.sh index 7959bdb597..1914f53be9 100644 --- a/docker/files/run_examples.sh +++ b/docker/files/run_examples.sh @@ -3,7 +3,7 @@ set -e set -x set -u -WORKDIR=${WORK:-/work} +WORKDIR=${WORKDIR:-/work} arr=$@ tmp_var=$( IFS=$' '; echo "${arr[*]}" ) example_id=${tmp_var//[^A-Za-z0-9_-]/_} From d2048482793dd93120c6211af70b293d38007d94 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 13 Feb 2018 21:24:51 -0500 Subject: [PATCH 20/28] CI: Move codecov into CI step --- .circleci/config.yml | 10 ++++++++++ .circleci/test_complete.sh | 12 ------------ .circleci/test_py2_fmri_spm_nested_multiproc_l2.sh | 3 --- .circleci/test_py2_pytest.sh | 3 --- .circleci/test_py3_docs.sh | 3 --- .circleci/test_py3_fmri_fsl_feeds_linear_l1.sh | 3 --- .circleci/test_py3_fmri_fsl_reuse_linear_l1.sh | 3 --- .circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh | 3 --- .circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh | 3 --- .circleci/test_py3_fmri_spm_linear_3d.sh | 3 --- .circleci/test_py3_fmri_spm_linear_4d.sh | 3 --- .circleci/test_py3_fmri_spm_nested_multiproc_l1.sh | 3 --- .circleci/test_py3_pytest.sh | 3 --- 13 files changed, 10 insertions(+), 45 deletions(-) delete mode 100644 .circleci/test_complete.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index ee5683f1bd..f31cf1ed3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,6 +95,12 @@ _get_codecov: &_get_codecov command: | pip install --no-cache-dir codecov +_run_codecov: &_run_codecov + name: Run codecov + command: | + codecov --file "$WORKDIR/tests/coverage*.xml" --root "$HOME/nipype" --flags unittests -e "$CIRCLE_JOB" + codecov --file "$WORKDIR/tests/smoketest*.xml" --root "$HOME/nipype" --flags smoketests -e "$CIRCLE_JOB" + version: 2 jobs: @@ -164,6 +170,7 @@ jobs: no_output_timeout: 30m environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_docs.sh + - run: *_run_codecov - store_artifacts: *store_artifacts_kwds test_py3_fmri_fsl_spm: @@ -201,6 +208,7 @@ jobs: no_output_timeout: 40m environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh + - run: *_run_codecov - store_artifacts: *store_artifacts_kwds test_py3_fmri_spm_dartel_multiproc: @@ -228,6 +236,7 @@ jobs: no_output_timeout: 30m environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh + - run: *_run_codecov - store_artifacts: *store_artifacts_kwds test_fmri_spm_nested_multiproc: @@ -256,6 +265,7 @@ jobs: no_output_timeout: 30m environment: *test_environment command: bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh + - run: *_run_codecov - store_artifacts: *store_artifacts_kwds - run: name: Save Docker images to workspace diff --git a/.circleci/test_complete.sh b/.circleci/test_complete.sh deleted file mode 100644 index 5c138a2d82..0000000000 --- a/.circleci/test_complete.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Source after running test and storing relevant return code in $exitcode - -# Exit with error if any of the tests failed -if [ "$exitcode" != "0" ]; then exit 1; fi - -codecov --file "${WORKDIR}/tests/coverage*.xml" \ - --root "${HOME}/nipype/" --flags unittests -e SCRIPT_NAME - -codecov --file "${WORKDIR}/tests/smoketest*.xml" \ - --root "${HOME}/nipype/" --flags smoketests -e SCRIPT_NAME diff --git a/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh index f575e40e0d..3def0151d1 100644 --- a/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh +++ b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py2_pytest.sh b/.circleci/test_py2_pytest.sh index b89f44b046..a00a14c659 100644 --- a/.circleci/test_py2_pytest.sh +++ b/.circleci/test_py2_pytest.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_docs.sh b/.circleci/test_py3_docs.sh index 951a096f22..f2c10a7c7b 100644 --- a/.circleci/test_py3_docs.sh +++ b/.circleci/test_py3_docs.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh index 435d08738a..6c7eeffc93 100644 --- a/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh +++ b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh index 9ac6d77d91..d00c863619 100644 --- a/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh +++ b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh b/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh index ac17336b4b..52153780c9 100644 --- a/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh +++ b/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ level1 -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh b/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh index ad4326eb47..f86f7aa6e3 100644 --- a/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh +++ b/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ l2pipeline -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_linear_3d.sh b/.circleci/test_py3_fmri_spm_linear_3d.sh index dd3542b542..58a5de6b02 100644 --- a/.circleci/test_py3_fmri_spm_linear_3d.sh +++ b/.circleci/test_py3_fmri_spm_linear_3d.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_linear_4d.sh b/.circleci/test_py3_fmri_spm_linear_4d.sh index 05a21fc786..5d18e977c0 100644 --- a/.circleci/test_py3_fmri_spm_linear_4d.sh +++ b/.circleci/test_py3_fmri_spm_linear_4d.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh index de1dcd77fb..7be9bd5b75 100644 --- a/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh +++ b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 -exitcode=$? - -. $DIR/test_complete.sh diff --git a/.circleci/test_py3_pytest.sh b/.circleci/test_py3_pytest.sh index dcb91209a3..bcae56ecf2 100644 --- a/.circleci/test_py3_pytest.sh +++ b/.circleci/test_py3_pytest.sh @@ -5,6 +5,3 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . $DIR/test_init.sh docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh -exitcode=$? - -. $DIR/test_complete.sh From e5be9ac6808d8d2e571bc4ec2cf0ef018578cdeb Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 13 Feb 2018 21:36:11 -0500 Subject: [PATCH 21/28] CI: Replace test_init with DOCKER_IMAGE, -ux --- .circleci/config.yml | 23 ++++++++++--------- .circleci/test_init.sh | 12 ---------- .../test_py2_fmri_spm_nested_multiproc_l2.sh | 4 ---- .circleci/test_py2_pytest.sh | 4 ---- .circleci/test_py3_docs.sh | 4 ---- .../test_py3_fmri_fsl_feeds_linear_l1.sh | 4 ---- .../test_py3_fmri_fsl_reuse_linear_l1.sh | 4 ---- .../test_py3_fmri_spm_dartel_multiproc_l1.sh | 4 ---- .../test_py3_fmri_spm_dartel_multiproc_l2.sh | 4 ---- .circleci/test_py3_fmri_spm_linear_3d.sh | 4 ---- .circleci/test_py3_fmri_spm_linear_4d.sh | 4 ---- .../test_py3_fmri_spm_nested_multiproc_l1.sh | 4 ---- .circleci/test_py3_pytest.sh | 4 ---- 13 files changed, 12 insertions(+), 67 deletions(-) delete mode 100644 .circleci/test_init.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index f31cf1ed3b..267d9f448d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,7 @@ _store_artifacts_kwds: &store_artifacts_kwds _test_environment: &test_environment WORKDIR: /home/circleci/work + DOCKER_IMAGE: "nipype/nipype" _generate_dockerfiles: &generate_dockerfiles name: Generate Dockerfiles @@ -159,17 +160,17 @@ jobs: name: Run pytests (py36) no_output_timeout: 30m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_pytest.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_pytest.sh - run: name: Run pytests (py27) no_output_timeout: 30m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py2_pytest.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py2_pytest.sh - run: name: Build docs (py36) no_output_timeout: 30m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_docs.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_docs.sh - run: *_run_codecov - store_artifacts: *store_artifacts_kwds @@ -192,22 +193,22 @@ jobs: name: Run FSL FEEDS pipeline (py36) no_output_timeout: 40m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh - run: name: Run FSL reuse pipeline (py36) no_output_timeout: 40m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh - run: name: Run SPM test workflow - 3D inputs (py36) no_output_timeout: 40m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh - run: name: Run SPM test workflow - 4D inputs (py36) no_output_timeout: 40m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh - run: *_run_codecov - store_artifacts: *store_artifacts_kwds @@ -230,12 +231,12 @@ jobs: name: Run SPM DARTEL Level 1 pipeline (py36) no_output_timeout: 1h environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh - run: name: Run SPM DARTEL Level 2 pipeline (py36) no_output_timeout: 30m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh - run: *_run_codecov - store_artifacts: *store_artifacts_kwds @@ -259,12 +260,12 @@ jobs: name: Run SPM Nested Level 1 pipeline (py36) no_output_timeout: 1h environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh - run: name: Run SPM Nested Level 2 pipeline (py27) no_output_timeout: 30m environment: *test_environment - command: bash /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh + command: bash -ux /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - run: *_run_codecov - store_artifacts: *store_artifacts_kwds - run: diff --git a/.circleci/test_init.sh b/.circleci/test_init.sh deleted file mode 100644 index 373d5efe7a..0000000000 --- a/.circleci/test_init.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Source before running tests - -# Setting # $ help set -set -e # Exit immediately if a command exits with a non-zero status. -set -u # Treat unset variables as an error when substituting. -set -x # Print command traces before executing command. - -export DOCKER_IMAGE="nipype/nipype" - -export SCRIPT_NAME=`basename $0` diff --git a/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh index 3def0151d1..1aa2a4f8c5 100644 --- a/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh +++ b/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline diff --git a/.circleci/test_py2_pytest.sh b/.circleci/test_py2_pytest.sh index a00a14c659..6ea0ddedfc 100644 --- a/.circleci/test_py2_pytest.sh +++ b/.circleci/test_py2_pytest.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh diff --git a/.circleci/test_py3_docs.sh b/.circleci/test_py3_docs.sh index f2c10a7c7b..a050caf66c 100644 --- a/.circleci/test_py3_docs.sh +++ b/.circleci/test_py3_docs.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh diff --git a/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh index 6c7eeffc93..9666829b74 100644 --- a/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh +++ b/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline diff --git a/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh index d00c863619..48be49d80d 100644 --- a/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh +++ b/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow diff --git a/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh b/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh index 52153780c9..4208eed506 100644 --- a/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh +++ b/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ level1 diff --git a/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh b/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh index f86f7aa6e3..86119e7654 100644 --- a/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh +++ b/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ l2pipeline diff --git a/.circleci/test_py3_fmri_spm_linear_3d.sh b/.circleci/test_py3_fmri_spm_linear_3d.sh index 58a5de6b02..27c2c92a1a 100644 --- a/.circleci/test_py3_fmri_spm_linear_3d.sh +++ b/.circleci/test_py3_fmri_spm_linear_3d.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d diff --git a/.circleci/test_py3_fmri_spm_linear_4d.sh b/.circleci/test_py3_fmri_spm_linear_4d.sh index 5d18e977c0..cd255d60ca 100644 --- a/.circleci/test_py3_fmri_spm_linear_4d.sh +++ b/.circleci/test_py3_fmri_spm_linear_4d.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d diff --git a/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh index 7be9bd5b75..a6d2133a42 100644 --- a/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh +++ b/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 diff --git a/.circleci/test_py3_pytest.sh b/.circleci/test_py3_pytest.sh index bcae56ecf2..545f430308 100644 --- a/.circleci/test_py3_pytest.sh +++ b/.circleci/test_py3_pytest.sh @@ -1,7 +1,3 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -. $DIR/test_init.sh - docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh From 60d822b84a764b8a25e2da8792cd0a99ed6cf8d0 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 13 Feb 2018 21:51:49 -0500 Subject: [PATCH 22/28] ENH: Move doc building, save artifacts --- .circleci/config.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 267d9f448d..baa1da9eef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,11 +166,6 @@ jobs: no_output_timeout: 30m environment: *test_environment command: bash -ux /home/circleci/nipype/.circleci/test_py2_pytest.sh - - run: - name: Build docs (py36) - no_output_timeout: 30m - environment: *test_environment - command: bash -ux /home/circleci/nipype/.circleci/test_py3_docs.sh - run: *_run_codecov - store_artifacts: *store_artifacts_kwds @@ -268,6 +263,13 @@ jobs: command: bash -ux /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - run: *_run_codecov - store_artifacts: *store_artifacts_kwds + - run: + name: Build docs (py36) + no_output_timeout: 30m + environment: *test_environment + command: bash -ux /home/circleci/nipype/.circleci/test_py3_docs.sh + - store_artifacts: + path: /home/circleci/work/docs - run: name: Save Docker images to workspace no_output_timeout: 60m From e32cbccb1c4c7747d2364aef947c5f8bcac1f05b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 13 Feb 2018 22:29:06 -0500 Subject: [PATCH 23/28] CI: Run codecov in WORKDIR --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index baa1da9eef..67ccee11d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,7 @@ _get_codecov: &_get_codecov _run_codecov: &_run_codecov name: Run codecov + environment: *test_environment command: | codecov --file "$WORKDIR/tests/coverage*.xml" --root "$HOME/nipype" --flags unittests -e "$CIRCLE_JOB" codecov --file "$WORKDIR/tests/smoketest*.xml" --root "$HOME/nipype" --flags smoketests -e "$CIRCLE_JOB" From a06530eaad35ace1e96a8ef5ad30f337280cf3c9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 14 Feb 2018 08:46:33 -0500 Subject: [PATCH 24/28] CI: Split coverage and smoke tests, un-quote asterisks --- .circleci/config.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 67ccee11d0..78443e6ff0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,12 +96,17 @@ _get_codecov: &_get_codecov command: | pip install --no-cache-dir codecov -_run_codecov: &_run_codecov - name: Run codecov +_run_codecov_coverage: &_run_codecov_coverage + name: Run codecov (coverage) environment: *test_environment command: | - codecov --file "$WORKDIR/tests/coverage*.xml" --root "$HOME/nipype" --flags unittests -e "$CIRCLE_JOB" - codecov --file "$WORKDIR/tests/smoketest*.xml" --root "$HOME/nipype" --flags smoketests -e "$CIRCLE_JOB" + codecov --file $WORKDIR/tests/coverage*.xml --root "$HOME/nipype" --flags unittests -e CIRCLE_JOB + +_run_codecov_smoke: &_run_codecov_smoke + name: Run codecov (smoke tests) + environment: *test_environment + command: | + codecov --file $WORKDIR/tests/smoketest*.xml --root "$HOME/nipype" --flags smoketests -e CIRCLE_JOB version: 2 @@ -167,7 +172,7 @@ jobs: no_output_timeout: 30m environment: *test_environment command: bash -ux /home/circleci/nipype/.circleci/test_py2_pytest.sh - - run: *_run_codecov + - run: *_run_codecov_coverage - store_artifacts: *store_artifacts_kwds test_py3_fmri_fsl_spm: @@ -205,7 +210,7 @@ jobs: no_output_timeout: 40m environment: *test_environment command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh - - run: *_run_codecov + - run: *_run_codecov_smoke - store_artifacts: *store_artifacts_kwds test_py3_fmri_spm_dartel_multiproc: @@ -233,7 +238,7 @@ jobs: no_output_timeout: 30m environment: *test_environment command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh - - run: *_run_codecov + - run: *_run_codecov_smoke - store_artifacts: *store_artifacts_kwds test_fmri_spm_nested_multiproc: @@ -262,7 +267,7 @@ jobs: no_output_timeout: 30m environment: *test_environment command: bash -ux /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh - - run: *_run_codecov + - run: *_run_codecov_smoke - store_artifacts: *store_artifacts_kwds - run: name: Build docs (py36) From a9c170a189a46a444725bbd1bd2f23cf701c03a3 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 14 Feb 2018 09:56:03 -0500 Subject: [PATCH 25/28] CI: Do not save docker images pointlessly --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78443e6ff0..b18f3998ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,18 +277,20 @@ jobs: - store_artifacts: path: /home/circleci/work/docs - run: - name: Save Docker images to workspace + name: Save Docker images to workspace if on master no_output_timeout: 60m command: | + if [ "$CIRCLE_BRANCH" = "master" -a -z "$CIRCLE_PULL_REQUEST" ]; then docker save nipype/nipype:base \ nipype/nipype:latest \ nipype/nipype:py27 \ nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz \ && du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz + fi - persist_to_workspace: root: /tmp paths: - - docker/* + - docker deploy: docker: From f28c47942a159f6e1d3c4f8eccdec60280d4978f Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 14 Feb 2018 13:06:50 -0500 Subject: [PATCH 26/28] CI: Store test results --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b18f3998ed..e9bcc0657e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -174,6 +174,7 @@ jobs: command: bash -ux /home/circleci/nipype/.circleci/test_py2_pytest.sh - run: *_run_codecov_coverage - store_artifacts: *store_artifacts_kwds + - store_test_results: *store_artifacts_kwds test_py3_fmri_fsl_spm: machine: *machine_kwds From 8982b27ddde2b3c6759182222bb7d2c61d680b82 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 14 Feb 2018 14:52:06 -0500 Subject: [PATCH 27/28] CI: Update environment in attempt to bump codecov --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9bcc0657e..0f2077a3ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,7 @@ _store_artifacts_kwds: &store_artifacts_kwds _test_environment: &test_environment WORKDIR: /home/circleci/work DOCKER_IMAGE: "nipype/nipype" + CIRCLE_PR_NUMBER: "${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}" _generate_dockerfiles: &generate_dockerfiles name: Generate Dockerfiles From d424f919d0eeeced4ce28c2a5eb16b05fad0e5bb Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 14 Feb 2018 16:31:45 -0500 Subject: [PATCH 28/28] CI: Attempt #2 setting PR number --- .circleci/config.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f2077a3ab..569a18fc20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,13 @@ _store_artifacts_kwds: &store_artifacts_kwds _test_environment: &test_environment WORKDIR: /home/circleci/work DOCKER_IMAGE: "nipype/nipype" - CIRCLE_PR_NUMBER: "${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}" + +_set_pr_number: &set_pr_number + name: Set PR number + command: | + echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV + source $BASH_ENV + echo $CIRCLE_PR_NUMBER _generate_dockerfiles: &generate_dockerfiles name: Generate Dockerfiles @@ -155,6 +161,7 @@ jobs: path: /home/circleci/nipype - attach_workspace: at: /tmp + - run: *set_pr_number - run: *generate_dockerfiles - run: *modify_nipype_version - run: *get_base_image @@ -185,6 +192,7 @@ jobs: path: /home/circleci/nipype - attach_workspace: at: /tmp + - run: *set_pr_number - run: *generate_dockerfiles - run: *modify_nipype_version - run: *get_base_image @@ -223,6 +231,7 @@ jobs: path: /home/circleci/nipype - attach_workspace: at: /tmp + - run: *set_pr_number - run: *generate_dockerfiles - run: *modify_nipype_version - run: *get_base_image @@ -251,6 +260,7 @@ jobs: path: /home/circleci/nipype - attach_workspace: at: /tmp + - run: *set_pr_number - run: *generate_dockerfiles - run: *modify_nipype_version - run: *get_base_image