Skip to content

Commit 8be53b7

Browse files
effigieseffigies
committed
TEST: Split Circle workflow
1 parent 04d8e0e commit 8be53b7

7 files changed

+192
-27
lines changed

‎.circleci/config.yml

Lines changed: 132 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,15 @@ jobs:
3535
- docker/Dockerfile.base-pruned
3636
- docker/get_base_image.sh
3737

38-
39-
build_and_test:
40-
parallelism: 4
38+
fetch_base:
4139
machine:
4240
# Ubuntu 14.04 with Docker 17.10.0-ce
4341
image: circleci/classic:201710-02
4442
working_directory: /home/circleci/nipype
4543
steps:
4644
- checkout:
4745
path: /home/circleci/nipype
46+
- setup_remote_docker
4847
- attach_workspace:
4948
at: /tmp
5049
- run:
@@ -75,34 +74,141 @@ jobs:
7574
exit 1
7675
fi
7776
- run:
78-
name: Build main image (py36)
77+
name: Save Docker images to workspace
78+
no_output_timeout: 60m
79+
command: |
80+
docker save nipype/nipype:base | gzip -1 > /tmp/docker/nipype-base.tar.gz \
81+
&& du -h /tmp/docker/nipype-base.tar.gz
82+
- persist_to_workspace:
83+
root: /tmp
84+
paths:
85+
- docker/*
86+
87+
build_py2:
88+
machine:
89+
# Ubuntu 14.04 with Docker 17.10.0-ce
90+
image: circleci/classic:201710-02
91+
working_directory: /home/circleci/nipype
92+
steps:
93+
- checkout:
94+
path: /home/circleci/nipype
95+
- setup_remote_docker
96+
- attach_workspace:
97+
at: /tmp
98+
- run:
99+
name: Load saved Docker images.
100+
no_output_timeout: 60m
101+
command: |
102+
docker load < /tmp/docker/nipype-base.tar.gz
103+
- run:
104+
name: Get test dependencies and generate Dockerfiles
105+
command: |
106+
pip install --no-cache-dir codecov
107+
make gen-dockerfiles
108+
- run:
109+
name: Modify Nipype version if necessary
110+
command: |
111+
if [ "$CIRCLE_TAG" != "" ]; then
112+
sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
113+
fi
114+
- run:
115+
name: Build main image (py27)
79116
no_output_timeout: 60m
80117
command: |
81118
e=1 && for i in {1..5}; do
82119
docker build \
83120
--rm=false \
84-
--tag nipype/nipype:latest \
85-
--tag nipype/nipype:py36 \
121+
--tag nipype/nipype:py27 \
122+
--build-arg PYTHON_VERSION_MAJOR=2 \
123+
--build-arg PYTHON_VERSION_MINOR=7 \
86124
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
87125
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
88-
--build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
126+
--build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
89127
&& e=0 && break || sleep 15
90128
done && [ "$e" -eq "0" ]
91129
- run:
92-
name: Build main image (py27)
130+
name: Save Docker images to workspace
131+
no_output_timeout: 60m
132+
command: |
133+
docker save nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-py27.tar.gz \
134+
&& du -h /tmp/docker/nipype-py27.tar.gz
135+
- persist_to_workspace:
136+
root: /tmp
137+
paths:
138+
- docker/*
139+
140+
141+
build_py3:
142+
machine:
143+
# Ubuntu 14.04 with Docker 17.10.0-ce
144+
image: circleci/classic:201710-02
145+
working_directory: /home/circleci/nipype
146+
steps:
147+
- checkout:
148+
path: /home/circleci/nipype
149+
- setup_remote_docker
150+
- attach_workspace:
151+
at: /tmp
152+
- run:
153+
name: Load saved Docker images.
154+
no_output_timeout: 60m
155+
command: |
156+
docker load < /tmp/docker/nipype-base.tar.gz
157+
- run:
158+
name: Get test dependencies and generate Dockerfiles
159+
command: |
160+
pip install --no-cache-dir codecov
161+
make gen-dockerfiles
162+
- run:
163+
name: Modify Nipype version if necessary
164+
command: |
165+
if [ "$CIRCLE_TAG" != "" ]; then
166+
sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
167+
fi
168+
- run:
169+
name: Build main image (py36)
93170
no_output_timeout: 60m
94171
command: |
95172
e=1 && for i in {1..5}; do
96173
docker build \
97174
--rm=false \
98-
--tag nipype/nipype:py27 \
99-
--build-arg PYTHON_VERSION_MAJOR=2 \
100-
--build-arg PYTHON_VERSION_MINOR=7 \
175+
--tag nipype/nipype:latest \
176+
--tag nipype/nipype:py36 \
101177
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
102178
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
103-
--build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
179+
--build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
104180
&& e=0 && break || sleep 15
105181
done && [ "$e" -eq "0" ]
182+
- run:
183+
name: Save Docker images to workspace
184+
no_output_timeout: 60m
185+
command: |
186+
docker save nipype/nipype:latest \
187+
nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-latest-py36.tar.gz \
188+
&& du -h /tmp/docker/nipype-latest-py36.tar.gz
189+
- persist_to_workspace:
190+
root: /tmp
191+
paths:
192+
- docker/*
193+
194+
195+
build_and_test:
196+
machine:
197+
# Ubuntu 14.04 with Docker 17.10.0-ce
198+
image: circleci/classic:201710-02
199+
working_directory: /home/circleci/nipype
200+
steps:
201+
- attach_workspace:
202+
at: /tmp
203+
- run:
204+
name: Load saved Docker images.
205+
no_output_timeout: 60m
206+
command: |
207+
docker load < /tmp/docker/nipype-base.tar.gz \
208+
&& docker load < /tmp/docker/nipype-py27.tar.gz \
209+
&& docker load < /tmp/docker/nipype-latest-py36.tar.gz
210+
- checkout:
211+
path: /home/circleci/nipype
106212
- run:
107213
name: Download test data
108214
no_output_timeout: 20m
@@ -120,26 +226,15 @@ jobs:
120226
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
121227
- run:
122228
name: Run tests
123-
no_output_timeout: 4h
229+
no_output_timeout: 2h
124230
environment:
125231
WORKDIR: /home/circleci/work
126232
command: |
127233
mkdir -p "$WORKDIR"
128234
chmod -R 777 "$WORKDIR"
129-
bash /home/circleci/nipype/.circleci/tests.sh
235+
bash /home/circleci/nipype/.circleci/pytests.sh
130236
- store_artifacts:
131237
path: /home/circleci/work/tests
132-
- run:
133-
name: Save Docker images to workspace
134-
no_output_timeout: 60m
135-
command: |
136-
if [ "$CIRCLE_NODE_INDEX" -eq "0" ] && [ "$CIRCLE_BRANCH" == "master" ]; then
137-
docker save nipype/nipype:base \
138-
nipype/nipype:latest \
139-
nipype/nipype:py36 \
140-
nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz
141-
du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz
142-
fi
143238
- persist_to_workspace:
144239
root: /tmp
145240
paths:
@@ -180,12 +275,22 @@ jobs:
180275

181276
workflows:
182277
version: 2
183-
build_test_deply:
278+
build_test_deploy:
184279
jobs:
185280
- compare_base_dockerfiles
186-
- build_and_test:
281+
- fetch_base:
187282
requires:
188283
- compare_base_dockerfiles
284+
- build_py2:
285+
requires:
286+
- fetch_base
287+
- build_py3:
288+
requires:
289+
- fetch_base
290+
- build_and_test:
291+
requires:
292+
- build_py2
293+
- build_py3
189294
- deploy:
190295
filters:
191296
branches:

‎.circleci/pytests.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
. test_init.sh
4+
5+
# These tests are manually balanced based on previous build timings.
6+
# They may need to be rebalanced in the future.
7+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \
8+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \
9+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh
10+
exitcode=$?
11+
12+
. test_complete.sh

‎.circleci/test_complete.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
#
3+
# Source after running test and storing relevant return code in $exitcode
4+
5+
# Exit with error if any of the tests failed
6+
if [ "$exitcode" != "0" ]; then exit 1; fi
7+
8+
codecov --file "${WORKDIR}/tests/coverage*.xml" \
9+
--root "${HOME}/nipype/" --flags unittests -e SCRIPT_NAME
10+
11+
codecov --file "${WORKDIR}/tests/smoketest*.xml" \
12+
--root "${HOME}/nipype/" --flags smoketests -e SCRIPT_NAME

‎.circleci/test_init.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
#
3+
# Source before running tests
4+
5+
# Setting # $ help set
6+
set -e # Exit immediately if a command exits with a non-zero status.
7+
set -u # Treat unset variables as an error when substituting.
8+
set -x # Print command traces before executing command.
9+
10+
export DOCKER_IMAGE="nipype/nipype"
11+
12+
export SCRIPT_NAME=`basename $0`

‎.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
. test_init.sh
4+
5+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
6+
exitcode=$?
7+
8+
. test_complete.sh

‎.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
. test_init.sh
4+
5+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline
6+
exitcode=$?
7+
8+
. test_complete.sh

‎.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
. test_init.sh
4+
5+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
6+
exitcode=$?
7+
8+
. test_complete.sh

0 commit comments

Comments
 (0)