From 1a7fd36fd7b9198e9d7d7ad7510d22bcc32a59a7 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 16 May 2017 16:05:01 -0700 Subject: [PATCH 1/3] play around caching: investigate if the docker cache is too big --- circle.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index b4fb288357..46a6ef25cd 100644 --- a/circle.yml +++ b/circle.yml @@ -27,8 +27,11 @@ dependencies: - mkdir -p $WORKDIR && sudo setfacl -d -m group:ubuntu:rwx $WORKDIR && sudo setfacl -m group:ubuntu:rwx $WORKDIR - mkdir -p $HOME/docker $HOME/examples $WORKDIR/tests $WORKDIR/logs $WORKDIR/crashfiles ${CIRCLE_TEST_REPORTS}/tests/ - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi - - docker load --input $HOME/docker/cache.tar || true : - timeout: 6000 + - docker load --input $HOME/docker/ubuntu.tar || true + - docker load --input $HOME/docker/nipype-base.tar || true + - docker load --input $HOME/docker/nipype-3.tar || true + - docker load --input $HOME/docker/nipype-2.tar || true + - docker images override: # Get data - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi @@ -36,26 +39,31 @@ dependencies: - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi - if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py; fi # Docker - - docker images + - docker pull ubuntu:xenial-20161213 + - docker save -o $HOME/docker/ubuntu.tar ubuntu:xenial-20161213 - ? | e=1 && for i in {1..5}; do docker build --rm=false -f docker/base.Dockerfile -t nipype/base:latest . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 21600 + - docker save -o $HOME/docker/nipype-base.tar nipype/base:latest : + timeout: 6000 - ? | e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 6000 + - docker save -o $HOME/docker/nipype-3.tar nipype/nipype:latest nipype/nipype:py36: + timeout: 6000 - ? | e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG-py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 6000 - - docker save -o $HOME/docker/cache.tar ubuntu:xenial-20161213 nipype/base:latest nipype/nipype:py36 nipype/nipype:py27 : + - docker save -o $HOME/docker/nipype-2.tar nipype/nipype:py27 : timeout: 6000 test: From 21c984f3f7e2cc0a101c299ee679fba8f48c6970 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 16 May 2017 23:14:18 -0700 Subject: [PATCH 2/3] Update circle.yml --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 46a6ef25cd..2c018f1e54 100644 --- a/circle.yml +++ b/circle.yml @@ -51,11 +51,11 @@ dependencies: timeout: 6000 - ? | e=1 && for i in {1..5}; do - docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; + docker build --rm=false -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 6000 - - docker save -o $HOME/docker/nipype-3.tar nipype/nipype:latest nipype/nipype:py36: + - docker save -o $HOME/docker/nipype-3.tar nipype/nipype:py36: timeout: 6000 - ? | e=1 && for i in {1..5}; do From 65c71f640e0a64ec032aeb6f53f9020aea68dad8 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 18 May 2017 12:05:14 -0700 Subject: [PATCH 3/3] reunite the cached images --- circle.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/circle.yml b/circle.yml index 2c018f1e54..f7641869fc 100644 --- a/circle.yml +++ b/circle.yml @@ -27,10 +27,7 @@ dependencies: - mkdir -p $WORKDIR && sudo setfacl -d -m group:ubuntu:rwx $WORKDIR && sudo setfacl -m group:ubuntu:rwx $WORKDIR - mkdir -p $HOME/docker $HOME/examples $WORKDIR/tests $WORKDIR/logs $WORKDIR/crashfiles ${CIRCLE_TEST_REPORTS}/tests/ - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi - - docker load --input $HOME/docker/ubuntu.tar || true - - docker load --input $HOME/docker/nipype-base.tar || true - - docker load --input $HOME/docker/nipype-3.tar || true - - docker load --input $HOME/docker/nipype-2.tar || true + - docker load --input $HOME/docker/nipype.tar || true - docker images override: # Get data @@ -39,31 +36,25 @@ dependencies: - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi - if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py; fi # Docker - - docker pull ubuntu:xenial-20161213 - - docker save -o $HOME/docker/ubuntu.tar ubuntu:xenial-20161213 - ? | e=1 && for i in {1..5}; do docker build --rm=false -f docker/base.Dockerfile -t nipype/base:latest . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 21600 - - docker save -o $HOME/docker/nipype-base.tar nipype/base:latest : - timeout: 6000 - ? | e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 6000 - - docker save -o $HOME/docker/nipype-3.tar nipype/nipype:py36: - timeout: 6000 - ? | e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG-py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : timeout: 6000 - - docker save -o $HOME/docker/nipype-2.tar nipype/nipype:py27 : + - docker save -o $HOME/docker/nipype.tar ubuntu:xenial-20161213 nipype/base:latest nipype/nipype:py36 nipype/nipype:py27 : timeout: 6000 test: