From 92d8875d5bb5bfd725bda413008fa66b2f4c29ba Mon Sep 17 00:00:00 2001 From: oesteban Date: Mon, 15 May 2017 10:38:35 -0700 Subject: [PATCH 1/4] [ENH] Revise CircleCI settings Try to build nipype/base in circle build, and cache it properly. --- circle.yml | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/circle.yml b/circle.yml index 32d3ed16f6..b4fb288357 100644 --- a/circle.yml +++ b/circle.yml @@ -27,22 +27,35 @@ dependencies: - mkdir -p $WORKDIR && sudo setfacl -d -m group:ubuntu:rwx $WORKDIR && sudo setfacl -m group:ubuntu:rwx $WORKDIR - mkdir -p $HOME/docker $HOME/examples $WORKDIR/tests $WORKDIR/logs $WORKDIR/crashfiles ${CIRCLE_TEST_REPORTS}/tests/ - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi - override: - - if [[ -e "$HOME/docker/cache.tar" ]]; then docker load --input $HOME/docker/cache.tar; fi : + - docker load --input $HOME/docker/cache.tar || true : timeout: 6000 - - docker images - - docker pull nipype/base:latest + override: + # Get data - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi - if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py; fi -# - e=1 && for i in {1..5}; do docker build -f docker/base.Dockerfile --rm=false -t nipype/base:latest . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : -# timeout: 21600 - - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : + # Docker + - docker images + - ? | + e=1 && for i in {1..5}; do + docker build --rm=false -f docker/base.Dockerfile -t nipype/base:latest . && e=0 && break || sleep 15; + done && [ "$e" -eq "0" ] + : + timeout: 21600 + - ? | + e=1 && for i in {1..5}; do + docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; + done && [ "$e" -eq "0" ] + : timeout: 6000 - - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG-py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] : + - ? | + e=1 && for i in {1..5}; do + docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG-py27 . && e=0 && break || sleep 15; + done && [ "$e" -eq "0" ] + : timeout: 6000 - - docker save -o $HOME/docker/cache.tar nipype/base:latest nipype/nipype:py36 nipype/nipype:py27 : + - docker save -o $HOME/docker/cache.tar ubuntu:xenial-20161213 nipype/base:latest nipype/nipype:py36 nipype/nipype:py27 : timeout: 6000 test: From 50fa2357c6612b608995c51d7e5351e0c91f6a0e Mon Sep 17 00:00:00 2001 From: oesteban Date: Mon, 15 May 2017 10:57:23 -0700 Subject: [PATCH 2/4] [Docker] Install minimized version of FS6 - Replaces the full installation of FS by a minimized version by @satra - Depends on #2022 --- docker/base.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 8f9afde859..817fb12652 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -44,7 +44,7 @@ RUN apt-get update && \ WORKDIR /opt # Installing freesurfer -- do it first so that it is cached early -RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \ +RUN curl -sSL https://www.dropbox.com/s/0ldeovrmh26vabq/recon-all-freesurfer6.min.tgz?dl=0 | tar zxv -C /opt \ --exclude='freesurfer/trctrain' \ --exclude='freesurfer/subjects/fsaverage_sym' \ --exclude='freesurfer/subjects/fsaverage3' \ From 712bd4534b2e1501c32cba48267b8b378148ed8b Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 16 May 2017 10:14:08 -0400 Subject: [PATCH 3/4] updated freesurfer URL and added ANTS as well --- docker/base.Dockerfile | 50 +++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 817fb12652..c1388d6558 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -44,39 +44,25 @@ RUN apt-get update && \ WORKDIR /opt # Installing freesurfer -- do it first so that it is cached early -RUN curl -sSL https://www.dropbox.com/s/0ldeovrmh26vabq/recon-all-freesurfer6.min.tgz?dl=0 | tar zxv -C /opt \ - --exclude='freesurfer/trctrain' \ - --exclude='freesurfer/subjects/fsaverage_sym' \ - --exclude='freesurfer/subjects/fsaverage3' \ - --exclude='freesurfer/subjects/fsaverage4' \ - --exclude='freesurfer/subjects/fsaverage5' \ - --exclude='freesurfer/subjects/fsaverage6' \ - --exclude='freesurfer/subjects/cvs_avg35' \ - --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \ - --exclude='freesurfer/subjects/bert' \ - --exclude='freesurfer/subjects/V1_average' \ - --exclude='freesurfer/average/mult-comp-cor' \ - --exclude='freesurfer/lib/cuda' \ - --exclude='freesurfer/lib/qt' - -ENV FSL_DIR=/usr/share/fsl/5.0 \ +#----------------------------------------------------------------------------- +# 3. Install FreeSurfer v6.0 (minimized with reprozip): +# https://github.com/freesurfer/freesurfer/issues/70 +#----------------------------------------------------------------------------- +RUN curl -sSL https://dl.dropbox.com/s/y3hvbdgfu4pfad8/recon-all-freesurfer6-1.min.tgz?dl=0 | tar zx -C /opt +ENV FS_OVERRIDE=0 \ OS=Linux \ - FS_OVERRIDE=0 \ - FIX_VERTEX_AREA= \ FSF_OUTPUT_FORMAT=nii.gz \ + FIX_VERTEX_AREA=\ FREESURFER_HOME=/opt/freesurfer -ENV SUBJECTS_DIR=$FREESURFER_HOME/subjects \ - FUNCTIONALS_DIR=$FREESURFER_HOME/sessions \ - MNI_DIR=$FREESURFER_HOME/mni \ - LOCAL_DIR=$FREESURFER_HOME/local \ - FSFAST_HOME=$FREESURFER_HOME/fsfast \ - MINC_BIN_DIR=$FREESURFER_HOME/mni/bin \ - MINC_LIB_DIR=$FREESURFER_HOME/mni/lib \ - MNI_DATAPATH=$FREESURFER_HOME/mni/data \ - FMRI_ANALYSIS_DIR=$FREESURFER_HOME/fsfast -ENV PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \ - MNI_PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \ - PATH=$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH +ENV MNI_DIR=$FREESURFER_HOME/mni \ + SUBJECTS_DIR=$FREESURFER_HOME/subjects +ENV PERL5LIB=$MNI_DIR/share/perl5 \ + MNI_PERL5LIB=$MNI_DIR/share/perl5 \ + MINC_BIN_DIR=$MNI_DIR/bin \ + MINC_LIB_DIR=$MNI_DIR/lib \ + MNI_DATAPATH=$MNI_DIR/data +ENV PATH=$FREESURFER_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH +ENV FSL_DIR=/usr/share/fsl/5.0 RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh # Enable neurodebian @@ -122,8 +108,8 @@ ENV FSLDIR=/usr/share/fsl/5.0 \ # Installing and setting up ANTs RUN mkdir -p /opt/ants && \ - curl -sSL "https://github.com/stnava/ANTs/releases/download/v2.1.0/Linux_Ubuntu14.04.tar.bz2" \ - | tar -xjC /opt/ants --strip-components 1 + curl -sSL "https://dl.dropbox.com/s/2f4sui1z6lcgyek/ANTs-Linux-centos5_x86_64-v2.2.0-0740f91.tar.gz?dl=0" \ + | tar -zx -C /opt ENV ANTSPATH=/opt/ants \ PATH=$ANTSPATH:$PATH From d32be0b9de86430f715667c98f49495a5e65b69e Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 16 May 2017 14:44:17 -0400 Subject: [PATCH 4/4] fix: use new link with fsaverage files for mris_expand --- docker/base.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index c1388d6558..1c4b1c490f 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -48,7 +48,7 @@ WORKDIR /opt # 3. Install FreeSurfer v6.0 (minimized with reprozip): # https://github.com/freesurfer/freesurfer/issues/70 #----------------------------------------------------------------------------- -RUN curl -sSL https://dl.dropbox.com/s/y3hvbdgfu4pfad8/recon-all-freesurfer6-1.min.tgz?dl=0 | tar zx -C /opt +RUN curl -sSL https://dl.dropbox.com/s/pbaisn6m5qpi9uu/recon-all-freesurfer6-2.min.tgz?dl=0 | tar zx -C /opt ENV FS_OVERRIDE=0 \ OS=Linux \ FSF_OUTPUT_FORMAT=nii.gz \