[FIX] fix cluster threshold in randomise + change default prefix

.circleci/config.yml

@@ -0,0 +1,194 @@
+version: 2
+jobs:
+
+  compare_base_dockerfiles:
+    docker:
+      - image: docker:17.10.0-ce-git
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - setup_remote_docker
+      - run:
+          name: Generate and prune base Dockerfile in preparation for cache check
+          working_directory: /home/circleci/nipype/docker
+          command: |
+            mkdir -p /tmp/docker
+            ash ./generate_dockerfiles.sh -b
+
+            # Use the sha256 sum of the pruned Dockerfile as the cache key.
+            ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned
+      - restore_cache:
+          key: dockerfile-cache-v1-master-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
+      - run:
+          name: Determine how to get base image
+          command: |
+            if [ -f /tmp/docker/cache/Dockerfile.base-pruned ]; then
+              echo "Cache found. Will pull base image."
+              echo 'export GET_BASE=PULL' > /tmp/docker/get_base_image.sh
+            else
+              echo "Cache not found. Will build base image."
+              echo 'export GET_BASE=BUILD' > /tmp/docker/get_base_image.sh
+            fi
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - docker/Dockerfile.base-pruned
+            - docker/get_base_image.sh
+
+
+  build_and_test:
+    parallelism: 4
+    machine:
+      # Ubuntu 14.04 with Docker 17.10.0-ce
+      image: circleci/classic:201710-02
+    working_directory: /home/circleci/nipype
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - attach_workspace:
+          at: /tmp
+      - run:
+          name: Get test dependencies and generate Dockerfiles
+          command: |
+            pip install --no-cache-dir codecov
+            make gen-dockerfiles
+      - run:
+          name: Modify Nipype version if necessary
+          command: |
+            if [ "$CIRCLE_TAG" != "" ]; then
+              sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
+            fi
+      - run:
+          name: Get base image (pull or build)
+          no_output_timeout: 60m
+          command: |
+            source /tmp/docker/get_base_image.sh
+            if [ "$GET_BASE" == "PULL" ]; then
+              echo "Pulling base image ..."
+              docker pull nipype/nipype:base
+            elif [ "$GET_BASE" == "BUILD" ]; then
+              e=1 && for i in {1..5}; do
+                docker build -t nipype/nipype:base - < docker/Dockerfile.base  && e=0 && break || sleep 15
+              done && [ "$e" -eq "0" ]
+            else
+              echo "Error: method to get base image not understood"
+              exit 1
+            fi
+      - run:
+          name: Build main image (py36)
+          no_output_timeout: 60m
+          command: |
+            e=1 && for i in {1..5}; do
+              docker build \
+                --rm=false \
+                --tag nipype/nipype:latest \
+                --tag nipype/nipype:py36 \
+                --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+                --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
+                --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
+              && e=0 && break || sleep 15
+            done && [ "$e" -eq "0" ]
+      - run:
+          name: Build main image (py27)
+          no_output_timeout: 60m
+          command: |
+            e=1 && for i in {1..5}; do
+              docker build \
+                --rm=false \
+                --tag nipype/nipype:py27 \
+                --build-arg PYTHON_VERSION_MAJOR=2 \
+                --build-arg PYTHON_VERSION_MINOR=7 \
+                --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+                --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
+                --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
+              && e=0 && break || sleep 15
+            done && [ "$e" -eq "0" ]
+      - run:
+          name: Download test data
+          no_output_timeout: 20m
+          working_directory: /home/circleci/examples
+          environment:
+            OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
+          command: |
+            export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
+            curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj
+
+            export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
+            curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz
+
+            export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
+            curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
+      - run:
+          name: Run tests
+          no_output_timeout: 4h
+          environment:
+            WORKDIR: /home/circleci/work
+          command: |
+            mkdir -p "$WORKDIR"
+            chmod -R 777 "$WORKDIR"
+            bash /home/circleci/nipype/.circleci/tests.sh
+      - store_artifacts:
+          path: /home/circleci/work/tests
+      - run:
+          name: Save Docker images to workspace
+          no_output_timeout: 60m
+          command: |
+            if [ "$CIRCLE_NODE_INDEX" -eq "0" ] && [ "$CIRCLE_BRANCH" == "master" ]; then
+              docker save nipype/nipype:base \
+                          nipype/nipype:latest \
+                          nipype/nipype:py36 \
+                          nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz
+              du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz
+            fi
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - docker/*
+
+
+  deploy:
+    docker:
+      - image: docker:17.10.0-ce-git
+    steps:
+      - setup_remote_docker
+      - attach_workspace:
+          at: /tmp
+      - run:
+          name: Load saved Docker images.
+          no_output_timeout: 60m
+          command: |
+            docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz
+      - run:
+          name: Push to DockerHub
+          no_output_timeout: 120m
+          command: |
+            echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
+            docker push nipype/nipype:base
+            docker push nipype/nipype:latest
+            docker push nipype/nipype:py36
+            docker push nipype/nipype:py27
+      - run:
+          name: Move pruned Dockerfile to /tmp/docker/cache directory
+          command: |
+            mkdir -p /tmp/docker/cache/
+            mv /tmp/docker/Dockerfile.base-pruned /tmp/docker/cache/Dockerfile.base-pruned
+      - save_cache:
+          paths:
+            - /tmp/docker/cache/Dockerfile.base-pruned
+          key: dockerfile-cache-v1-{{ .Branch }}-{{ checksum "/tmp/docker/cache/Dockerfile.base-pruned" }}
+
+
+workflows:
+  version: 2
+  build_test_deply:
+    jobs:
+      - compare_base_dockerfiles
+      - build_and_test:
+          requires:
+            - compare_base_dockerfiles
+      - deploy:
+          filters:
+            branches:
+              only: master
+          requires:
+            - build_and_test

.circleci/tests.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Balance nipype testing workflows across CircleCI build nodes
+#
+
+# Setting      # $ help set
+set -e         # Exit immediately if a command exits with a non-zero status.
+set -u         # Treat unset variables as an error when substituting.
+set -x         # Print command traces before executing command.
+
+if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then
+  echo "These tests were designed to be run at 4x parallelism."
+  exit 1
+fi
+
+DOCKER_IMAGE="nipype/nipype"
+
+# These tests are manually balanced based on previous build timings.
+# They may need to be rebalanced in the future.
+case ${CIRCLE_NODE_INDEX} in
+  0)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data"  "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
+    exitcode=$?
+    ;;
+  1)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
+    exitcode=$?
+    ;;
+  2)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
+    exitcode=$?
+    ;;
+  3)
+       docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline \
+    && docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
+    exitcode=$?
+    ;;
+esac
+
+# Exit with error if any of the tests failed
+if [ "$exitcode" != "0" ]; then exit 1; fi
+
+codecov --file "${WORKDIR}/tests/coverage*.xml" \
+  --root "${HOME}/nipype/" --flags unittests -e CIRCLE_NODE_INDEX
+
+codecov --file "${WORKDIR}/tests/smoketest*.xml" \
+  --root "${HOME}/nipype/" --flags smoketests -e CIRCLE_NODE_INDEX

.dockerignore

@@ -34,4 +34,4 @@ Vagrantfile
 .mailmap
 
 # Previous coverage results
-.coverage
+.coverage

.mailmap

@@ -1,5 +1,6 @@
 Aimi Watanabe <watanabe.aimi@gmail.com> stymy <rschadmin@cmi-rsch-li002.childmind.org>
 Aimi Watanabe <watanabe.aimi@gmail.com> stymy <watanabe.aimi@gmail.com>
+Alejandro de la Vega <aleph4@gmail.com> adelavega <delavega@utexas.edu>
 Alexander Schaefer <alexschaefer83@gmail.com> Alexander Schaefer <aschaefer@melisse.cbs.mpg.de>
 Alexander Schaefer <alexschaefer83@gmail.com> alexschaefer83 <alexschaefer83@gmail.com>
 Alexander Schaefer <alexschaefer83@gmail.com> aschaefer <aschaefer@aschaefer-U36SD.(none)>
@@ -56,11 +57,13 @@ Franz Liem <franz.liem@gmail.com> fliem <franz.liem>
 Gael Varoquaux <gael.varoquaux@normalesup.org> GaelVaroquaux
 Gael Varoquaux <gael.varoquaux@normalesup.org> GaelVaroquaux <gael.varoquaux@noramlesup.org>
 Gavin Cooper <Gavin.Cooper@newcastle.edu.au> gjcooper <gjcooper@gmail.com>
+Gilles de Hollander <gilles.de.hollander@gmail.com> Gilles86 <gilles.de.hollander@gmail.com>
 Hans Johnson <hans-johnson@uiowa.edu> Hans Johnson <hans.j.johnson@gmail.com>
 Hans Johnson <hans-johnson@uiowa.edu> hjmjohnson <hans.j.johnson@gmail.com>
 Horea Christian <horea.christ@yandex.com> Horea Christian <h.chr@mail.ru>
 Isaac Schwabacher <ischwabacher@wisc.edu> ischwabacher <ischwabacher@wisc.edu>
 James Kent <james-kent@uiowa.edu> jdkent <james-kent@uiowa.edu>
+Jakub Kaczmarzyk <jakubk@mit.edu> kaczmarj <jakubk@mit.edu>
 Jason Wong <jf126dolphin@gmail.com> Jason <jf126dolphin@gmail.com>
 Jason Wong <jf126dolphin@gmail.com> jason <jf126dolphin@gmail.com>
 Jason Wong <jf126dolphin@gmail.com> Jason W <jf126dolphin@gmail.com>
@@ -72,7 +75,9 @@ Joerg Stadler <Joerg.Stadler@lin-magdeburg.de> Jörg Stadler <Stadler@lin-magdeb
 Joke Durnez <joke.durnez@gmail.com> jokedurnez <joke.durnez@gmail.com>
 Josh Warner <silvertrumpet999+github@gmail.com> Josh Warner (Mac) <warner.joshua@mayo.edu>
 Kai Schlamp <schlamp@gmx.de> medihack <schlamp@gmx.de>Jessica Forbes <jessica-forbes@uiowa.edu> jessicaforbes <jessica-forbes@uiowa.edu>
+Kevin Sitek <ksitek@mit.edu> sitek <ksitek@mit.edu>
 Leonie Lampe <leonie.lampe@gmail.com> Leonie Lmape <leonie.lampe@gmail.com>
+Lukas Snoek <lukassnoek@gmail.com> Lukas Snoek <lukassnoek@gmail.com>
 Mathias Goncalves <goncalves.mathias@gmail.com> mathiasg <mathiasg@mit.edu>
 Michael Dayan <mdayan.research@gmail.com> Michael <mdayan.research@gmail.com>
 Michael Dayan <mdayan.research@gmail.com> Michael <mdayan.comp@gmail.com>
@@ -83,6 +88,7 @@ Michael Waskom <michael.l.waskom@gmail.com> Michael Waskom <mwaskom@mit.edu>
 Michael Waskom <michael.l.waskom@gmail.com> Michael Waskom <mwaskom@stanford.edu>
 Michael Waskom <michael.l.waskom@gmail.com> mwaskom <mwaskom@mit.edu>
 Michael Waskom <michael.l.waskom@gmail.com> mwaskom <mwaskom@stanford.edu>
+Michael Waskom <michael.l.waskom@gmail.com> mwaskom <mwaskom@nyu.edu>
 Oscar Esteban <code@oscaresteban.es> Oscar Esteban <code@oscaresteban>
 Oscar Esteban <code@oscaresteban.es> oesteban <code@oscaresteban.es>
 Russell Poldrack <poldrack@gmail.com> Russ Poldrack <poldrack@gmail.com>
@@ -101,5 +107,6 @@ Steven Giavasis <sgiava77@gmail.com> sgiavasis <sgiavasis@ieee.org>
 Tristan Glatard <tristan.glatard@mcgill.ca> Tristan Glatard <tristan.glatard@creatis.insa-lyon.fr>
 Victor Saase <victor@saase.net> vsaase <victor@saase.net>
 William Triplett <wtriplett@gmail.com> William Triplett <wtriplett@ufl.edu>
+Wolfgang Pauli <Wolfgang.m.Pauli@gmail.com> Wolfgang Pauli <Wolfgang.m.Pauli@gmail.com>
 Yaroslav Halchenko <debian@onerussian.com> Yaroslav Halchenko <yoh@dartmouth.edu>
 pipolose <pipolose@pipoloses-air.watson.ibm.com> pipolose <pipolose@pipoloses-MacBook-Air.local>

.travis.yml

@@ -8,9 +8,10 @@ python:
 - 3.5
 - 3.6
 env:
-- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler"
-- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler"
-- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit"
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" PIP_FLAGS="--pre" CI_SKIP_TEST=1
 before_install:
 - function apt_inst {
   if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi &&
@@ -33,15 +34,20 @@ before_install:
   hash -r &&
   conda config --set always_yes yes --set changeps1 no &&
   conda update -q conda &&
-  conda install python=${TRAVIS_PYTHON_VERSION} &&
   conda config --add channels conda-forge &&
-  conda install -y nipype icu &&
-  rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; }
+  conda install python=${TRAVIS_PYTHON_VERSION} &&
+  conda install -y icu &&
+  pip install -r requirements.txt &&
+  pushd $HOME;
+  git clone https://github.com/INCF/pybids.git;
+  cd pybids;
+  pip install -e .;
+  popd; }
 #  Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi
 - travis_retry apt_inst
 - travis_retry conda_inst
 install:
-- travis_retry pip install -e .[$NIPYPE_EXTRAS]
+- travis_retry pip install $PIP_FLAGS -e .[$NIPYPE_EXTRAS]
 script:
 - py.test -v --doctest-modules nipype
 deploy:
@@ -53,4 +59,4 @@ deploy:
     tags: true
     repo: nipy/nipype
     branch: master
-  distributions: "sdist"
+  distributions: "sdist bdist_wheel"