Merge branch 'master' of github.com:nipy/nipype into fix/mnibias

Author: mgxd

.dockerignore

@@ -15,9 +15,6 @@ nipype.egg-info
 src/**/*
 src/
 
-# releasing
-Makefile
-
 # git
 .gitignore
 .git/**/*

CHANGES

@@ -1,6 +1,11 @@
 Upcoming release 0.13
 =====================
 
+* ENH: DVARS includes intensity normalization feature - turned on by default (https://github.com/nipy/nipype/pull/1827)
+* FIX: DVARS is correctly using sum of squares instead of standard deviation (https://github.com/nipy/nipype/pull/1827)
+* ENH: Refactoring of nipype.interfaces.utility (https://github.com/nipy/nipype/pull/1828)
+* FIX: CircleCI were failing silently. Some fixes to tests (https://github.com/nipy/nipype/pull/1833)
+* FIX: Issues in Docker image permissions, and docker documentation (https://github.com/nipy/nipype/pull/1825)
 * ENH: Revised all Dockerfiles and automated deployment to Docker Hub
        from CircleCI (https://github.com/nipy/nipype/pull/1815)
 * FIX: Semaphore capture using MultiProc plugin (https://github.com/nipy/nipype/pull/1689)

Dockerfile

@@ -31,12 +31,19 @@
 # Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile
 #
 FROM ubuntu:xenial-20161213
+MAINTAINER The nipype developers https://github.com/nipy/nipype
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Pre-cache neurodebian key
+COPY docker/files/neurodebian.gpg /root/.neurodebian.gpg
 
 # Prepare environment
-RUN apt-get update && \
+RUN apt-key add /root/.neurodebian.gpg && \
+    apt-get update && \
     apt-get install -y --no-install-recommends curl bzip2 ca-certificates xvfb && \
     curl -sSL http://neuro.debian.net/lists/xenial.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \
-    apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \
+    apt-key adv --refresh-keys --keyserver hkp://ha.pool.sks-keyservers.net 0xA5D32F012649A5A9 || true; \
     apt-get update
 
 # Installing freesurfer
@@ -78,6 +85,7 @@ RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW
 # Installing Neurodebian packages (FSL, AFNI, git)
 RUN apt-get install -y --no-install-recommends \
                     fsl-core=5.0.9-1~nd+1+nd16.04+1 \
+                    fsl-mni152-templates=5.0.7-2 \
                     afni=16.2.07~dfsg.1-2~nd16.04+1
 
 ENV FSLDIR=/usr/share/fsl/5.0 \
@@ -171,7 +179,7 @@ RUN conda config --add channels conda-forge --add channels intel && \
     chmod +x /usr/local/miniconda/bin/* && \
     conda config --set always_yes yes --set changeps1 no && \
     conda update -q conda && \
-    chmod +x /usr/local/miniconda/bin/* && \
+    chmod +x /usr/local/miniconda/bin/*; sync && \
     conda install -y mkl=2017.0.1 \
                      numpy=1.11.2 \
                      scipy=0.18.1 \
@@ -182,7 +190,8 @@ RUN conda config --add channels conda-forge --add channels intel && \
                      libxslt=1.1.29 \
                      traits=4.6.0 \
                      psutil=5.0.1 \
-                     icu=58.1
+                     icu=58.1 && \
+    find /usr/local/miniconda/ -exec chmod 775 {} +
 
 # matplotlib cleanups: set default backend, precaching fonts
 RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc && \
@@ -195,7 +204,7 @@ ENV MKL_NUM_THREADS=1 \
 
 # Installing dev requirements (packages that are not in pypi)
 WORKDIR /root/
-ADD requirements.txt requirements.txt
+COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt && \
     rm -rf ~/.cache/pip
 

Makefile

@@ -32,6 +32,7 @@ trailing-spaces:
 
 clean-pyc:
 	find . -name "*.pyc" | xargs rm -f
+	find . -name "__pycache__" -type d | xargs rm -rf
 
 clean-so:
 	find . -name "*.so" | xargs rm -f
@@ -70,7 +71,7 @@ html:
 
 specs:
 	@echo "Checking specs and autogenerating spec tests"
-	python tools/checkspecs.py
+	env PYTHONPATH=".:$(PYTHONPATH)" python tools/checkspecs.py
 
 check: check-before-commit # just a shortcut
 check-before-commit: specs trailing-spaces html test

circle.yml

@@ -25,22 +25,21 @@ dependencies:
     - mkdir -p $SCRATCH && sudo setfacl -d -m group:ubuntu:rwx $SCRATCH && sudo setfacl -m group:ubuntu:rwx $SCRATCH
     - mkdir -p $HOME/docker $HOME/examples $SCRATCH/pytest $SCRATCH/logs
   override:
+    - if [[ -e $HOME/docker/cache.tar ]]; then docker load --input $HOME/docker/image.tar; else echo 'No docker image found in cache'; fi :
+        timeout: 6000
     - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi
     - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi
     - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi
-    - if [[ -e $HOME/docker/image.tar ]]; then docker load -i $HOME/docker/image.tar; fi
-    - if [[ -e $HOME/docker/image27.tar ]]; then docker load -i $HOME/docker/image27.tar; fi
-    - if [[ -e $HOME/docker/image35.tar ]]; then docker load -i $HOME/docker/image35.tar; fi
+    - docker images
     - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
     - e=1 && for i in {1..5}; do docker build -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 21600
-    - docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . :
+    - e=1 && for i in {1..5}; do docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 1600
-    - docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . :
+    - e=1 && for i in {1..5}; do docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 1600
-    - docker save nipype/nipype:latest > $HOME/docker/image.tar
-    - docker save nipype/nipype_test:py27 > $HOME/docker/image27.tar
-    - docker save nipype/nipype_test:py35 > $HOME/docker/image35.tar
+    - docker save -o $HOME/docker/cache.tar nipype/nipype:latest nipype/nipype_test:py27 nipype/nipype_test:py35 :
+        timeout: 6000
 
 test:
   override:

doc/devel/testing_nipype.rst

@@ -107,26 +107,47 @@ This will skip any tests that require matlab.
 Testing Nipype using Docker
 ---------------------------
 
-As of :code:`nipype-0.13`, Nipype is tested inside Docker containers. Once the developer
-`has installed the Docker Engine <https://docs.docker.com/engine/installation/>`_, testing
-Nipype is as easy as follows::
+As of :code:`nipype-0.13`, Nipype is tested inside Docker containers. First, install the
+`Docker Engine <https://docs.docker.com/engine/installation/>`_.
+Nipype has one base docker image called nipype/nipype, and several additional test images
+for various Python versions.
+
+The base nipype/nipype image is built as follows::
 
   cd path/to/nipype/
-  docker build -f docker/nipype_test/Dockerfile_py27 -t nipype/nipype_test:py27
-  docker run -it --rm -v /etc/localtime:/etc/localtime:ro \
-                      -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" \
+  docker build -t nipype/nipype .
+
+This base image contains several useful tools (FreeSurfer, AFNI, FSL, ANTs, etc.) and
+a nipype installation, all in Python 3.5.
+It is possible to fetch a built image from the latest master branch of nipype
+using::
+
+  docker run -it --rm nipype/nipype:master
+
+
+The docker run command will then open the container and offer a bash shell for the
+developer.
+
+The additional test images have several test scripts installed. For instance,
+to build and run all tests on Python 2.7::
+
+  cd path/to/nipype/
+  docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 .
+  docker run -it --rm -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" \
                       -v ~/examples:/root/examples:ro \
                       -v ~/scratch:/scratch \
                       -w /root/src/nipype \
-                      nipype/nipype_test:py27 /usr/bin/run_pytest.sh
+                      nipype/nipype_test:py27 /usr/bin/run_pytests.sh
 
 For running nipype in Python 3.5::
 
   cd path/to/nipype/
-  docker build -f docker/nipype_test/Dockerfile_py35 -t nipype/nipype_test:py35
-  docker run -it --rm -v /etc/localtime:/etc/localtime:ro \
-                      -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" \
+  docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 .
+  docker run -it --rm -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" \
                       -v ~/examples:/root/examples:ro \
                       -v ~/scratch:/scratch \
                       -w /root/src/nipype \
-                      nipype/nipype_test:py35 /usr/bin/run_pytest.sh
+                      nipype/nipype_test:py35 /usr/bin/run_pytests.sh
+
+The last two examples assume that the example data is downladed into ~/examples and
+the ~/scratch folder will be created if it does not exist previously.

docker/Dockerfile_base

@@ -0,0 +1,71 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQGiBEQ7TOgRBADvaRsIZ3VZ6Qy7PlDpdMm97m0OfvouOj/HhjOM4M3ECbGn4cYh
+vN1gK586s3sUsUcNQ8LuWvNsYhxYsVTZymCReJMEDxod0U6/z/oIbpWv5svF3kpl
+ogA66Ju/6cZx62RiCSOkskI6A3Waj6xHyEo8AGOPfzbMoOOQ1TS1u9s2FwCgxziL
+wADvKYlDZnWM03QtqIJVD8UEAOks9Q2OqFoqKarj6xTRdOYIBVEp2jhozZUZmLmz
+pKL9E4NKGfixqxdVimFcRUGM5h7R2w7ORqXjCzpiPmgdv3jJLWDnmHLmMYRYQc8p
+5nqo8mxuO3zJugxBemWoacBDd1MJaH7nK20Hsk9L/jvU/qLxPJotMStTnwO+EpsK
+HlihA/9ZpvzR1QWNUd9nSuNR3byJhaXvxqQltsM7tLqAT4qAOJIcMjxr+qESdEbx
+NHM5M1Y21ZynrsQw+Fb1WHXNbP79vzOxHoZR0+OXe8uUpkri2d9iOocre3NUdpOO
+JHtl6cGGTFILt8tSuOVxMT/+nlo038JQB2jARe4B85O0tkPIPbQybmV1cm8uZGVi
+aWFuLm5ldCBhcmNoaXZlIDxtaWNoYWVsLmhhbmtlQGdtYWlsLmNvbT6IRgQQEQgA
+BgUCTVHJKwAKCRCNEUVjdcAkyOvzAJ0abJz+f2a6VZG1c9T8NHMTYh1atwCgt0EE
+3ZZd/2in64jSzu0miqhXbOKISgQQEQIACgUCSotRlwMFAXgACgkQ93+NsjFEvg8n
+JgCfWcdJbILBtpLZCocvOzlLPqJ0Fn0AoI4EpJRxoUnrtzBGUC1MqecU7WsDiGAE
+ExECACAFAkqLUWcCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCl0y8BJkml
+qVklAJ4h2V6MdQkSAThF5c2Gkq6eSoIQYQCeM0DWyB9Bl+tTPSTYXwwZi2uoif20
+QmFwc3kuZ3NlLnVuaS1tYWdkZWJ1cmcuZGUgRGViaWFuIEFyY2hpdmUgPG1pY2hh
+ZWwuaGFua2VAZ21haWwuY29tPohGBBARAgAGBQJEO03FAAoJEPd/jbIxRL4PU18A
+n3tn7i4qdlMi8kHbYWFoabsKc9beAJ9sl/leZNCYNMGhz+u6BQgyeLKw94heBBMR
+AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA
+n27DvtZizNEbhz3wRUPQMiQjtqdvAJ9rS9YdPe5h5o5gHx3mw3BSkOttdYheBBMR
+AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA
+oLhwWL+E+2I9lrUf4Lf26quOK9vLAKC9ZpIF2tUirFFkBWnQvu13/TA0SokCHAQQ
+AQIABgUCTSNBgQAKCRDAc9Iof/uem4NpEACQ8jxmaCaS/qk/Y4GiwLA5bvKosG3B
+iARZ2v5UWqCZQ1tS56yKse/lCIzXQqU9BnYW6wOI2rvFf9meLfd8h96peG6oKscs
+fbclLDIf68bBvGBQaD0VYFi/Fk/rxmTQBOCQ3AJZs8O5rIM4gPGE0QGvSZ1h7VRw
+3Uyeg4jKXLIeJn2xEmOJgt3auAR2FyKbzHaX9JCoByJZ/eU23akNl9hgt7ePlpXo
+74KNYC58auuMUhCq3BQDB+II4ERYMcmFp1N5ZG05Cl6jcaRRHDXz+Ax6DWprRI1+
+RH/Yyae6LmKpeJNwd+vM14aawnNO9h8IAQ+aJ3oYZdRhGyybbin3giJ10hmWveg/
+Pey91Nh9vBCHdDkdPU0s9zE7z/PHT0c5ccZRukxfZfkrlWQ5iqu3V064ku5f4PBy
+8UPSkETcjYgDnrdnwqIAO+oVg/SFlfsOzftnwUrvwIcZlXAgtP6MEEAs/38e/JIN
+g4VrpdAy7HMGEUsh6Ah6lvGQr+zBnG44XwKfl7e0uCYkrAzUJRGM5vx9iXvFMcMu
+jv9EBNNBOU8/Y6MBDzGZhgaoeI27nrUvaveJXjAiDKAQWBLjtQjINZ8I9uaSGOul
+8kpbFavE4eS3+KhISrSHe4DuAa3dk9zI+FiPvXY1ZyfQBtNpR+gYFY6VxMbHhY1U
+lSLHO2eUIQLdYbRITmV1cm9EZWJpYW4gQXJjaGl2ZSBLZXkgPHBrZy1leHBwc3kt
+bWFpbnRhaW5lcnNAbGlzdHMuYWxpb3RoLmRlYmlhbi5vcmc+iEYEEBEIAAYFAk1R
+yQYACgkQjRFFY3XAJMgEWwCggx4Gqlcrt76TSMlbU94cESo55AEAoJ3asQEMpe8t
+QUX+5aikw3z1AUoCiEoEEBECAAoFAkqf/3cDBQF4AAoJEPd/jbIxRL4PxyMAoKUI
+RPWlHCj/+HSFfwhos68wcSwmAKChuC00qutDro+AOo+uuq6YoHXj+ohgBBMRAgAg
+BQJKn/8bAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQpdMvASZJpalDggCe
+KF9KOgOPdQbFnKXl8KtHory4EEwAnA7jxgorE6kk2QHEXFSF8LzOOH4GiGMEExEC
+ACMCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAUCSp//RgIZAQAKCRCl0y8BJkml
+qekFAKCRyt4+FoCzmBbRUUP3Cr8PzH++IgCgkno4vdjsWdyAey8e0KpITTXMFrmJ
+AhwEEAECAAYFAk0jQYEACgkQwHPSKH/7npsFfw/+P8B8hpM3+T1fgboBa4R32deu
+n8m6b8vZMXwuo/awQtMpzjem8JGXSUQm8iiX4hDtjq6ZoPrlN8T4jNmviBt/F5jI
+Jji/PYmhq+Zn9s++mfx+aF4IJrcHJWFkg/6kJzn4oSdl/YlvKf4VRCcQNtj4xV87
+GsdamnzU17XapLVMbSaVKh+6Af7ZLDerEH+iAq733HsYaTK+1xKmN7EFVXgS7bZ1
+9C4LTzc97bVHSywpT9yIrg9QQs/1kshfVIHDKyhjF6IwzSVbeGAIL3Oqo5zOMkWv
+7JlEIkkhTyl+FETxNMTMYjAk+Uei3kRodneq3YBF2uFYSEzrXQgHAyn37geiaMYj
+h8wu6a85nG1NS0SdxiZDIePmbvD9vWxFZUWYJ/h9ifsLivWcVXlvHoQ0emd+n2ai
+FhAck2xsuyHgnGIZMHww5IkQdu/TMqvbcR6d8Xulh+C4Tq7ppy+oTLADSBKII++p
+JQioYydRD529EUJgVlhyH27X6YAk3FuRD3zYZRYS2QECiKXvS665o3JRJ0ZSqNgv
+YOom8M0zz6bI9grnUoivMI4o7ISpE4ZwffEd37HVzmraaUHDXRhkulFSf1ImtXoj
+V9nNSM5p/+9eP7OioTZhSote6Vj6Ja1SZeRkXZK7BwqPbdO0VsYOb7G//ZiOlqs+
+paRr92G/pwBfj5Dq8EK5Ag0ERDtM9RAIAN0EJqBPvLN0tEin/y4Fe0R4n+E+zNXg
+bBsq4WidwyUFy3h/6u86FYvegXwUqVS2OsEs5MwPcCVJOfaEthF7I89QJnP9Nfx7
+V5I9yFB53o9ii38BN7X+9gSjpfwXOvf/wIDfggxX8/wRFel37GRB7TiiABRArBez
+s5x+zTXvT++WPhElySj0uY8bjVR6tso+d65K0UesvAa7PPWeRS+3nhqABSFLuTTT
+MMbnVXCGesBrYHlFVXClAYrSIOX8Ub/UnuEYs9+hIV7U4jKzRF9WJhIC1cXHPmOh
+vleAf/I9h/0KahD7HLYud40pNBo5tW8jSfp2/Q8TIE0xxshd51/xy4MAAwUH+wWn
+zsYVk981OKUEXul8JPyPxbw05fOd6gF4MJ3YodO+6dfoyIl3bewk+11KXZQALKaO
+1xmkAEO1RqizPeetoadBVkQBp5xPudsVElUTOX0pTYhkUd3iBilsCYKK1/KQ9KzD
+I+O/lRsm6L9lc6rV0IgPU00P4BAwR+x8Rw7TJFbuS0miR3lP1NSguz+/kpjxzmGP
+LyHJ+LVDYFkk6t0jPXhqFdUY6McUTBDEvavTGlVO062l9APTmmSMVFDsPN/rBes2
+rYhuuT+lDp+gcaS1UoaYCIm9kKOteQBnowX9V74Z+HKEYLtwILaSnNe6/fNSTvyj
+g0z+R+sPCY4nHewbVC+ISQQYEQIACQUCRDtM9QIbDAAKCRCl0y8BJkmlqbecAJ9B
+UdSKVg9H+fQNyP5sbOjj4RDtdACfXHrRHa2+XjJP0dhpvJ8IfvYnQsU=
+=fAJZ
+-----END PGP PUBLIC KEY BLOCK-----

docker/files/neurodebian.gpg

@@ -6,6 +6,4 @@ set -u
 mkdir -p /scratch/docs
 make html 2>&1 | tee /scratch/builddocs.log
 cp -r /root/src/nipype/doc/_build/html/* /scratch/docs/
-chmod 777 -R /scratch/docs
-chmod 777 /scratch/builddocs.log
 cat /scratch/builddocs.log && if grep -q "ERROR" /scratch/builddocs.log; then false; else true; fi

docker/files/run_builddocs.sh

@@ -15,5 +15,3 @@ coverage run /root/src/nipype/tools/run_examples.py $@
 arr=$@
 tmp_var=$( IFS=$' '; echo "${arr[*]}" )
 coverage xml -o "/scratch/smoketest_${tmp_var//[^A-Za-z0-9_-]/_}.xml"
-
-chmod 777 -R /scratch/logs