Merge branch 'master' of git://github.com/nipy/nipype into rel/0.14.0

Author: mgxd

.circleci/config.yml

@@ -67,8 +67,9 @@ jobs:
               echo "Pulling base image ..."
               docker pull nipype/nipype:base
             elif [ "$GET_BASE" == "BUILD" ]; then
-              echo "Building base image ..."
-              docker build -t nipype/nipype:base - < docker/Dockerfile.base
+              e=1 && for i in {1..5}; do
+                docker build -t nipype/nipype:base - < docker/Dockerfile.base  && e=0 && break || sleep 15
+              done && [ "$e" -eq "0" ]
             else
               echo "Error: method to get base image not understood"
               exit 1
@@ -138,15 +139,11 @@ jobs:
                           nipype/nipype:py36 \
                           nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz
               du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz
-            else
-              # Workaround for `persist_to_workspace` to succeed when we are
-              # not deploying Docker images.
-              touch /tmp/docker/nipype-base-latest-py36-py27.tar.gz
             fi
       - persist_to_workspace:
           root: /tmp
           paths:
-            - docker/nipype-base-latest-py36-py27.tar.gz
+            - docker/*
 
 
   deploy:

doc/users/plugins.rst

@@ -82,6 +82,10 @@ Optional arguments::
   exceed the total amount of resources available (memory and threads), when
   ``False`` (default), only a warning will be issued.
 
+  maxtasksperchild : number of nodes to run on each process before refreshing 
+  the worker (default: 10).
+  
+
 To distribute processing on a multicore machine, simply call::
 
   workflow.run(plugin='MultiProc')

docker/generate_dockerfiles.sh

@@ -53,8 +53,8 @@ do
 done
 
 
-# neurodocker version 0.3.1-19-g8d02eb4
-NEURODOCKER_IMAGE="kaczmarj/neurodocker@sha256:6b5f92f413b9710b7581e62293a8f74438b14ce7e4ab1ce68db2a09f7c64375a"
+# neurodocker version 0.3.1-22-gb0ee069
+NEURODOCKER_IMAGE="kaczmarj/neurodocker@sha256:c670ec2e0666a63d4e017a73780f66554283e294f3b12250928ee74b8a48bc59"
 
 # neurodebian:stretch-non-free pulled on November 3, 2017
 BASE_IMAGE="neurodebian@sha256:7590552afd0e7a481a33314724ae27f76ccedd05ffd7ac06ec38638872427b9b"

nipype/pipeline/plugins/multiproc.py

@@ -60,6 +60,7 @@ def run_node(node, updatehash, taskid):
 class NonDaemonProcess(Process):
     """A non-daemon process to support internal multiprocessing.
     """
+
     def _get_daemon(self):
         return False
 
@@ -104,6 +105,8 @@ class MultiProcPlugin(DistributedPluginBase):
     - scheduler: sort jobs topologically (``'tsort'``, default value)
         or prioritize jobs by, first, memory consumption and, second,
         number of threads (``'mem_thread'`` option).
+    - maxtasksperchild: number of nodes to run on each process before
+        refreshing the worker (default: 10).
 
     """
 
@@ -116,6 +119,7 @@ def __init__(self, plugin_args=None):
 
         # Read in options or set defaults.
         non_daemon = self.plugin_args.get('non_daemon', True)
+        maxtasks = self.plugin_args.get('maxtasksperchild', 10)
         self.processors = self.plugin_args.get('n_procs', cpu_count())
         self.memory_gb = self.plugin_args.get('memory_gb',  # Allocate 90% of system memory
                                               get_system_total_memory_gb() * 0.9)
@@ -124,7 +128,14 @@ def __init__(self, plugin_args=None):
         # Instantiate different thread pools for non-daemon processes
         logger.debug('MultiProcPlugin starting in "%sdaemon" mode (n_procs=%d, mem_gb=%0.2f)',
                      'non' if non_daemon else '', self.processors, self.memory_gb)
-        self.pool = (NonDaemonPool if non_daemon else Pool)(processes=self.processors)
+
+        NipypePool = NonDaemonPool if non_daemon else Pool
+        try:
+            self.pool = NipypePool(processes=self.processors,
+                                   maxtasksperchild=maxtasks)
+        except TypeError:
+            self.pool = NipypePool(processes=self.processors)
+
         self._stats = None
 
     def _async_callback(self, args):

nipype/pkg_info.py

@@ -3,14 +3,13 @@
 
 from future import standard_library
 standard_library.install_aliases()
+from builtins import open
 import configparser
 
 import os
 import sys
 import subprocess
 
-from .info import VERSION
-
 COMMIT_INFO_FNAME = 'COMMIT_INFO.txt'
 PY3 = sys.version_info[0] >= 3
 
@@ -52,7 +51,8 @@ def pkg_commit_hash(pkg_path):
         cfg_parser = configparser.RawConfigParser()
     else:
         cfg_parser = configparser.ConfigParser()
-    cfg_parser.read(pth)
+    with open(pth, encoding='utf-8') as fp:
+        cfg_parser.readfp(fp)
     archive_subst = cfg_parser.get('commit hash', 'archive_subst_hash')
     if not archive_subst.startswith('$Format'):  # it has been substituted
         return 'archive substitution', archive_subst
@@ -86,6 +86,9 @@ def get_pkg_info(pkg_path):
        with named parameters of interest
     '''
     src, hsh = pkg_commit_hash(pkg_path)
+    from .info import VERSION
+    if not PY3:
+        src, hsh, VERSION = src.encode(), hsh.encode(), VERSION.encode()
     import networkx
     import nibabel
     import numpy