Merge pull request #1623 from oesteban/fix/FromFileWorkflows

[FIX] Bugs found after #1572 and #1591

Author: oesteban

examples/smri_ants_registration.py

@@ -24,6 +24,7 @@
 import urllib.error
 import urllib.parse
 from nipype.interfaces.ants import Registration
+from nipype.testing import example_data
 
 """
 2. Download T1 volumes into home directory
@@ -61,11 +62,12 @@
 
 """
 3. Define the parameters of the registration. Settings are
-saved in ``smri_ants_registration_settings.json``.
+found in the file ``smri_ants_registration_settings.json``
+distributed with the ``example_data`` of `nipype`.
 
 """
 
-reg = Registration(from_file='./smri_ants_registration_settings.json')
+reg = Registration(from_file=example_data('smri_ants_registration_settings.json'))
 reg.inputs.fixed_image = input_images[0]
 reg.inputs.moving_image = input_images[1]
 

nipype/interfaces/base.py

@@ -47,6 +47,7 @@
 nipype_version = LooseVersion(__version__)
 iflogger = logging.getLogger('interface')
 
+FLOAT_FORMAT = '{:.10f}'.format
 PY35 = sys.version_info >= (3, 5)
 PY3 = sys.version_info[0] > 2
 
@@ -536,6 +537,14 @@ def _clean_container(self, object, undefinedval=None, skipundefined=False):
                     out = undefinedval
         return out
 
+    def has_metadata(self, name, metadata, value=None, recursive=True):
+        """
+        Return has_metadata for the requested trait name in this
+        interface
+        """
+        return has_metadata(self.trait(name).trait_type, metadata, value,
+                            recursive)
+
     def get_hashval(self, hash_method=None):
         """Return a dictionary of our items with hashes for each file.
 
@@ -560,61 +569,61 @@ def get_hashval(self, hash_method=None):
         dict_withhash = []
         dict_nofilename = []
         for name, val in sorted(self.get().items()):
-            if isdefined(val):
-                trait = self.trait(name)
-                if has_metadata(trait.trait_type, "nohash", True):
-                    continue
-                hash_files = (not has_metadata(trait.trait_type, "hash_files",
-                                               False) and not
-                              has_metadata(trait.trait_type, "name_source"))
-                dict_nofilename.append((name,
-                                        self._get_sorteddict(val, hash_method=hash_method,
-                                                             hash_files=hash_files)))
-                dict_withhash.append((name,
-                                      self._get_sorteddict(val, True, hash_method=hash_method,
-                                                           hash_files=hash_files)))
+            if not isdefined(val) or self.has_metadata(name, "nohash", True):
+                # skip undefined traits and traits with nohash=True
+                continue
+
+            hash_files = (not self.has_metadata(name, "hash_files", False) and not
+                          self.has_metadata(name, "name_source"))
+            dict_nofilename.append((name,
+                                    self._get_sorteddict(val, hash_method=hash_method,
+                                                         hash_files=hash_files)))
+            dict_withhash.append((name,
+                                  self._get_sorteddict(val, True, hash_method=hash_method,
+                                                       hash_files=hash_files)))
         return dict_withhash, md5(encode_dict(dict_nofilename).encode()).hexdigest()
 
-    def _get_sorteddict(self, object, dictwithhash=False, hash_method=None,
+
+    def _get_sorteddict(self, objekt, dictwithhash=False, hash_method=None,
                         hash_files=True):
-        if isinstance(object, dict):
+        if isinstance(objekt, dict):
             out = []
-            for key, val in sorted(object.items()):
+            for key, val in sorted(objekt.items()):
                 if isdefined(val):
                     out.append((key,
                                 self._get_sorteddict(val, dictwithhash,
                                                      hash_method=hash_method,
                                                      hash_files=hash_files)))
-        elif isinstance(object, (list, tuple)):
+        elif isinstance(objekt, (list, tuple)):
             out = []
-            for val in object:
+            for val in objekt:
                 if isdefined(val):
                     out.append(self._get_sorteddict(val, dictwithhash,
                                                     hash_method=hash_method,
                                                     hash_files=hash_files))
-            if isinstance(object, tuple):
+            if isinstance(objekt, tuple):
                 out = tuple(out)
         else:
-            if isdefined(object):
-                if (hash_files and isinstance(object, (str, bytes)) and
-                        os.path.isfile(object)):
+            if isdefined(objekt):
+                if (hash_files and isinstance(objekt, (str, bytes)) and
+                        os.path.isfile(objekt)):
                     if hash_method is None:
                         hash_method = config.get('execution', 'hash_method')
 
                     if hash_method.lower() == 'timestamp':
-                        hash = hash_timestamp(object)
+                        hash = hash_timestamp(objekt)
                     elif hash_method.lower() == 'content':
-                        hash = hash_infile(object)
+                        hash = hash_infile(objekt)
                     else:
                         raise Exception("Unknown hash method: %s" % hash_method)
                     if dictwithhash:
-                        out = (object, hash)
+                        out = (objekt, hash)
                     else:
                         out = hash
-                elif isinstance(object, float):
-                    out = '%.10f' % object
+                elif isinstance(objekt, float):
+                    out = FLOAT_FORMAT(objekt)
                 else:
-                    out = object
+                    out = objekt
         return out
 
 
@@ -769,7 +778,10 @@ def __init__(self, from_file=None, **inputs):
         self.num_threads = 1
 
         if from_file is not None:
-            self.load_inputs_from_json(from_file, overwrite=False)
+            self.load_inputs_from_json(from_file, overwrite=True)
+
+            for name, value in list(inputs.items()):
+                setattr(self.inputs, name, value)
 
 
     @classmethod

nipype/interfaces/tests/test_base.py

@@ -10,10 +10,11 @@
 import tempfile
 import shutil
 import warnings
+import simplejson as json
 
 from nipype.testing import (assert_equal, assert_not_equal, assert_raises,
                             assert_true, assert_false, with_setup, package_check,
-                            skipif)
+                            skipif, example_data)
 import nipype.interfaces.base as nib
 from nipype.utils.filemanip import split_filename
 from nipype.interfaces.base import Undefined, config
@@ -497,10 +498,21 @@ def __init__(self, **inputs):
     bif6.load_inputs_from_json(tmp_json)
     yield assert_equal, bif6.inputs.get_traitsfree(), inputs_dict
 
-def assert_not_raises(fn, *args, **kwargs):
-    fn(*args, **kwargs)
-    return True
+    # test get hashval in a complex interface
+    from nipype.interfaces.ants import Registration
+    settings = example_data(example_data('smri_ants_registration_settings.json'))
+    with open(settings) as setf:
+        data_dict = json.load(setf)
+
+    tsthash = Registration()
+    tsthash.load_inputs_from_json(settings)
+    yield assert_equal, {}, check_dict(data_dict, tsthash.inputs.get_traitsfree())
 
+    tsthash2 = Registration(from_file=settings)
+    yield assert_equal, {}, check_dict(data_dict, tsthash2.inputs.get_traitsfree())
+
+    _, hashvalue = tsthash.inputs.get_hashval(hash_method='timestamp')
+    yield assert_equal, 'ec5755e07287e04a4b409e03b77a517c', hashvalue
 
 def test_input_version():
     class InputSpec(nib.TraitedSpec):
@@ -738,3 +750,27 @@ def test_global_CommandLine_output():
     yield assert_equal, res.runtime.stdout, ''
     os.chdir(pwd)
     teardown_file(tmpd)
+
+def assert_not_raises(fn, *args, **kwargs):
+    fn(*args, **kwargs)
+    return True
+
+def check_dict(ref_dict, tst_dict):
+    """Compare dictionaries of inputs and and those loaded from json files"""
+    def to_list(x):
+        if isinstance(x, tuple):
+            x = list(x)
+
+        if isinstance(x, list):
+            for i, xel in enumerate(x):
+                x[i] = to_list(xel)
+
+        return x
+
+    failed_dict = {}
+    for key, value in list(ref_dict.items()):
+        newval = to_list(tst_dict[key])
+        if newval != value:
+            failed_dict[key] = (value, newval)
+    return failed_dict
+

examples/smri_ants_registration_settings.json renamed to nipype/testing/data/smri_ants_registration_settings.json

@@ -89,24 +89,38 @@ def encode_dict(value):
 
     """
     if sys.version_info[0] > 2:
-        return str(value)
-
-    if isinstance(value, str):
-        value = value.encode()
+        retval = str(value)
+    else:
+        retval = encode_dict_py27(value)
+    return retval
 
-    if isinstance(value, tuple):
-        val0 = encode_dict(value[0])
-        val1 = encode_dict(value[1])
-        return '(' + val0 + ', ' + val1 + ')'
+def encode_dict_py27(value):
+    """
+    Encode dictionary for python 2
+    """
 
-    if isinstance(value, list):
-        retval = '['
+    istuple = isinstance(value, tuple)
+    if isinstance(value, (tuple, list)):
+        retval = '(' if istuple else '['
+        nels = len(value)
         for i, v in enumerate(value):
-            if i > 0:
+            venc = encode_dict_py27(v)
+            if venc.startswith("u'") or venc.startswith('u"'):
+                venc = venc[1:]
+            retval += venc
+
+            if i < nels - 1:
                 retval += ', '
-            retval += encode_dict(v)
-        return retval + ']'
-    return repr(value)
+
+        if istuple and nels == 1:
+            retval += ','
+        retval += ')' if istuple else ']'
+        return retval
+
+    retval = repr(value).decode()
+    if retval.startswith("u'") or retval.startswith('u"'):
+        retval = retval[1:]
+    return retval
 
 def fname_presuffix(fname, prefix='', suffix='', newpath=None, use_ext=True):
     """Manipulates path and name of input filename

nipype/utils/filemanip.py

@@ -10,11 +10,11 @@
 
 from ...testing import assert_equal, assert_true, assert_false, TempFATFS
 from ...utils.filemanip import (save_json, load_json,
-                                    fname_presuffix, fnames_presuffix,
-                                    hash_rename, check_forhash,
-                                    copyfile, copyfiles,
-                                    filename_to_list, list_to_filename,
-                                    split_filename, get_related_files)
+                                fname_presuffix, fnames_presuffix,
+                                hash_rename, check_forhash,
+                                copyfile, copyfiles,
+                                filename_to_list, list_to_filename,
+                                split_filename, get_related_files)
 
 import numpy as np