Fetch upstream and Merge branch 'master' into enh/NormalizeMapsAlgorithm

Conflicts:
	CHANGES

Author: oesteban

.travis.yml

@@ -27,7 +27,7 @@ install:
   - pip install python-coveralls --use-mirrors
   - pip install nose-cov --use-mirrors
   - pip install https://github.com/RDFLib/rdflib/archive/master.zip
-  - pip install https://github.com/satra/prov/archive/enh/rdf.zip
+  - pip install https://github.com/trungdong/prov/archive/rdf.zip
   - python setup.py install
 
 # Run test

CHANGES

@@ -2,7 +2,9 @@ Next Release
 ============
 
 * ENH: New misc algorithm: NormalizeProbabilityMapSet
+* API: Interfaces to external packages are no longer available in the top-level ``nipype`` namespace, and must be imported directly (e.g. ``from nipype.interfaces import fsl``).
 * ENH: New ANTs interface: ApplyTransformsToPoints
+* ENH: New FreeSurfer workflow: create_skullstripped_recon_flow()
 * FIX: MRTrix tracking algorithms were ignoring mask parameters.
 
 Release 0.9.2 (January 31, 2014)

doc/users/mapnode_and_iterables.rst

@@ -61,7 +61,7 @@ reduce scenario.
 
 You might have also noticed that the iterfield arguments expects a list of input
 names instead of just one name. This suggests that there can be more than one!
-Even thou a bit confusing this is true. You can specify more than one input to 
+Even though a bit confusing this is true. You can specify more than one input to 
 iterate over but the lists that you provide (for all the inputs specified in 
 iterfield) have to have the same length. MapNode will then pair the parameters up
 and run the first instance with first set of parameters and second with second set
@@ -114,7 +114,7 @@ should look like this:
    "A" -> "B3" -> "C3";
 
 Of course you can do it manually by creating copies of all the nodes for 
-different parameter set, but this can be very time consuming, especially when is
+different parameter set, but this can be very time consuming, especially when there
 are more than one node taking inputs from "B". Luckily nipype supports this 
 scenario! Its called iterables and and you use it this way:
 

doc/users/spmmcr.rst

@@ -9,7 +9,7 @@ the following commands are executed at the beginning of your script:
 
 .. testcode::
 
-    from nipype import spm
+    from nipype.interfaces import spm
     matlab_cmd = '/path/to/run_spm8.sh /path/to/Compiler_Runtime/v713/ script'
     spm.SPMCommand.set_mlab_paths(matlab_cmd=matlab_cmd, use_mcr=True)
 

examples/fmri_spm.py

@@ -18,7 +18,7 @@
 from nipype import config
 config.enable_provenance()
 
-from nipype import spm, fsl
+from nipype.interfaces import spm, fsl
 
 # In order to use this example with SPM's matlab common runtime
 # matlab_cmd = ('/Users/satra/Downloads/spm8/run_spm8.sh '

examples/rsfmri_fsl.py

@@ -6,12 +6,12 @@
 rsfMRI: FSL - CSF regressed
 ===========================
 
-A pipeline example that uses intergrates several interfaces to
+A pipeline example that intergrates several interfaces to
 perform a first and second level analysis on a two-subject data
 set.
 
 
-1. Tell python where to find the appropriate functions.
+1. Tell Python where to find the appropriate functions.
 """
 
 import numpy as np

examples/rsfmri_preprocessing.py

@@ -156,7 +156,7 @@ def motion_regressors(motion_params, order=2, derivatives=2):
 
 
 def build_filter1(motion_params, comp_norm, outliers):
-    """Builds a regressor set comprisong motion parameters, composite norm and
+    """Builds a regressor set comparison motion parameters, composite norm and
     outliers
 
     The outliers are added as a single time point column for each outlier
@@ -364,7 +364,7 @@ def create_workflow(files,
     fssource.inputs.subject_id = subject_id
     fssource.inputs.subjects_dir = os.environ['SUBJECTS_DIR']
 
-    # Extract wm+csf, brain masks by eroding freesurfer lables and then
+    # Extract wm+csf, brain masks by eroding freesurfer labels and then
     # transform the masks into the space of the median
     wmcsf = Node(freesurfer.Binarize(), name='wmcsfmask')
     mask = wmcsf.clone('anatmask')

nipype/__init__.py

@@ -75,6 +75,5 @@ def _test_local_install():
 
 
 from pipeline import Node, MapNode, JoinNode, Workflow
-from interfaces import (fsl, spm, freesurfer, afni, ants, slicer, dipy, nipy,
-                        mrtrix, camino, DataGrabber, DataSink, SelectFiles,
+from interfaces import (DataGrabber, DataSink, SelectFiles,
                         IdentityInterface, Rename, Function, Select, Merge)

nipype/external/provcopy.py

@@ -262,6 +262,13 @@ def get_anon_id(self, obj, local_prefix="id"):
 class Literal(object):
     def __init__(self, value, datatype=None, langtag=None):
         self._value = value
+        if langtag:
+            if datatype is None:
+                logger.debug('Assuming prov:InternationalizedString as the type of "%s"@%s' % (value, langtag))
+                datatype = PROV["InternationalizedString"]
+            elif datatype != PROV["InternationalizedString"]:
+                logger.warn('Invalid data type (%s) for "%s"@%s, overridden as prov:InternationalizedString.' % (value, langtag))
+                datatype = PROV["InternationalizedString"]
         self._datatype = datatype
         self._langtag = langtag
 
@@ -291,14 +298,14 @@ def has_no_langtag(self):
 
     def provn_representation(self):
         if self._langtag:
-            #  a langtag can only goes with string
+            #  a language tag can only go with prov:InternationalizedString
             return u'%s@%s' % (_ensure_multiline_string_triple_quoted(self._value), unicode(self._langtag))
         else:
             return u'%s %%%% %s' % (_ensure_multiline_string_triple_quoted(self._value), unicode(self._datatype))
 
     def json_representation(self):
         if self._langtag:
-            #  a langtag can only goes with string
+            #  a language tag can only go with prov:InternationalizedString
             return {'$': unicode(self._value), 'lang': self._langtag}
         else:
             if isinstance(self._datatype, QName):
@@ -424,6 +431,9 @@ def __init__(self, record_type, attribute_id):
         self.attribute_id = attribute_id
         self.args += (PROV_N_MAP[record_type], attribute_id)
 
+    def __str__(self):
+        return 'Missing the required attribute "%s" in %s' % (PROV_ID_ATTRIBUTES_MAP[self.attribute_id], PROV_N_MAP[self.record_type])
+
 
 class ProvExceptionNotValidAttribute(ProvException):
     def __init__(self, record_type, attribute, attribute_types):
@@ -432,6 +442,9 @@ def __init__(self, record_type, attribute, attribute_types):
         self.attribute_types = attribute_types
         self.args += (PROV_N_MAP[record_type], unicode(attribute), attribute_types)
 
+    def __str__(self):
+        return 'Invalid attribute value: %s. %s expected' % (self.attribute, self.attribute_types)
+
 
 class ProvExceptionCannotUnifyAttribute(ProvException):
     def __init__(self, identifier, record_type1, record_type2):
@@ -440,6 +453,9 @@ def __init__(self, identifier, record_type1, record_type2):
         self.record_type2 = record_type2
         self.args += (identifier, PROV_N_MAP[record_type1], PROV_N_MAP[record_type2])
 
+    def __str__(self):
+        return 'Cannot unify two records of type %s and %s with same identifier (%s)' % (self.identifier, PROV_N_MAP[self.record_type1], PROV_N_MAP[self.record_type2])
+
 
 class ProvExceptionContraint(ProvException):
     def __init__(self, record_type, attribute1, attribute2, msg):
@@ -517,12 +533,10 @@ def get_value(self):
     def _auto_literal_conversion(self, literal):
         '''This method normalise datatype for literals
         '''
+        if isinstance(literal, URIRef):
+            return literal
+
         if isinstance(literal, basestring):
-            # try if this is a QName
-            qname = self._bundle.valid_identifier(literal)
-            if isinstance(qname, QName):
-                return qname
-            # if not a QName, convert all strings to unicode
             return unicode(literal)
 
         if isinstance(literal, Literal) and literal.has_no_langtag():
@@ -539,7 +553,9 @@ def parse_extra_attributes(self, extra_attributes):
         if isinstance(extra_attributes, dict):
             #  Converting the dictionary into a list of tuples (i.e. attribute-value pairs)
             extra_attributes = extra_attributes.items()
-        attr_set = set((self._bundle.valid_identifier(attribute), self._auto_literal_conversion(value)) for attribute, value in extra_attributes)
+        attr_set = set((self._bundle.valid_identifier(attribute),
+                        self._auto_literal_conversion(value))
+                       for attribute, value in extra_attributes)
         return attr_set
 
     def add_extra_attributes(self, extra_attributes):
@@ -653,7 +669,7 @@ def optional_attribute(self, attributes, attribute_id, attribute_types):
         return self._validate_attribute(attribute, attribute_types)
 
     def __eq__(self, other):
-        if self.__class__ != other.__class__:
+        if self.get_prov_type() != other.get_prov_type():
             return False
         if self._identifier and not (self._identifier == other._identifier):
             return False
@@ -682,6 +698,13 @@ def __eq__(self, other):
         sattr = sorted(self._extra_attributes, key=_normalise_attributes) if self._extra_attributes else None
         oattr = sorted(other._extra_attributes, key=_normalise_attributes) if other._extra_attributes else None
         if sattr != oattr:
+            if logger.isEnabledFor(logging.DEBUG):
+                for spair, opair in zip(sattr, oattr):
+                    # Log the first unequal pair of attributes
+                    if spair != opair:
+                        logger.debug("Equality (ProvRecord): unequal attribute-value pairs - %s = %s - %s = %s",
+                                     spair[0], spair[1], opair[0], opair[1])
+                        break
             return False
         return True
 
@@ -740,17 +763,28 @@ def rdf(self, graph=None, subj=None):
                 graph.add((subj, pred, obj))
         if self._extra_attributes:
             for (attr, value) in self._extra_attributes:
+                try:
+                    # try if there is a RDF representation defined
+                    obj = value.rdf_representation()
+                except Exception, e:
+                    obj = RDFLiteral(value)
+                if attr == PROV['location']:
+                    pred = PROV['atLocation'].rdf_representation()
+                    if isinstance(value, (URIRef, QName)):
+                        if isinstance(value, QName):
+                            value = URIRef(value.get_uri())
+                        graph.add((subj, pred, value))
+                        graph.add((value, RDF.type,
+                                   PROV['Location'].rdf_representation()))
+                    else:
+                        graph.add((subj, pred, obj))
+                    continue
                 if attr == PROV['type']:
                     pred = RDF.type
                 elif attr == PROV['label']:
                     pred = RDFS.label
                 else:
                     pred = attr.rdf_representation()
-                try:
-                    # try if there is a RDF representation defined
-                    obj = value.rdf_representation()
-                except Exception, e:
-                    obj = RDFLiteral(value)
                 graph.add((subj, pred, obj))
         return graph
 
@@ -853,8 +887,7 @@ def add_attributes(self, attributes, extra_attributes):
         startTime = self.optional_attribute(attributes, PROV_ATTR_STARTTIME, datetime.datetime)
         endTime = self.optional_attribute(attributes, PROV_ATTR_ENDTIME, datetime.datetime)
         if startTime and endTime and startTime > endTime:
-            #  TODO Raise logic exception here
-            pass
+            raise ValueError('StartTime %s > EndTime %s' % (startTime, endTime))
         attributes = OrderedDict()
         attributes[PROV_ATTR_STARTTIME] = startTime
         attributes[PROV_ATTR_ENDTIME] = endTime
@@ -1337,8 +1370,8 @@ def get_valid_identifier(self, identifier):
                 #  create and return an identifier in the default namespace
                 return self._default[identifier]
             else:
-                #  TODO Should an exception raised here
-                return Identifier(identifier)
+                # This is not an identifier
+                return None
 
     def get_anonymous_identifier(self, local_prefix='id'):
         self._anon_id_count += 1
@@ -1362,11 +1395,7 @@ def __init__(self, bundle=None, identifier=None, attributes=None, other_attribut
         self._records = list()
         self._id_map = dict()
         self._bundles = dict()
-        if bundle is None:
-            self._namespaces = NamespaceManager(namespaces)
-        else:
-            self._namespaces = bundle._namespaces
-            self._namespaces.add_namespaces(namespaces)
+        self._namespaces = NamespaceManager(namespaces, parent=(bundle._namespaces if bundle is not None else None))
 
         #  Initializing record-specific attributes
         super(ProvBundle, self).__init__(bundle, identifier, attributes, other_attributes, asserted)
@@ -1379,9 +1408,6 @@ def get_default_namespace(self):
         return self._namespaces.get_default_namespace()
 
     def add_namespace(self, namespace_or_prefix, uri=None):
-        if self._bundle is not None:  # This is a bundle
-            logger.warn("Namespace cannot be added into a bundle. It will be added to the document instead.")
-
         if uri is None:
             self._namespaces.add_namespace(namespace_or_prefix)
         else:
@@ -1398,10 +1424,12 @@ def get_anon_id(self, record):
         return self._namespaces.get_anonymous_identifier()
 
     def get_records(self, class_or_type_or_tuple=None):
-        if class_or_type_or_tuple is None:
-            return self._records
+        # Only returning asserted records
+        results = [rec for rec in self._records if rec.is_asserted()]
+        if class_or_type_or_tuple:
+            return filter(lambda rec: isinstance(rec, class_or_type_or_tuple), results)
         else:
-            return filter(lambda rec: isinstance(rec, class_or_type_or_tuple), self._records)
+            return results
 
     def get_record(self, identifier):
         if identifier is None:
@@ -1453,22 +1481,21 @@ def _encode_json_representation(self, value):
                 return value
 
     def _decode_json_representation(self, literal):
-        try:
+        if isinstance(literal, dict):
+            # complex type
             value = literal['$']
-            if 'lang' in literal:
-                return Literal(value, langtag=literal['lang'])
+            datatype = literal['type'] if 'type' in literal else None
+            langtag = literal['lang'] if 'lang' in literal else None
+            if datatype == u'xsd:anyURI':
+                return Identifier(value)
+            elif datatype == u'xsd:QName':
+                return self.valid_identifier(value)
             else:
-                datatype = literal['type']
-                if datatype == u'xsd:anyURI':
-                    return Identifier(value)
-                elif datatype == u'xsd:QName':
-                    return self.valid_identifier(value)
-                else:
-                    # The literal of standard Python types is not converted here
-                    # It will be automatically converted when added to a record by _auto_literal_conversion()
-                    return Literal(value, self.valid_identifier(datatype))
-        except:
-            #  simple type, just return it
+                # The literal of standard Python types is not converted here
+                # It will be automatically converted when added to a record by _auto_literal_conversion()
+                return Literal(value, self.valid_identifier(datatype), langtag)
+        else:
+            # simple type, just return it
             return literal
 
     def _encode_JSON_container(self):
@@ -1701,12 +1728,10 @@ def get_flattened(self):
         return document
 
     def __eq__(self, other):
-        try:
-            other_records = set(other._records)
-        except:
-            #  other is not a bundle
+        if not isinstance(other, ProvBundle):
             return False
-        this_records = set(self._records)
+        other_records = set(other.get_records())
+        this_records = set(self.get_records())
         if len(this_records) != len(other_records):
             return False
         #  check if all records for equality
@@ -1721,12 +1746,12 @@ def __eq__(self, other):
                         other_records.remove(record_b)
                         continue
                     else:
-                        logger.debug("Unequal PROV records:")
-                        logger.debug("%s" % unicode(record_a))
-                        logger.debug("%s" % unicode(record_b))
+                        logger.debug("Equality (ProvBundle): Unequal PROV records:")
+                        logger.debug("%s", unicode(record_a))
+                        logger.debug("%s", unicode(record_b))
                         return False
                 else:
-                    logger.debug("Could not find a record with this identifier: %s" % unicode(record_a._identifier))
+                    logger.debug("Equality (ProvBundle): Could not find a record with this identifier: %s", unicode(record_a._identifier))
                     return False
             else:
                 #  Manually look for the record
@@ -1737,7 +1762,7 @@ def __eq__(self, other):
                         found = True
                         break
                 if not found:
-                    logger.debug("Could not find this record: %s" % unicode(record_a))
+                    logger.debug("Equality (ProvBundle): Could not find this record: %s", unicode(record_a))
                     return False
         return True
 

nipype/interfaces/__init__.py

@@ -9,4 +9,3 @@
 
 from io import DataGrabber, DataSink, SelectFiles
 from utility import IdentityInterface, Rename, Function, Select, Merge
-import fsl, spm, freesurfer, afni, ants, slicer, dipy, nipy, mrtrix, camino