From e655a5add7d36cd75b06e6b441d5bc22df3878e2 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 09:24:36 -0800 Subject: [PATCH 01/10] [ENH] Refactoring of nipype.interfaces.utility Split utility.py into a module --- nipype/interfaces/utility/__init__.py | 15 + .../{utility.py => utility/base.py} | 263 +----------------- nipype/interfaces/utility/csv.py | 99 +++++++ nipype/interfaces/utility/wrappers.py | 191 +++++++++++++ 4 files changed, 312 insertions(+), 256 deletions(-) create mode 100644 nipype/interfaces/utility/__init__.py rename nipype/interfaces/{utility.py => utility/base.py} (57%) create mode 100644 nipype/interfaces/utility/csv.py create mode 100644 nipype/interfaces/utility/wrappers.py diff --git a/nipype/interfaces/utility/__init__.py b/nipype/interfaces/utility/__init__.py new file mode 100644 index 0000000000..abe1eb92b0 --- /dev/null +++ b/nipype/interfaces/utility/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +""" +Package contains interfaces for using existing functionality in other packages + +Requires Packages to be installed +""" +from __future__ import print_function, division, unicode_literals, absolute_import +__docformat__ = 'restructuredtext' + +from .base import (IdentityInterface, Rename, Select, Split, Merge, + AssertEqual) +from .csv import CSVReader +from .wrappers import Function diff --git a/nipype/interfaces/utility.py b/nipype/interfaces/utility/base.py similarity index 57% rename from nipype/interfaces/utility.py rename to nipype/interfaces/utility/base.py index 7ef2d1bad3..91a1aca97b 100644 --- a/nipype/interfaces/utility.py +++ b/nipype/interfaces/utility/base.py @@ -10,7 +10,7 @@ >>> os.chdir(datadir) """ from __future__ import print_function, division, unicode_literals, absolute_import -from builtins import zip, range, str, open +from builtins import range from future import standard_library standard_library.install_aliases() @@ -20,22 +20,12 @@ import numpy as np import nibabel as nb -from nipype import logging -from .base import (traits, TraitedSpec, DynamicTraitedSpec, File, - Undefined, isdefined, OutputMultiPath, runtime_profile, - InputMultiPath, BaseInterface, BaseInterfaceInputSpec) -from .io import IOBase, add_traits -from ..utils.filemanip import (filename_to_list, copyfile, split_filename) -from ..utils.misc import getsource, create_function_from_source - -logger = logging.getLogger('interface') -if runtime_profile: - try: - import psutil - except ImportError as exc: - logger.info('Unable to import packages needed for runtime profiling. '\ - 'Turning off runtime profiler. Reason: %s' % exc) - runtime_profile = False +from ..base import (traits, TraitedSpec, DynamicTraitedSpec, File, + Undefined, isdefined, OutputMultiPath, InputMultiPath, + BaseInterface, BaseInterfaceInputSpec) +from ..io import IOBase, add_traits +from ...utils.filemanip import filename_to_list, copyfile, split_filename + class IdentityInterface(IOBase): """Basic interface class generates identity mappings @@ -357,165 +347,6 @@ def _list_outputs(self): return outputs -class FunctionInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): - function_str = traits.Str(mandatory=True, desc='code for function') - - -class Function(IOBase): - """Runs arbitrary function as an interface - - Examples - -------- - - >>> func = 'def func(arg1, arg2=5): return arg1 + arg2' - >>> fi = Function(input_names=['arg1', 'arg2'], output_names=['out']) - >>> fi.inputs.function_str = func - >>> res = fi.run(arg1=1) - >>> res.outputs.out - 6 - - """ - - input_spec = FunctionInputSpec - output_spec = DynamicTraitedSpec - - def __init__(self, input_names, output_names, function=None, imports=None, - **inputs): - """ - - Parameters - ---------- - - input_names: single str or list - names corresponding to function inputs - output_names: single str or list - names corresponding to function outputs. - has to match the number of outputs - function : callable - callable python object. must be able to execute in an - isolated namespace (possibly in concert with the ``imports`` - parameter) - imports : list of strings - list of import statements that allow the function to execute - in an otherwise empty namespace - """ - - super(Function, self).__init__(**inputs) - if function: - if hasattr(function, '__call__'): - try: - self.inputs.function_str = getsource(function) - except IOError: - raise Exception('Interface Function does not accept ' - 'function objects defined interactively ' - 'in a python session') - elif isinstance(function, (str, bytes)): - self.inputs.function_str = function - else: - raise Exception('Unknown type of function') - self.inputs.on_trait_change(self._set_function_string, - 'function_str') - self._input_names = filename_to_list(input_names) - self._output_names = filename_to_list(output_names) - add_traits(self.inputs, [name for name in self._input_names]) - self.imports = imports - self._out = {} - for name in self._output_names: - self._out[name] = None - - def _set_function_string(self, obj, name, old, new): - if name == 'function_str': - if hasattr(new, '__call__'): - function_source = getsource(new) - elif isinstance(new, (str, bytes)): - function_source = new - self.inputs.trait_set(trait_change_notify=False, - **{'%s' % name: function_source}) - - def _add_output_traits(self, base): - undefined_traits = {} - for key in self._output_names: - base.add_trait(key, traits.Any) - undefined_traits[key] = Undefined - base.trait_set(trait_change_notify=False, **undefined_traits) - return base - - def _run_interface(self, runtime): - # Get workflow logger for runtime profile error reporting - from nipype import logging - logger = logging.getLogger('workflow') - - # Create function handle - function_handle = create_function_from_source(self.inputs.function_str, - self.imports) - - # Wrapper for running function handle in multiprocessing.Process - # Can catch exceptions and report output via multiprocessing.Queue - def _function_handle_wrapper(queue, **kwargs): - try: - out = function_handle(**kwargs) - queue.put(out) - except Exception as exc: - queue.put(exc) - - # Get function args - args = {} - for name in self._input_names: - value = getattr(self.inputs, name) - if isdefined(value): - args[name] = value - - # Profile resources if set - if runtime_profile: - from nipype.interfaces.base import get_max_resources_used - import multiprocessing - # Init communication queue and proc objs - queue = multiprocessing.Queue() - proc = multiprocessing.Process(target=_function_handle_wrapper, - args=(queue,), kwargs=args) - - # Init memory and threads before profiling - mem_mb = 0 - num_threads = 0 - - # Start process and profile while it's alive - proc.start() - while proc.is_alive(): - mem_mb, num_threads = \ - get_max_resources_used(proc.pid, mem_mb, num_threads, - pyfunc=True) - - # Get result from process queue - out = queue.get() - # If it is an exception, raise it - if isinstance(out, Exception): - raise out - - # Function ran successfully, populate runtime stats - setattr(runtime, 'runtime_memory_gb', mem_mb / 1024.0) - setattr(runtime, 'runtime_threads', num_threads) - else: - out = function_handle(**args) - - if len(self._output_names) == 1: - self._out[self._output_names[0]] = out - else: - if isinstance(out, tuple) and (len(out) != len(self._output_names)): - raise RuntimeError('Mismatch in number of expected outputs') - - else: - for idx, name in enumerate(self._output_names): - self._out[name] = out[idx] - - return runtime - - def _list_outputs(self): - outputs = self._outputs().get() - for key in self._output_names: - outputs[key] = self._out[key] - return outputs - - class AssertEqualInputSpec(BaseInterfaceInputSpec): volume1 = File(exists=True, mandatory=True) volume2 = File(exists=True, mandatory=True) @@ -532,83 +363,3 @@ def _run_interface(self, runtime): if not np.all(data1 == data2): raise RuntimeError('Input images are not exactly equal') return runtime - - -class CSVReaderInputSpec(DynamicTraitedSpec, TraitedSpec): - in_file = File(exists=True, mandatory=True, desc='Input comma-seperated value (CSV) file') - header = traits.Bool(False, usedefault=True, desc='True if the first line is a column header') - - -class CSVReader(BaseInterface): - """ - Examples - -------- - - >>> reader = CSVReader() # doctest: +SKIP - >>> reader.inputs.in_file = 'noHeader.csv' # doctest: +SKIP - >>> out = reader.run() # doctest: +SKIP - >>> out.outputs.column_0 == ['foo', 'bar', 'baz'] # doctest: +SKIP - True - >>> out.outputs.column_1 == ['hello', 'world', 'goodbye'] # doctest: +SKIP - True - >>> out.outputs.column_2 == ['300.1', '5', '0.3'] # doctest: +SKIP - True - - >>> reader = CSVReader() # doctest: +SKIP - >>> reader.inputs.in_file = 'header.csv' # doctest: +SKIP - >>> reader.inputs.header = True # doctest: +SKIP - >>> out = reader.run() # doctest: +SKIP - >>> out.outputs.files == ['foo', 'bar', 'baz'] # doctest: +SKIP - True - >>> out.outputs.labels == ['hello', 'world', 'goodbye'] # doctest: +SKIP - True - >>> out.outputs.erosion == ['300.1', '5', '0.3'] # doctest: +SKIP - True - - """ - input_spec = CSVReaderInputSpec - output_spec = DynamicTraitedSpec - _always_run = True - - def _append_entry(self, outputs, entry): - for key, value in zip(self._outfields, entry): - outputs[key].append(value) - return outputs - - def _parse_line(self, line): - line = line.replace('\n', '') - entry = [x.strip() for x in line.split(',')] - return entry - - def _get_outfields(self): - with open(self.inputs.in_file, 'r') as fid: - entry = self._parse_line(fid.readline()) - if self.inputs.header: - self._outfields = tuple(entry) - else: - self._outfields = tuple(['column_' + str(x) for x in range(len(entry))]) - return self._outfields - - def _run_interface(self, runtime): - self._get_outfields() - return runtime - - def _outputs(self): - return self._add_output_traits(super(CSVReader, self)._outputs()) - - def _add_output_traits(self, base): - return add_traits(base, self._get_outfields()) - - def _list_outputs(self): - outputs = self.output_spec().get() - isHeader = True - for key in self._outfields: - outputs[key] = [] # initialize outfields - with open(self.inputs.in_file, 'r') as fid: - for line in fid.readlines(): - if self.inputs.header and isHeader: # skip header line - isHeader = False - continue - entry = self._parse_line(line) - outputs = self._append_entry(outputs, entry) - return outputs diff --git a/nipype/interfaces/utility/csv.py b/nipype/interfaces/utility/csv.py new file mode 100644 index 0000000000..05e3e6b16d --- /dev/null +++ b/nipype/interfaces/utility/csv.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""CSV Handling utilities + + Change directory to provide relative paths for doctests + >>> import os + >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) + >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) + >>> os.chdir(datadir) +""" +from __future__ import print_function, division, unicode_literals, absolute_import +from builtins import zip, range, str, open + +from future import standard_library +standard_library.install_aliases() + +from ..base import traits, TraitedSpec, DynamicTraitedSpec, File, BaseInterface +from ..io import add_traits + + +class CSVReaderInputSpec(DynamicTraitedSpec, TraitedSpec): + in_file = File(exists=True, mandatory=True, desc='Input comma-seperated value (CSV) file') + header = traits.Bool(False, usedefault=True, desc='True if the first line is a column header') + + +class CSVReader(BaseInterface): + """ + Examples + -------- + + >>> reader = CSVReader() # doctest: +SKIP + >>> reader.inputs.in_file = 'noHeader.csv' # doctest: +SKIP + >>> out = reader.run() # doctest: +SKIP + >>> out.outputs.column_0 == ['foo', 'bar', 'baz'] # doctest: +SKIP + True + >>> out.outputs.column_1 == ['hello', 'world', 'goodbye'] # doctest: +SKIP + True + >>> out.outputs.column_2 == ['300.1', '5', '0.3'] # doctest: +SKIP + True + + >>> reader = CSVReader() # doctest: +SKIP + >>> reader.inputs.in_file = 'header.csv' # doctest: +SKIP + >>> reader.inputs.header = True # doctest: +SKIP + >>> out = reader.run() # doctest: +SKIP + >>> out.outputs.files == ['foo', 'bar', 'baz'] # doctest: +SKIP + True + >>> out.outputs.labels == ['hello', 'world', 'goodbye'] # doctest: +SKIP + True + >>> out.outputs.erosion == ['300.1', '5', '0.3'] # doctest: +SKIP + True + + """ + input_spec = CSVReaderInputSpec + output_spec = DynamicTraitedSpec + _always_run = True + + def _append_entry(self, outputs, entry): + for key, value in zip(self._outfields, entry): + outputs[key].append(value) + return outputs + + def _parse_line(self, line): + line = line.replace('\n', '') + entry = [x.strip() for x in line.split(',')] + return entry + + def _get_outfields(self): + with open(self.inputs.in_file, 'r') as fid: + entry = self._parse_line(fid.readline()) + if self.inputs.header: + self._outfields = tuple(entry) + else: + self._outfields = tuple(['column_' + str(x) for x in range(len(entry))]) + return self._outfields + + def _run_interface(self, runtime): + self._get_outfields() + return runtime + + def _outputs(self): + return self._add_output_traits(super(CSVReader, self)._outputs()) + + def _add_output_traits(self, base): + return add_traits(base, self._get_outfields()) + + def _list_outputs(self): + outputs = self.output_spec().get() + isHeader = True + for key in self._outfields: + outputs[key] = [] # initialize outfields + with open(self.inputs.in_file, 'r') as fid: + for line in fid.readlines(): + if self.inputs.header and isHeader: # skip header line + isHeader = False + continue + entry = self._parse_line(line) + outputs = self._append_entry(outputs, entry) + return outputs diff --git a/nipype/interfaces/utility/wrappers.py b/nipype/interfaces/utility/wrappers.py new file mode 100644 index 0000000000..c11901fb24 --- /dev/null +++ b/nipype/interfaces/utility/wrappers.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""Various utilities + + Change directory to provide relative paths for doctests + >>> import os + >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) + >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) + >>> os.chdir(datadir) +""" +from __future__ import print_function, division, unicode_literals, absolute_import + +from future import standard_library +standard_library.install_aliases() + + +from nipype import logging +from ..base import (traits, DynamicTraitedSpec, Undefined, isdefined, runtime_profile, + BaseInterfaceInputSpec) +from ..io import IOBase, add_traits +from ...utils.filemanip import filename_to_list +from ...utils.misc import getsource, create_function_from_source + +logger = logging.getLogger('interface') +if runtime_profile: + try: + import psutil + except ImportError as exc: + logger.info('Unable to import packages needed for runtime profiling. '\ + 'Turning off runtime profiler. Reason: %s' % exc) + runtime_profile = False + + +class FunctionInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): + function_str = traits.Str(mandatory=True, desc='code for function') + + +class Function(IOBase): + """Runs arbitrary function as an interface + + Examples + -------- + + >>> func = 'def func(arg1, arg2=5): return arg1 + arg2' + >>> fi = Function(input_names=['arg1', 'arg2'], output_names=['out']) + >>> fi.inputs.function_str = func + >>> res = fi.run(arg1=1) + >>> res.outputs.out + 6 + + """ + + input_spec = FunctionInputSpec + output_spec = DynamicTraitedSpec + + def __init__(self, input_names, output_names, function=None, imports=None, + **inputs): + """ + + Parameters + ---------- + + input_names: single str or list + names corresponding to function inputs + output_names: single str or list + names corresponding to function outputs. + has to match the number of outputs + function : callable + callable python object. must be able to execute in an + isolated namespace (possibly in concert with the ``imports`` + parameter) + imports : list of strings + list of import statements that allow the function to execute + in an otherwise empty namespace + """ + + super(Function, self).__init__(**inputs) + if function: + if hasattr(function, '__call__'): + try: + self.inputs.function_str = getsource(function) + except IOError: + raise Exception('Interface Function does not accept ' + 'function objects defined interactively ' + 'in a python session') + elif isinstance(function, (str, bytes)): + self.inputs.function_str = function + else: + raise Exception('Unknown type of function') + self.inputs.on_trait_change(self._set_function_string, + 'function_str') + self._input_names = filename_to_list(input_names) + self._output_names = filename_to_list(output_names) + add_traits(self.inputs, [name for name in self._input_names]) + self.imports = imports + self._out = {} + for name in self._output_names: + self._out[name] = None + + def _set_function_string(self, obj, name, old, new): + if name == 'function_str': + if hasattr(new, '__call__'): + function_source = getsource(new) + elif isinstance(new, (str, bytes)): + function_source = new + self.inputs.trait_set(trait_change_notify=False, + **{'%s' % name: function_source}) + + def _add_output_traits(self, base): + undefined_traits = {} + for key in self._output_names: + base.add_trait(key, traits.Any) + undefined_traits[key] = Undefined + base.trait_set(trait_change_notify=False, **undefined_traits) + return base + + def _run_interface(self, runtime): + # Get workflow logger for runtime profile error reporting + from nipype import logging + logger = logging.getLogger('workflow') + + # Create function handle + function_handle = create_function_from_source(self.inputs.function_str, + self.imports) + + # Wrapper for running function handle in multiprocessing.Process + # Can catch exceptions and report output via multiprocessing.Queue + def _function_handle_wrapper(queue, **kwargs): + try: + out = function_handle(**kwargs) + queue.put(out) + except Exception as exc: + queue.put(exc) + + # Get function args + args = {} + for name in self._input_names: + value = getattr(self.inputs, name) + if isdefined(value): + args[name] = value + + # Profile resources if set + if runtime_profile: + from nipype.interfaces.base import get_max_resources_used + import multiprocessing + # Init communication queue and proc objs + queue = multiprocessing.Queue() + proc = multiprocessing.Process(target=_function_handle_wrapper, + args=(queue,), kwargs=args) + + # Init memory and threads before profiling + mem_mb = 0 + num_threads = 0 + + # Start process and profile while it's alive + proc.start() + while proc.is_alive(): + mem_mb, num_threads = \ + get_max_resources_used(proc.pid, mem_mb, num_threads, + pyfunc=True) + + # Get result from process queue + out = queue.get() + # If it is an exception, raise it + if isinstance(out, Exception): + raise out + + # Function ran successfully, populate runtime stats + setattr(runtime, 'runtime_memory_gb', mem_mb / 1024.0) + setattr(runtime, 'runtime_threads', num_threads) + else: + out = function_handle(**args) + + if len(self._output_names) == 1: + self._out[self._output_names[0]] = out + else: + if isinstance(out, tuple) and (len(out) != len(self._output_names)): + raise RuntimeError('Mismatch in number of expected outputs') + + else: + for idx, name in enumerate(self._output_names): + self._out[name] = out[idx] + + return runtime + + def _list_outputs(self): + outputs = self._outputs().get() + for key in self._output_names: + outputs[key] = self._out[key] + return outputs From 1880079df4b9837892c2848113f8fe3b153ff13e Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 09:35:52 -0800 Subject: [PATCH 02/10] refactoring tests --- nipype/interfaces/utility/__init__.py | 2 - nipype/interfaces/utility/tests/test_base.py | 51 ++++++++++++++ nipype/interfaces/utility/tests/test_csv.py | 32 +++++++++ .../tests/test_wrappers.py} | 70 +------------------ 4 files changed, 84 insertions(+), 71 deletions(-) create mode 100644 nipype/interfaces/utility/tests/test_base.py create mode 100644 nipype/interfaces/utility/tests/test_csv.py rename nipype/interfaces/{tests/test_utility.py => utility/tests/test_wrappers.py} (60%) diff --git a/nipype/interfaces/utility/__init__.py b/nipype/interfaces/utility/__init__.py index abe1eb92b0..084acb569c 100644 --- a/nipype/interfaces/utility/__init__.py +++ b/nipype/interfaces/utility/__init__.py @@ -6,8 +6,6 @@ Requires Packages to be installed """ -from __future__ import print_function, division, unicode_literals, absolute_import -__docformat__ = 'restructuredtext' from .base import (IdentityInterface, Rename, Select, Split, Merge, AssertEqual) diff --git a/nipype/interfaces/utility/tests/test_base.py b/nipype/interfaces/utility/tests/test_base.py new file mode 100644 index 0000000000..645952d026 --- /dev/null +++ b/nipype/interfaces/utility/tests/test_base.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import print_function, unicode_literals +import os +import pytest + +from nipype.interfaces import utility +import nipype.pipeline.engine as pe + + +def test_rename(tmpdir): + os.chdir(str(tmpdir)) + + # Test very simple rename + _ = open("file.txt", "w").close() + rn = utility.Rename(in_file="file.txt", format_string="test_file1.txt") + res = rn.run() + outfile = str(tmpdir.join("test_file1.txt")) + assert res.outputs.out_file == outfile + assert os.path.exists(outfile) + + # Now a string-formatting version + rn = utility.Rename(in_file="file.txt", format_string="%(field1)s_file%(field2)d", keep_ext=True) + # Test .input field creation + assert hasattr(rn.inputs, "field1") + assert hasattr(rn.inputs, "field2") + + # Set the inputs + rn.inputs.field1 = "test" + rn.inputs.field2 = 2 + res = rn.run() + outfile = str(tmpdir.join("test_file2.txt")) + assert res.outputs.out_file == outfile + assert os.path.exists(outfile) + + +@pytest.mark.parametrize("args, expected", [ + ({} , ([0], [1,2,3])), + ({"squeeze" : True}, (0 , [1,2,3])) + ]) +def test_split(tmpdir, args, expected): + os.chdir(str(tmpdir)) + + node = pe.Node(utility.Split(inlist=list(range(4)), + splits=[1, 3], + **args), + name='split_squeeze') + res = node.run() + assert res.outputs.out1 == expected[0] + assert res.outputs.out2 == expected[1] diff --git a/nipype/interfaces/utility/tests/test_csv.py b/nipype/interfaces/utility/tests/test_csv.py new file mode 100644 index 0000000000..86ac95a371 --- /dev/null +++ b/nipype/interfaces/utility/tests/test_csv.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import print_function, unicode_literals + +from nipype.interfaces import utility + + +def test_csvReader(tmpdir): + header = "files,labels,erosion\n" + lines = ["foo,hello,300.1\n", + "bar,world,5\n", + "baz,goodbye,0.3\n"] + for x in range(2): + name = str(tmpdir.join("testfile.csv")) + with open(name, 'w') as fid: + reader = utility.CSVReader() + if x % 2 == 0: + fid.write(header) + reader.inputs.header = True + fid.writelines(lines) + fid.flush() + reader.inputs.in_file = name + out = reader.run() + if x % 2 == 0: + assert out.outputs.files == ['foo', 'bar', 'baz'] + assert out.outputs.labels == ['hello', 'world', 'goodbye'] + assert out.outputs.erosion == ['300.1', '5', '0.3'] + else: + assert out.outputs.column_0 == ['foo', 'bar', 'baz'] + assert out.outputs.column_1 == ['hello', 'world', 'goodbye'] + assert out.outputs.column_2 == ['300.1', '5', '0.3'] diff --git a/nipype/interfaces/tests/test_utility.py b/nipype/interfaces/utility/tests/test_wrappers.py similarity index 60% rename from nipype/interfaces/tests/test_utility.py rename to nipype/interfaces/utility/tests/test_wrappers.py index 77fa276e1c..d9f1da255a 100644 --- a/nipype/interfaces/tests/test_utility.py +++ b/nipype/interfaces/utility/tests/test_wrappers.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import print_function, unicode_literals # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import print_function, unicode_literals import os import pytest @@ -9,32 +9,6 @@ import nipype.pipeline.engine as pe -def test_rename(tmpdir): - os.chdir(str(tmpdir)) - - # Test very simple rename - _ = open("file.txt", "w").close() - rn = utility.Rename(in_file="file.txt", format_string="test_file1.txt") - res = rn.run() - outfile = str(tmpdir.join("test_file1.txt")) - assert res.outputs.out_file == outfile - assert os.path.exists(outfile) - - # Now a string-formatting version - rn = utility.Rename(in_file="file.txt", format_string="%(field1)s_file%(field2)d", keep_ext=True) - # Test .input field creation - assert hasattr(rn.inputs, "field1") - assert hasattr(rn.inputs, "field2") - - # Set the inputs - rn.inputs.field1 = "test" - rn.inputs.field2 = 2 - res = rn.run() - outfile = str(tmpdir.join("test_file2.txt")) - assert res.outputs.out_file == outfile - assert os.path.exists(outfile) - - def test_function(tmpdir): os.chdir(str(tmpdir)) @@ -89,48 +63,6 @@ def test_function_with_imports(tmpdir): node.run() -@pytest.mark.parametrize("args, expected", [ - ({} , ([0], [1,2,3])), - ({"squeeze" : True}, (0 , [1,2,3])) - ]) -def test_split(tmpdir, args, expected): - os.chdir(str(tmpdir)) - - node = pe.Node(utility.Split(inlist=list(range(4)), - splits=[1, 3], - **args), - name='split_squeeze') - res = node.run() - assert res.outputs.out1 == expected[0] - assert res.outputs.out2 == expected[1] - - -def test_csvReader(tmpdir): - header = "files,labels,erosion\n" - lines = ["foo,hello,300.1\n", - "bar,world,5\n", - "baz,goodbye,0.3\n"] - for x in range(2): - name = str(tmpdir.join("testfile.csv")) - with open(name, 'w') as fid: - reader = utility.CSVReader() - if x % 2 == 0: - fid.write(header) - reader.inputs.header = True - fid.writelines(lines) - fid.flush() - reader.inputs.in_file = name - out = reader.run() - if x % 2 == 0: - assert out.outputs.files == ['foo', 'bar', 'baz'] - assert out.outputs.labels == ['hello', 'world', 'goodbye'] - assert out.outputs.erosion == ['300.1', '5', '0.3'] - else: - assert out.outputs.column_0 == ['foo', 'bar', 'baz'] - assert out.outputs.column_1 == ['hello', 'world', 'goodbye'] - assert out.outputs.column_2 == ['300.1', '5', '0.3'] - - def test_aux_connect_function(tmpdir): """ This tests excution nodes with multiple inputs and auxiliary function inside the Workflow connect function. From 6c7436e402a7368c8fd231c23b305603ebe6e97e Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 09:41:17 -0800 Subject: [PATCH 03/10] update specs --- nipype/interfaces/{ => utility}/tests/test_auto_AssertEqual.py | 2 +- nipype/interfaces/{ => utility}/tests/test_auto_CSVReader.py | 2 +- nipype/interfaces/{ => utility}/tests/test_auto_Function.py | 2 +- .../{ => utility}/tests/test_auto_IdentityInterface.py | 2 +- nipype/interfaces/{ => utility}/tests/test_auto_Merge.py | 2 +- nipype/interfaces/{ => utility}/tests/test_auto_Rename.py | 2 +- nipype/interfaces/{ => utility}/tests/test_auto_Select.py | 2 +- nipype/interfaces/{ => utility}/tests/test_auto_Split.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename nipype/interfaces/{ => utility}/tests/test_auto_AssertEqual.py (93%) rename nipype/interfaces/{ => utility}/tests/test_auto_CSVReader.py (95%) rename nipype/interfaces/{ => utility}/tests/test_auto_Function.py (95%) rename nipype/interfaces/{ => utility}/tests/test_auto_IdentityInterface.py (93%) rename nipype/interfaces/{ => utility}/tests/test_auto_Merge.py (96%) rename nipype/interfaces/{ => utility}/tests/test_auto_Rename.py (96%) rename nipype/interfaces/{ => utility}/tests/test_auto_Select.py (96%) rename nipype/interfaces/{ => utility}/tests/test_auto_Split.py (96%) diff --git a/nipype/interfaces/tests/test_auto_AssertEqual.py b/nipype/interfaces/utility/tests/test_auto_AssertEqual.py similarity index 93% rename from nipype/interfaces/tests/test_auto_AssertEqual.py rename to nipype/interfaces/utility/tests/test_auto_AssertEqual.py index 4bfd775566..e59d5eb02b 100644 --- a/nipype/interfaces/tests/test_auto_AssertEqual.py +++ b/nipype/interfaces/utility/tests/test_auto_AssertEqual.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import AssertEqual +from ..base import AssertEqual def test_AssertEqual_inputs(): diff --git a/nipype/interfaces/tests/test_auto_CSVReader.py b/nipype/interfaces/utility/tests/test_auto_CSVReader.py similarity index 95% rename from nipype/interfaces/tests/test_auto_CSVReader.py rename to nipype/interfaces/utility/tests/test_auto_CSVReader.py index 7e2862947c..b2f47eb3be 100644 --- a/nipype/interfaces/tests/test_auto_CSVReader.py +++ b/nipype/interfaces/utility/tests/test_auto_CSVReader.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import CSVReader +from ..csv import CSVReader def test_CSVReader_inputs(): diff --git a/nipype/interfaces/tests/test_auto_Function.py b/nipype/interfaces/utility/tests/test_auto_Function.py similarity index 95% rename from nipype/interfaces/tests/test_auto_Function.py rename to nipype/interfaces/utility/tests/test_auto_Function.py index 1e7b395aaa..580002713a 100644 --- a/nipype/interfaces/tests/test_auto_Function.py +++ b/nipype/interfaces/utility/tests/test_auto_Function.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import Function +from ..wrappers import Function def test_Function_inputs(): diff --git a/nipype/interfaces/tests/test_auto_IdentityInterface.py b/nipype/interfaces/utility/tests/test_auto_IdentityInterface.py similarity index 93% rename from nipype/interfaces/tests/test_auto_IdentityInterface.py rename to nipype/interfaces/utility/tests/test_auto_IdentityInterface.py index 214042c365..7adb95ee88 100644 --- a/nipype/interfaces/tests/test_auto_IdentityInterface.py +++ b/nipype/interfaces/utility/tests/test_auto_IdentityInterface.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import IdentityInterface +from ..base import IdentityInterface def test_IdentityInterface_inputs(): diff --git a/nipype/interfaces/tests/test_auto_Merge.py b/nipype/interfaces/utility/tests/test_auto_Merge.py similarity index 96% rename from nipype/interfaces/tests/test_auto_Merge.py rename to nipype/interfaces/utility/tests/test_auto_Merge.py index b2cda077da..d564935e52 100644 --- a/nipype/interfaces/tests/test_auto_Merge.py +++ b/nipype/interfaces/utility/tests/test_auto_Merge.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import Merge +from ..base import Merge def test_Merge_inputs(): diff --git a/nipype/interfaces/tests/test_auto_Rename.py b/nipype/interfaces/utility/tests/test_auto_Rename.py similarity index 96% rename from nipype/interfaces/tests/test_auto_Rename.py rename to nipype/interfaces/utility/tests/test_auto_Rename.py index 8c7725dd36..bf991aefbf 100644 --- a/nipype/interfaces/tests/test_auto_Rename.py +++ b/nipype/interfaces/utility/tests/test_auto_Rename.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import Rename +from ..base import Rename def test_Rename_inputs(): diff --git a/nipype/interfaces/tests/test_auto_Select.py b/nipype/interfaces/utility/tests/test_auto_Select.py similarity index 96% rename from nipype/interfaces/tests/test_auto_Select.py rename to nipype/interfaces/utility/tests/test_auto_Select.py index 0b8701e999..a8031df162 100644 --- a/nipype/interfaces/tests/test_auto_Select.py +++ b/nipype/interfaces/utility/tests/test_auto_Select.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import Select +from ..base import Select def test_Select_inputs(): diff --git a/nipype/interfaces/tests/test_auto_Split.py b/nipype/interfaces/utility/tests/test_auto_Split.py similarity index 96% rename from nipype/interfaces/tests/test_auto_Split.py rename to nipype/interfaces/utility/tests/test_auto_Split.py index 79d995cfbd..0c6232f920 100644 --- a/nipype/interfaces/tests/test_auto_Split.py +++ b/nipype/interfaces/utility/tests/test_auto_Split.py @@ -1,5 +1,5 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT -from ..utility import Split +from ..base import Split def test_Split_inputs(): From d35db933a89b69dfefea1bc4596c8fa46f1f68f6 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 10:11:21 -0800 Subject: [PATCH 04/10] fix doctests --- nipype/interfaces/utility/base.py | 15 ++++++++++----- nipype/interfaces/utility/csv.py | 12 ++++++++---- nipype/interfaces/utility/wrappers.py | 12 ++++++++---- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/nipype/interfaces/utility/base.py b/nipype/interfaces/utility/base.py index 91a1aca97b..fdc0784538 100644 --- a/nipype/interfaces/utility/base.py +++ b/nipype/interfaces/utility/base.py @@ -1,13 +1,18 @@ # -*- coding: utf-8 -*- # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Various utilities +""" +Various utilities Change directory to provide relative paths for doctests - >>> import os - >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) - >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) - >>> os.chdir(datadir) + + .. testsetup:: + import os + filepath = os.path.dirname( os.path.realpath( __file__ ) ) + datadir = os.path.realpath(os.path.join(filepath, '../../testing/data')) + os.chdir(datadir) + + """ from __future__ import print_function, division, unicode_literals, absolute_import from builtins import range diff --git a/nipype/interfaces/utility/csv.py b/nipype/interfaces/utility/csv.py index 05e3e6b16d..0f6419b55a 100644 --- a/nipype/interfaces/utility/csv.py +++ b/nipype/interfaces/utility/csv.py @@ -4,10 +4,14 @@ """CSV Handling utilities Change directory to provide relative paths for doctests - >>> import os - >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) - >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) - >>> os.chdir(datadir) + + .. testsetup:: + import os + filepath = os.path.dirname( os.path.realpath( __file__ ) ) + datadir = os.path.realpath(os.path.join(filepath, '../../testing/data')) + os.chdir(datadir) + + """ from __future__ import print_function, division, unicode_literals, absolute_import from builtins import zip, range, str, open diff --git a/nipype/interfaces/utility/wrappers.py b/nipype/interfaces/utility/wrappers.py index c11901fb24..542ee16001 100644 --- a/nipype/interfaces/utility/wrappers.py +++ b/nipype/interfaces/utility/wrappers.py @@ -4,10 +4,14 @@ """Various utilities Change directory to provide relative paths for doctests - >>> import os - >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) - >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) - >>> os.chdir(datadir) + + .. testsetup:: + import os + filepath = os.path.dirname( os.path.realpath( __file__ ) ) + datadir = os.path.realpath(os.path.join(filepath, '../../testing/data')) + os.chdir(datadir) + + """ from __future__ import print_function, division, unicode_literals, absolute_import From 6b6a1a4c3445e2dc81f17fb015c3b8a1482e38fb Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 10:17:18 -0800 Subject: [PATCH 05/10] update CHANGES --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 995e324966..8c136b5fed 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ Upcoming release 0.13 ===================== +* ENH: Refactoring of nipype.interfaces.utility (https://github.com/nipy/nipype/pull/1828) * FIX: Issues in Docker image permissions, and docker documentation (https://github.com/nipy/nipype/pull/1825) * ENH: Revised all Dockerfiles and automated deployment to Docker Hub from CircleCI (https://github.com/nipy/nipype/pull/1815) From b7b25b9f60e07f1bf666c7c76d521b28bcf5b85d Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 10:48:48 -0800 Subject: [PATCH 06/10] add __init__ to new utility module tests --- nipype/interfaces/utility/tests/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 nipype/interfaces/utility/tests/__init__.py diff --git a/nipype/interfaces/utility/tests/__init__.py b/nipype/interfaces/utility/tests/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/nipype/interfaces/utility/tests/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- From 06f8f2a7368372c4eeca92e14bf00a475017981d Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 11:36:23 -0800 Subject: [PATCH 07/10] fix #1829 --- .../interfaces/freesurfer/tests/test_utils.py | 5 ++--- nipype/testing/fixtures.py | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/nipype/interfaces/freesurfer/tests/test_utils.py b/nipype/interfaces/freesurfer/tests/test_utils.py index 0be4b34ec7..eff2946000 100644 --- a/nipype/interfaces/freesurfer/tests/test_utils.py +++ b/nipype/interfaces/freesurfer/tests/test_utils.py @@ -3,10 +3,9 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: from __future__ import print_function, division, unicode_literals, absolute_import from builtins import open -import os - +import os, os.path as op import pytest -from nipype.testing.fixtures import (create_files_in_directory_plus_dummy_file, +from nipype.testing.fixtures import (create_files_in_directory_plus_dummy_file, create_surf_file_in_directory) from nipype.interfaces.base import TraitError diff --git a/nipype/testing/fixtures.py b/nipype/testing/fixtures.py index e19981f487..30f71b7824 100644 --- a/nipype/testing/fixtures.py +++ b/nipype/testing/fixtures.py @@ -5,12 +5,17 @@ """ Pytest fixtures used in tests. """ +from __future__ import print_function, division, unicode_literals, absolute_import + import os import pytest import numpy as np - import nibabel as nb + +from io import open +from builtins import str + from nipype.interfaces.fsl import Info from nipype.interfaces.fsl.base import FSLCommand @@ -25,12 +30,13 @@ def analyze_pair_image_files(outdir, filelist, shape): def nifti_image_files(outdir, filelist, shape): + if not isinstance(filelist, (list, tuple)): + filelist = [filelist] + for f in filelist: - hdr = nb.Nifti1Header() - hdr.set_data_shape(shape) img = np.random.random(shape) - nb.save(nb.Nifti1Image(img, np.eye(4), hdr), - os.path.join(outdir, f)) + nb.Nifti1Image(img, np.eye(4), None).to_filename( + os.path.join(outdir, f)) @pytest.fixture() @@ -88,7 +94,7 @@ def create_surf_file_in_directory(request, tmpdir): cwd = os.getcwd() os.chdir(outdir) surf = 'lh.a.nii' - nifti_image_files(outdir, filelist=surf, shape=(1,100,1)) + nifti_image_files(outdir, filelist=surf, shape=(1, 100, 1)) def change_directory(): os.chdir(cwd) From ef3407a660246f9f0efb15df7b7b10a020371d99 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 15:12:33 -0800 Subject: [PATCH 08/10] use filename_to_list from filemanip --- nipype/testing/fixtures.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nipype/testing/fixtures.py b/nipype/testing/fixtures.py index 30f71b7824..8bdffa2f65 100644 --- a/nipype/testing/fixtures.py +++ b/nipype/testing/fixtures.py @@ -16,6 +16,7 @@ from io import open from builtins import str +from nipype.utils.filemanip import filename_to_list from nipype.interfaces.fsl import Info from nipype.interfaces.fsl.base import FSLCommand @@ -30,10 +31,7 @@ def analyze_pair_image_files(outdir, filelist, shape): def nifti_image_files(outdir, filelist, shape): - if not isinstance(filelist, (list, tuple)): - filelist = [filelist] - - for f in filelist: + for f in filename_to_list(filelist): img = np.random.random(shape) nb.Nifti1Image(img, np.eye(4), None).to_filename( os.path.join(outdir, f)) From f923bd565bab2ac67aece92dcfba82390a0a3fe4 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 15:15:49 -0800 Subject: [PATCH 09/10] use filename_to_list from filemanip in analyze_pair_image_files --- nipype/testing/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/testing/fixtures.py b/nipype/testing/fixtures.py index 8bdffa2f65..2a405742f7 100644 --- a/nipype/testing/fixtures.py +++ b/nipype/testing/fixtures.py @@ -22,7 +22,7 @@ def analyze_pair_image_files(outdir, filelist, shape): - for f in filelist: + for f in filename_to_list(filelist): hdr = nb.Nifti1Header() hdr.set_data_shape(shape) img = np.random.random(shape) From 9baa190f964281cdb7ac594384547a9a030c4494 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 21 Feb 2017 16:28:12 -0800 Subject: [PATCH 10/10] roll back testsetup blocks, some encoding fixes --- nipype/interfaces/utility/base.py | 26 ++++++++++++-------------- nipype/interfaces/utility/csv.py | 14 ++++++-------- nipype/interfaces/utility/wrappers.py | 14 +++++++------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/nipype/interfaces/utility/base.py b/nipype/interfaces/utility/base.py index fdc0784538..660a6e93a0 100644 --- a/nipype/interfaces/utility/base.py +++ b/nipype/interfaces/utility/base.py @@ -4,14 +4,12 @@ """ Various utilities - Change directory to provide relative paths for doctests - - .. testsetup:: - import os - filepath = os.path.dirname( os.path.realpath( __file__ ) ) - datadir = os.path.realpath(os.path.join(filepath, '../../testing/data')) - os.chdir(datadir) - + Change directory to provide relative paths for doctests + >>> import os + >>> filepath = os.path.dirname(os.path.realpath(__file__)) + >>> datadir = os.path.realpath(os.path.join(filepath, + ... '../../testing/data')) + >>> os.chdir(datadir) """ from __future__ import print_function, division, unicode_literals, absolute_import @@ -27,7 +25,7 @@ from ..base import (traits, TraitedSpec, DynamicTraitedSpec, File, Undefined, isdefined, OutputMultiPath, InputMultiPath, - BaseInterface, BaseInterfaceInputSpec) + BaseInterface, BaseInterfaceInputSpec, Str) from ..io import IOBase, add_traits from ...utils.filemanip import filename_to_list, copyfile, split_filename @@ -158,11 +156,10 @@ class RenameInputSpec(DynamicTraitedSpec): in_file = File(exists=True, mandatory=True, desc="file to rename") keep_ext = traits.Bool(desc=("Keep in_file extension, replace " "non-extension component of name")) - format_string = traits.String(mandatory=True, - desc=("Python formatting string for output " - "template")) - parse_string = traits.String(desc=("Python regexp parse string to define " - "replacement inputs")) + format_string = Str(mandatory=True, + desc="Python formatting string for output template") + parse_string = Str(desc="Python regexp parse string to define " + "replacement inputs") use_fullpath = traits.Bool(False, usedefault=True, desc="Use full path as input to regex parser") @@ -186,6 +183,7 @@ class Rename(IOBase): Examples -------- + >>> from nipype.interfaces.utility import Rename >>> rename1 = Rename() >>> rename1.inputs.in_file = "zstat1.nii.gz" diff --git a/nipype/interfaces/utility/csv.py b/nipype/interfaces/utility/csv.py index 0f6419b55a..0529a184a6 100644 --- a/nipype/interfaces/utility/csv.py +++ b/nipype/interfaces/utility/csv.py @@ -3,14 +3,12 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """CSV Handling utilities - Change directory to provide relative paths for doctests - - .. testsetup:: - import os - filepath = os.path.dirname( os.path.realpath( __file__ ) ) - datadir = os.path.realpath(os.path.join(filepath, '../../testing/data')) - os.chdir(datadir) - + Change directory to provide relative paths for doctests + >>> import os + >>> filepath = os.path.dirname(os.path.realpath(__file__)) + >>> datadir = os.path.realpath(os.path.join(filepath, + ... '../../testing/data')) + >>> os.chdir(datadir) """ from __future__ import print_function, division, unicode_literals, absolute_import diff --git a/nipype/interfaces/utility/wrappers.py b/nipype/interfaces/utility/wrappers.py index 542ee16001..b8e78a56e6 100644 --- a/nipype/interfaces/utility/wrappers.py +++ b/nipype/interfaces/utility/wrappers.py @@ -3,13 +3,12 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """Various utilities - Change directory to provide relative paths for doctests - - .. testsetup:: - import os - filepath = os.path.dirname( os.path.realpath( __file__ ) ) - datadir = os.path.realpath(os.path.join(filepath, '../../testing/data')) - os.chdir(datadir) + Change directory to provide relative paths for doctests + >>> import os + >>> filepath = os.path.dirname(os.path.realpath(__file__)) + >>> datadir = os.path.realpath(os.path.join(filepath, + ... '../../testing/data')) + >>> os.chdir(datadir) """ @@ -18,6 +17,7 @@ from future import standard_library standard_library.install_aliases() +from builtins import str, bytes from nipype import logging from ..base import (traits, DynamicTraitedSpec, Undefined, isdefined, runtime_profile,