From d7fdb6f3ab17b8b968b1ecf2d6896f35fbc5cbbd Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 7 Nov 2016 22:55:19 -0500 Subject: [PATCH 01/33] fix: mapnode debug error - closes #1690 --- nipype/pipeline/engine/nodes.py | 1 - nipype/pipeline/engine/tests/test_utils.py | 26 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 6391b2ac5a..ee73a93f79 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -1129,7 +1129,6 @@ def _node_runner(self, nodes, updatehash=False): node.run(updatehash=updatehash) except Exception as err: if str2bool(self.config['execution']['stop_on_first_crash']): - self._result = node.result raise finally: yield i, node, err diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 4d37beef4d..b3811b17a7 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -333,6 +333,7 @@ def test_multi_disconnected_iterable(): yield assert_equal, len(eg.nodes()), 60 rmtree(out_dir) + def test_provenance(): out_dir = mkdtemp() metawf = pe.Workflow(name='meta') @@ -345,3 +346,28 @@ def test_provenance(): yield assert_equal, len(psg.bundles), 2 yield assert_equal, len(psg.get_records()), 7 rmtree(out_dir) + + +def test_mapnode_crash(): + def myfunction(string): + return string + 'meh' + node = pe.MapNode(niu.Function(input_names=['WRONG'], + output_names=['newstring'], + function=myfunction), + iterfield=['WRONG'], + name='myfunc') + + node.inputs.WRONG = ['string' + str(i) for i in range(3)] + node.config = deepcopy(config._sections) + node.config['execution']['stop_on_first_crash'] = True + cwd = os.getcwd() + node.base_dir = mkdtemp() + + error_raised = False + try: + node.run() + except TypeError as e: + error_raised = True + os.chdir(cwd) + rmtree(node.base_dir) + yield assert_true, error_raised From fcbf5a1dea116235c267cffdea8844b84daee0db Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 7 Nov 2016 22:55:46 -0500 Subject: [PATCH 02/33] ref: change sleep poll duration back to 2 seconds --- doc/users/config_file.rst | 2 +- nipype/utils/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/users/config_file.rst b/doc/users/config_file.rst index 727c73f4cd..82cfc29871 100644 --- a/doc/users/config_file.rst +++ b/doc/users/config_file.rst @@ -134,7 +134,7 @@ Execution *poll_sleep_duration* This controls how long the job submission loop will sleep between submitting all pending jobs and checking for job completion. To be nice to cluster - schedulers the default is set to 60 seconds. + schedulers the default is set to 2 seconds. *xvfb_max_wait* Maximum time (in seconds) to wait for Xvfb to start, if the _redirect_x parameter of an Interface is True. diff --git a/nipype/utils/config.py b/nipype/utils/config.py index d55515c5ec..3bbeb22323 100644 --- a/nipype/utils/config.py +++ b/nipype/utils/config.py @@ -58,7 +58,7 @@ stop_on_unknown_version = false write_provenance = false parameterize_dirs = true -poll_sleep_duration = 60 +poll_sleep_duration = 2 xvfb_max_wait = 10 profile_runtime = false From 78afc2a97462984bcca04f5af29458dfe3769618 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 7 Nov 2016 22:55:57 -0500 Subject: [PATCH 03/33] fix bru2 doctest error --- nipype/interfaces/bru2nii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/bru2nii.py b/nipype/interfaces/bru2nii.py index 481aefb9ec..33a5551a83 100644 --- a/nipype/interfaces/bru2nii.py +++ b/nipype/interfaces/bru2nii.py @@ -43,7 +43,7 @@ class Bru2(CommandLine): >>> converter = Bru2() >>> converter.inputs.input_dir = "brukerdir" >>> converter.cmdline # doctest: +ELLIPSIS +IGNORE_UNICODE - 'Bru2 -o .../nipype/nipype/testing/data/brukerdir brukerdir' + 'Bru2 -o .../nipype/testing/data/brukerdir brukerdir' """ input_spec = Bru2InputSpec output_spec = Bru2OutputSpec From 75b38cb65b423a5dba6869e7dd3ada2dc731e2ee Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 13 Dec 2016 15:40:57 -0500 Subject: [PATCH 04/33] added error condition --- nipype/pipeline/engine/nodes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index ee73a93f79..574444f14f 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -522,7 +522,8 @@ def _load_resultfile(self, cwd): # Was this pickle created with Python 2.x? pickle.load(pkl_file, fix_imports=True, encoding='utf-8') logger.warn('Successfully loaded pickle in compatibility mode') - except (traits.TraitError, AttributeError, ImportError) as err: + except (traits.TraitError, AttributeError, ImportError, + EOFError) as err: if isinstance(err, (AttributeError, ImportError)): attribute_error = True logger.debug('attribute error: %s probably using ' From bb24c596f958f472ec88fd65e28c477d86a63749 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 14 Dec 2016 22:55:28 -0500 Subject: [PATCH 05/33] restoring 2 tests that had only import statement, changing comments --- nipype/interfaces/fsl/tests/test_BEDPOSTX.py | 2 ++ nipype/interfaces/fsl/tests/test_XFibres.py | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 nipype/interfaces/fsl/tests/test_BEDPOSTX.py create mode 100644 nipype/interfaces/fsl/tests/test_XFibres.py diff --git a/nipype/interfaces/fsl/tests/test_BEDPOSTX.py b/nipype/interfaces/fsl/tests/test_BEDPOSTX.py new file mode 100644 index 0000000000..c6ccb51c53 --- /dev/null +++ b/nipype/interfaces/fsl/tests/test_BEDPOSTX.py @@ -0,0 +1,2 @@ +# to ensure that the autotest is not written +from nipype.interfaces.fsl.dti import BEDPOSTX diff --git a/nipype/interfaces/fsl/tests/test_XFibres.py b/nipype/interfaces/fsl/tests/test_XFibres.py new file mode 100644 index 0000000000..032a27b986 --- /dev/null +++ b/nipype/interfaces/fsl/tests/test_XFibres.py @@ -0,0 +1,2 @@ +# to ensure that the autotest is not written +from nipype.interfaces.fsl.dti import XFibres From 9fb77301b0f029cc60d681a89c4583a4074f9a76 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 14 Dec 2016 23:38:55 -0500 Subject: [PATCH 06/33] fixing one assert in test_math --- nipype/interfaces/fsl/tests/test_maths.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nipype/interfaces/fsl/tests/test_maths.py b/nipype/interfaces/fsl/tests/test_maths.py index a94098df2c..40ba11b893 100644 --- a/nipype/interfaces/fsl/tests/test_maths.py +++ b/nipype/interfaces/fsl/tests/test_maths.py @@ -207,10 +207,8 @@ def test_stdimage(create_files_in_directory): assert stder.cmdline == cmdline%dim # Test the auto naming - stder = fsl.StdImage(in_file="a.nii") - #NOTE_dj, FAIL: this is failing (even the original version of the test with pytest) - #NOTE_dj: not sure if this should pass, it uses cmdline from interface.base.CommandLine - #assert stder.cmdline == "fslmaths a.nii -Tstd %s"%os.path.join(testdir, "a_std.nii") + stder = fsl.StdImage(in_file="a.nii", output_type='NIFTI') + assert stder.cmdline == "fslmaths a.nii -Tstd %s"%os.path.join(testdir, "a_std.nii") @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") From b261208a6ee77d5b5e377f0a403d54d2da3901ba Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 09:36:52 -0500 Subject: [PATCH 07/33] removing one test from interfaces/tests/test_base.py; it was repeating the same assert as the previous one --- nipype/interfaces/tests/test_base.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/nipype/interfaces/tests/test_base.py b/nipype/interfaces/tests/test_base.py index 81c6ab1368..b4d45c47f4 100644 --- a/nipype/interfaces/tests/test_base.py +++ b/nipype/interfaces/tests/test_base.py @@ -156,15 +156,6 @@ class DeprecationSpec1(nib.TraitedSpec): with pytest.raises(nib.TraitError): set_foo() assert len(w) == 0, 'no warnings, just errors' - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', UserWarning) - - class DeprecationSpec1numeric(nib.TraitedSpec): - foo = nib.traits.Int(deprecated='0.1') - spec_instance = DeprecationSpec1numeric() - set_foo = lambda: setattr(spec_instance, 'foo', 1) - with pytest.raises(nib.TraitError): set_foo() - assert len(w) == 0, 'no warnings, just errors' with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', '', UserWarning) From baa349b5843e5c9e9c86c5dc7499e6098332ed21 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 09:50:35 -0500 Subject: [PATCH 08/33] restring one skipif in interfaces/tests/test_io.py --- nipype/interfaces/tests/test_io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index dbc1fe6ac7..75eb323c4b 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -241,6 +241,7 @@ def test_datasink_to_s3(dummy_input, tmpdir): # Test AWS creds read from env vars +@pytest.mark.skipif(noboto3 or not fakes3, reason="boto3 or fakes3 library is not available") def test_aws_keys_from_env(): ''' Function to ensure the DataSink can successfully read in AWS From 74bd3b80b57fc5e6c270b690fc41017cd1530911 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 09:54:47 -0500 Subject: [PATCH 09/33] cleaning:removing pdb --- nipype/interfaces/fsl/tests/test_dti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/fsl/tests/test_dti.py b/nipype/interfaces/fsl/tests/test_dti.py index 78fa253a3d..00d8836b78 100644 --- a/nipype/interfaces/fsl/tests/test_dti.py +++ b/nipype/interfaces/fsl/tests/test_dti.py @@ -17,7 +17,7 @@ from nipype.interfaces.fsl import Info, no_fsl from nipype.interfaces.base import Undefined -import pytest, pdb +import pytest @pytest.fixture(scope="module") From 688774b238a91ce5441893c6d87ab08656f57a2d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 10:07:37 -0500 Subject: [PATCH 10/33] changing a simple assert to np.isclose in test_icc_anova --- nipype/algorithms/tests/test_icc_anova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/algorithms/tests/test_icc_anova.py b/nipype/algorithms/tests/test_icc_anova.py index 8177c89940..65b1e9c6ed 100644 --- a/nipype/algorithms/tests/test_icc_anova.py +++ b/nipype/algorithms/tests/test_icc_anova.py @@ -19,4 +19,4 @@ def test_ICC_rep_anova(): assert round(icc, 2) == 0.71 assert dfc == 3 assert dfe == 15 - assert r_var / (r_var + e_var) == icc + assert np.isclose(r_var / (r_var + e_var), icc) From 54f496c3f631636753e0546fcbfe0c54fe923b9e Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 14:48:01 -0500 Subject: [PATCH 11/33] adding the auto test for ACompCor --- nipype/algorithms/tests/test_auto_ACompCor.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 nipype/algorithms/tests/test_auto_ACompCor.py diff --git a/nipype/algorithms/tests/test_auto_ACompCor.py b/nipype/algorithms/tests/test_auto_ACompCor.py new file mode 100644 index 0000000000..b28b6086da --- /dev/null +++ b/nipype/algorithms/tests/test_auto_ACompCor.py @@ -0,0 +1,36 @@ +# AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT +from ..confounds import ACompCor + + +def test_ACompCor_inputs(): + input_map = dict(components_file=dict(usedefault=True, + ), + header=dict(), + ignore_exception=dict(nohash=True, + usedefault=True, + ), + mask_file=dict(), + num_components=dict(usedefault=True, + ), + realigned_file=dict(mandatory=True, + ), + regress_poly_degree=dict(usedefault=True, + ), + use_regress_poly=dict(usedefault=True, + ), + ) + inputs = ACompCor.input_spec() + + for key, metadata in list(input_map.items()): + for metakey, value in list(metadata.items()): + assert getattr(inputs.traits()[key], metakey) == value + + +def test_ACompCor_outputs(): + output_map = dict(components_file=dict(), + ) + outputs = ACompCor.output_spec() + + for key, metadata in list(output_map.items()): + for metakey, value in list(metadata.items()): + assert getattr(outputs.traits()[key], metakey) == value From e988b97c38553e6f9474bdfee44e63359c60eb3b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 14:52:55 -0500 Subject: [PATCH 12/33] changing test_auto_TCompCor.py to the output of checkspecs.py --- nipype/algorithms/tests/test_auto_TCompCor.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nipype/algorithms/tests/test_auto_TCompCor.py b/nipype/algorithms/tests/test_auto_TCompCor.py index 8b5984527e..e1da90befb 100644 --- a/nipype/algorithms/tests/test_auto_TCompCor.py +++ b/nipype/algorithms/tests/test_auto_TCompCor.py @@ -29,8 +29,22 @@ def test_TCompCor_inputs(): def test_TCompCor_outputs(): - output_map = dict(components_file=dict(), - high_variance_mask=dict() + output_map = dict(components_file=dict(usedefault=True, + ), + header=dict(), + high_variance_mask=dict(), + ignore_exception=dict(nohash=True, + usedefault=True, + ), + mask_file=dict(), + num_components=dict(usedefault=True, + ), + realigned_file=dict(mandatory=True, + ), + regress_poly_degree=dict(usedefault=True, + ), + use_regress_poly=dict(usedefault=True, + ), ) outputs = TCompCor.output_spec() From 547a3868c53f247ea9078ccd69c7ddcd956c6188 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 17:58:54 -0500 Subject: [PATCH 13/33] removing scripts from tools directory that are not used anymore --- tools/nipype_nightly.py | 91 ---------------------------------------- tools/report_coverage.py | 54 ------------------------ tools/setup.py | 12 ------ 3 files changed, 157 deletions(-) delete mode 100644 tools/nipype_nightly.py delete mode 100644 tools/report_coverage.py delete mode 100644 tools/setup.py diff --git a/tools/nipype_nightly.py b/tools/nipype_nightly.py deleted file mode 100644 index 9f647e903f..0000000000 --- a/tools/nipype_nightly.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -"""Simple script to update the trunk nightly, build the docs and push -to sourceforge. -""" - -from __future__ import print_function -from __future__ import unicode_literals -from builtins import open -import os -import sys -import subprocess - -dirname = '/home/cburns/src/nipy-sf/nipype/trunk/' - - -def run_cmd(cmd): - print(cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=os.environ, - shell=True) - output, error = proc.communicate() - returncode = proc.returncode - if returncode: - msg = 'Running cmd: %s\n Error: %s' % (cmd, error) - raise Exception(msg) - print(output) - - -def update_repos(): - """Update svn repository.""" - os.chdir(dirname) - cmd = 'svn update' - run_cmd(cmd) - - -def build_docs(): - """Build the sphinx documentation.""" - os.chdir(os.path.join(dirname, 'doc')) - cmd = 'make html' - run_cmd(cmd) - - -def push_to_sf(): - """Push documentation to sourceforge.""" - os.chdir(dirname + 'doc') - cmd = 'make sf_cburns' - run_cmd(cmd) - - -def setup_paths(): - # Cron has no PYTHONPATH defined, so we need to add the paths to - # all libraries we need. - pkg_path = '/home/cburns/local/lib/python2.6/site-packages/' - pkg_path_64 = '/home/cburns/local/lib64/python2.6/site-packages/' - - # Add the current directory to path - sys.path.insert(0, os.curdir) - # Add our local path, where we install nipype, to sys.path - sys.path.insert(0, pkg_path) - # Needed to add this to my path at one point otherwise import of - # apigen failed. - # sys.path.insert(2, '/home/cburns/src/nipy-sf/nipype/trunk/tools') - - # Add networkx, twisted, zope.interface and foolscap. - # Basically we need to add all the packages we need that are - # installed via setyptools, since it's uses the .pth files for - # this. - nx_path = os.path.join(pkg_path, 'networkx-0.99-py2.6.egg') - sys.path.insert(2, nx_path) - twisted_path = os.path.join(pkg_path_64, - 'Twisted-8.2.0-py2.6-linux-x86_64.egg') - sys.path.insert(2, twisted_path) - zope_path = os.path.join(pkg_path_64, - 'zope.interface-3.5.2-py2.6-linux-x86_64.egg') - sys.path.insert(2, zope_path) - foolscap_path = os.path.join(pkg_path, - 'foolscap-0.2.9-py2.6.egg') - sys.path.insert(2, foolscap_path) - - # Define our PYTHONPATH variable - os.environ['PYTHONPATH'] = ':'.join(sys.path) - -if __name__ == '__main__': - setup_paths() - prev_dir = os.path.abspath(os.curdir) - update_repos() - build_docs() - # push_to_sf() - os.chdir(prev_dir) diff --git a/tools/report_coverage.py b/tools/report_coverage.py deleted file mode 100644 index d02e2c7851..0000000000 --- a/tools/report_coverage.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -from __future__ import unicode_literals -from builtins import open -import subprocess - - -def run_tests(cmd): - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True) - stdout, stderr = proc.communicate() - if proc.returncode: - msg = 'Running cmd: %s\n Error: %s' % (cmd, error) - raise Exception(msg) - # Nose returns the output in stderr - return stderr - - -def grab_coverage(output): - """Grab coverage lines from nose output.""" - output = output.split('\n') - covout = [] - header = None - tcount = None - for line in output: - if line.startswith('nipype.interfaces.') or \ - line.startswith('nipype.pipeline.') or \ - line.startswith('nipype.utils.'): - # Remove the Missing lines, too noisy - percent_index = line.find('%') - percent_index += 1 - covout.append(line[:percent_index]) - if line.startswith('Name '): - header = line - if line.startswith('Ran '): - tcount = line - covout.insert(0, header) - covout.insert(1, '-' * 70) - covout.append('-' * 70) - covout.append(tcount) - return '\n'.join(covout) - - -def main(): - cmd = 'nosetests --with-coverage --cover-package=nipype' - print('From current directory, running cmd:') - print(cmd, '\n') - output = run_tests(cmd) - report = grab_coverage(output) - print(report) - -main() diff --git a/tools/setup.py b/tools/setup.py deleted file mode 100644 index fba34de078..0000000000 --- a/tools/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python - -from distutils.core import setup - -setup(name='Nipype Tools', - version='0.1', - description='Utilities used in nipype development', - author='Nipype Developers', - author_email='nipy-devel@neuroimaging.scipy.org', - url='http://nipy.sourceforge.net', - scripts=['./nipype_nightly.py', './report_coverage.py'] - ) From ce1b3e2de6d47ff6a5808f4883637bc717120fc1 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 18:00:05 -0500 Subject: [PATCH 14/33] removing fixes --- nipype/fixes/README.txt | 10 - nipype/fixes/__init__.py | 26 -- nipype/fixes/numpy/__init__.py | 2 - nipype/fixes/numpy/testing/__init__.py | 2 - nipype/fixes/numpy/testing/noseclasses.py | 359 ------------------- nipype/fixes/numpy/testing/nosetester.py | 409 ---------------------- nipype/fixes/numpy/testing/utils.py | 3 - 7 files changed, 811 deletions(-) delete mode 100644 nipype/fixes/README.txt delete mode 100644 nipype/fixes/__init__.py delete mode 100644 nipype/fixes/numpy/__init__.py delete mode 100644 nipype/fixes/numpy/testing/__init__.py delete mode 100644 nipype/fixes/numpy/testing/noseclasses.py delete mode 100644 nipype/fixes/numpy/testing/nosetester.py delete mode 100644 nipype/fixes/numpy/testing/utils.py diff --git a/nipype/fixes/README.txt b/nipype/fixes/README.txt deleted file mode 100644 index dd183c1739..0000000000 --- a/nipype/fixes/README.txt +++ /dev/null @@ -1,10 +0,0 @@ -This directory is meant to contain fixes to external packages, such as scipy, numpy -that are meant to eventually be moved upstream to these packages. - -When these changes find their way upstream and are released, -they can be deleted from the "fixes" directory when new versions of -NIPY are released. - -PACKAGES/MODULES: ---------- -scipy/stats_models: corresponds to module "scipy.stats.models" \ No newline at end of file diff --git a/nipype/fixes/__init__.py b/nipype/fixes/__init__.py deleted file mode 100644 index a04158a3ae..0000000000 --- a/nipype/fixes/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# We import numpy fixes during init of the testing package. We need to delay -# import of the testing package until after it has initialized - -from os.path import dirname - -# Cache for the actual testing functin -_tester = None - - -def test(*args, **kwargs): - """ test function for fixes subpackage - - This function defers import of the testing machinery so it can import from - us first. - - See nipy.test docstring for parameters and return values - """ - global _tester - if _tester is None: - from nipy.testing import Tester - _tester = Tester(dirname(__file__)).test - return _tester(*args, **kwargs) - -# Remind nose not to test the test function -test.__test__ = False diff --git a/nipype/fixes/numpy/__init__.py b/nipype/fixes/numpy/__init__.py deleted file mode 100644 index 7850043b8f..0000000000 --- a/nipype/fixes/numpy/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# -*- coding: utf-8 -*- -# numpy fixes package diff --git a/nipype/fixes/numpy/testing/__init__.py b/nipype/fixes/numpy/testing/__init__.py deleted file mode 100644 index 87ed9ba529..0000000000 --- a/nipype/fixes/numpy/testing/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# -*- coding: utf-8 -*- -# Package init for fixes.numpy.testing diff --git a/nipype/fixes/numpy/testing/noseclasses.py b/nipype/fixes/numpy/testing/noseclasses.py deleted file mode 100644 index db7ae585e1..0000000000 --- a/nipype/fixes/numpy/testing/noseclasses.py +++ /dev/null @@ -1,359 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from builtins import object -# These classes implement a doctest runner plugin for nose, a "known failure" -# error class, and a customized TestProgram for NumPy. - -# Because this module imports nose directly, it should not -# be used except by nosetester.py to avoid a general NumPy -# dependency on nose. - -import os -import doctest - -import nose -from nose.plugins import doctests as npd -from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin -from nose.plugins.base import Plugin -from nose.util import src -import numpy -from .nosetester import get_package_name -import inspect - -# Some of the classes in this module begin with 'Numpy' to clearly distinguish -# them from the plethora of very similar names from nose/unittest/doctest - -# ----------------------------------------------------------------------------- -# Modified version of the one in the stdlib, that fixes a python bug (doctests -# not found in extension modules, http://bugs.python.org/issue3158) - - -class NumpyDocTestFinder(doctest.DocTestFinder): - - def _from_module(self, module, object): - """ - Return true if the given object is defined in the given - module. - """ - if module is None: - # print '_fm C1' # dbg - return True - elif inspect.isfunction(object): - # print '_fm C2' # dbg - return module.__dict__ is object.__globals__ - elif inspect.isbuiltin(object): - # print '_fm C2-1' # dbg - return module.__name__ == object.__module__ - elif inspect.isclass(object): - # print '_fm C3' # dbg - return module.__name__ == object.__module__ - elif inspect.ismethod(object): - # This one may be a bug in cython that fails to correctly set the - # __module__ attribute of methods, but since the same error is easy - # to make by extension code writers, having this safety in place - # isn't such a bad idea - # print '_fm C3-1' # dbg - return module.__name__ == object.__self__.__class__.__module__ - elif inspect.getmodule(object) is not None: - # print '_fm C4' # dbg - # print 'C4 mod',module,'obj',object # dbg - return module is inspect.getmodule(object) - elif hasattr(object, '__module__'): - # print '_fm C5' # dbg - return module.__name__ == object.__module__ - elif isinstance(object, property): - # print '_fm C6' # dbg - return True # [XX] no way not be sure. - else: - raise ValueError("object must be a class or function") - - def _find(self, tests, obj, name, module, source_lines, globs, seen): - """ - Find tests for the given object and any contained objects, and - add them to `tests`. - """ - - doctest.DocTestFinder._find(self, tests, obj, name, module, - source_lines, globs, seen) - - # Below we re-run pieces of the above method with manual modifications, - # because the original code is buggy and fails to correctly identify - # doctests in extension modules. - - # Local shorthands - from inspect import isroutine, isclass, ismodule, isfunction, \ - ismethod - - # Look for tests in a module's contained objects. - if ismodule(obj) and self._recurse: - for valname, val in list(obj.__dict__.items()): - valname1 = '%s.%s' % (name, valname) - if ((isroutine(val) or isclass(val)) and - self._from_module(module, val)): - - self._find(tests, val, valname1, module, source_lines, - globs, seen) - - # Look for tests in a class's contained objects. - if isclass(obj) and self._recurse: - # print 'RECURSE into class:',obj # dbg - for valname, val in list(obj.__dict__.items()): - # valname1 = '%s.%s' % (name, valname) # dbg - # print 'N',name,'VN:',valname,'val:',str(val)[:77] # dbg - # Special handling for staticmethod/classmethod. - if isinstance(val, staticmethod): - val = getattr(obj, valname) - if isinstance(val, classmethod): - val = getattr(obj, valname).__func__ - - # Recurse to methods, properties, and nested classes. - if ((isfunction(val) or isclass(val) or - ismethod(val) or isinstance(val, property)) and - self._from_module(module, val)): - valname = '%s.%s' % (name, valname) - self._find(tests, val, valname, module, source_lines, - globs, seen) - - -# second-chance checker; if the default comparison doesn't -# pass, then see if the expected output string contains flags that -# tell us to ignore the output -class NumpyOutputChecker(doctest.OutputChecker): - def check_output(self, want, got, optionflags): - ret = doctest.OutputChecker.check_output(self, want, got, - optionflags) - if not ret: - if "#random" in want: - return True - - # it would be useful to normalize endianness so that - # bigendian machines don't fail all the tests (and there are - # actually some bigendian examples in the doctests). Let's try - # making them all little endian - got = got.replace("'>", "'<") - want = want.replace("'>", "'<") - - # try to normalize out 32 and 64 bit default int sizes - for sz in [4, 8]: - got = got.replace("'>> import numpy as np - >>> np.testing.nosetester.get_package_name('nonsense') # doctest: +ALLOW_UNICODE - 'numpy' - - """ - - fullpath = filepath[:] - pkg_name = [] - while 'site-packages' in filepath or 'dist-packages' in filepath: - filepath, p2 = os.path.split(filepath) - if p2 in ('site-packages', 'dist-packages'): - break - pkg_name.append(p2) - - # if package name determination failed, just default to numpy/scipy - if not pkg_name: - if 'scipy' in fullpath: - return 'scipy' - else: - return 'numpy' - - # otherwise, reverse to get correct order and return - pkg_name.reverse() - - # don't include the outer egg directory - if pkg_name[0].endswith('.egg'): - pkg_name.pop(0) - - return '.'.join(pkg_name) - - -def import_nose(): - """ Import nose only when needed. - """ - fine_nose = True - minimum_nose_version = (0, 10, 0) - try: - import nose - from nose.tools import raises - except ImportError: - fine_nose = False - else: - if nose.__versioninfo__ < minimum_nose_version: - fine_nose = False - - if not fine_nose: - msg = 'Need nose >= %d.%d.%d for tests - see ' \ - 'http://somethingaboutorange.com/mrl/projects/nose' % \ - minimum_nose_version - - raise ImportError(msg) - - return nose - - -def run_module_suite(file_to_run=None): - if file_to_run is None: - f = sys._getframe(1) - file_to_run = f.f_locals.get('__file__', None) - if file_to_run is None: - raise AssertionError - - import_nose().run(argv=['', file_to_run]) - - -class NoseTester(object): - """ - Nose test runner. - - This class is made available as numpy.testing.Tester, and a test function - is typically added to a package's __init__.py like so:: - - from numpy.testing import Tester - test = Tester().test - - Calling this test function finds and runs all tests associated with the - package and all its sub-packages. - - Attributes - ---------- - package_path : str - Full path to the package to test. - package_name : str - Name of the package to test. - - Parameters - ---------- - package : module, str or None - The package to test. If a string, this should be the full path to - the package. If None (default), `package` is set to the module from - which `NoseTester` is initialized. - - """ - # Stuff to exclude from tests. These are from numpy.distutils - excludes = ['f2py_ext', - 'f2py_f90_ext', - 'gen_ext', - 'pyrex_ext', - 'swig_ext'] - - def __init__(self, package=None): - ''' Test class init - - Parameters - ---------- - package : string or module - If string, gives full path to package - If None, extract calling module path - Default is None - ''' - package_name = None - if package is None: - f = sys._getframe(1) - package_path = f.f_locals.get('__file__', None) - if package_path is None: - raise AssertionError - package_path = os.path.dirname(package_path) - package_name = f.f_locals.get('__name__', None) - elif isinstance(package, type(os)): - package_path = os.path.dirname(package.__file__) - package_name = getattr(package, '__name__', None) - else: - package_path = str(package) - - self.package_path = package_path - - # find the package name under test; this name is used to limit coverage - # reporting (if enabled) - if package_name is None: - package_name = get_package_name(package_path) - self.package_name = package_name - - def _test_argv(self, label, verbose, extra_argv): - ''' Generate argv for nosetest command - - Parameters - ---------- - label : {'fast', 'full', '', attribute identifier}, optional - see ``test`` docstring - verbose : int, optional - Verbosity value for test outputs, in the range 1-10. Default is 1. - extra_argv : list, optional - List with any extra arguments to pass to nosetests. - - Returns - ------- - argv : list - command line arguments that will be passed to nose - ''' - argv = [__file__, self.package_path, '-s'] - if label and label != 'full': - if not isinstance(label, (str, bytes)): - raise TypeError('Selection label should be a string') - if label == 'fast': - label = 'not slow' - argv += ['-A', label] - argv += ['--verbosity', str(verbose)] - if extra_argv: - argv += extra_argv - return argv - - def _show_system_info(self): - nose = import_nose() - - import numpy - print("NumPy version %s" % numpy.__version__) - npdir = os.path.dirname(numpy.__file__) - print("NumPy is installed in %s" % npdir) - - if 'scipy' in self.package_name: - import scipy - print("SciPy version %s" % scipy.__version__) - spdir = os.path.dirname(scipy.__file__) - print("SciPy is installed in %s" % spdir) - - pyversion = sys.version.replace('\n', '') - print("Python version %s" % pyversion) - print("nose version %d.%d.%d" % nose.__versioninfo__) - - def _get_custom_doctester(self): - """ Return instantiated plugin for doctests - - Allows subclassing of this class to override doctester - - A return value of None means use the nose builtin doctest plugin - """ - from .noseclasses import NumpyDoctest - return NumpyDoctest() - - def prepare_test_args(self, label='fast', verbose=1, extra_argv=None, - doctests=False, coverage=False): - """ - Run tests for module using nose. - - This method does the heavy lifting for the `test` method. It takes all - the same arguments, for details see `test`. - - See Also - -------- - test - - """ - # fail with nice error message if nose is not present - import_nose() - # compile argv - argv = self._test_argv(label, verbose, extra_argv) - # bypass tests noted for exclude - for ename in self.excludes: - argv += ['--exclude', ename] - # our way of doing coverage - if coverage: - argv += ['--cover-package=%s' % self.package_name, '--with-coverage', - '--cover-tests', '--cover-inclusive', '--cover-erase'] - # construct list of plugins - import nose.plugins.builtin - from .noseclasses import KnownFailure, Unplugger - plugins = [KnownFailure()] - plugins += [p() for p in nose.plugins.builtin.plugins] - # add doctesting if required - doctest_argv = '--with-doctest' in argv - if doctests is False and doctest_argv: - doctests = True - plug = self._get_custom_doctester() - if plug is None: - # use standard doctesting - if doctests and not doctest_argv: - argv += ['--with-doctest'] - else: # custom doctesting - if doctest_argv: # in fact the unplugger would take care of this - argv.remove('--with-doctest') - plugins += [Unplugger('doctest'), plug] - if doctests: - argv += ['--with-' + plug.name] - return argv, plugins - - def test(self, label='fast', verbose=1, extra_argv=None, doctests=False, - coverage=False): - """ - Run tests for module using nose. - - Parameters - ---------- - label : {'fast', 'full', '', attribute identifier}, optional - Identifies the tests to run. This can be a string to pass to - the nosetests executable with the '-A' option, or one of several - special values. Special values are: - * 'fast' - the default - which corresponds to the ``nosetests -A`` - option of 'not slow'. - * 'full' - fast (as above) and slow tests as in the - 'no -A' option to nosetests - this is the same as ''. - * None or '' - run all tests. - attribute_identifier - string passed directly to nosetests as '-A'. - verbose : int, optional - Verbosity value for test outputs, in the range 1-10. Default is 1. - extra_argv : list, optional - List with any extra arguments to pass to nosetests. - doctests : bool, optional - If True, run doctests in module. Default is False. - coverage : bool, optional - If True, report coverage of NumPy code. Default is False. - (This requires the `coverage module: - `_). - - Returns - ------- - result : object - Returns the result of running the tests as a - ``nose.result.TextTestResult`` object. - - Notes - ----- - Each NumPy module exposes `test` in its namespace to run all tests for it. - For example, to run all tests for numpy.lib: - - >>> np.lib.test() #doctest: +SKIP - - Examples - -------- - >>> result = np.lib.test() #doctest: +SKIP - Running unit tests for numpy.lib - ... - Ran 976 tests in 3.933s - - OK - - >>> result.errors #doctest: +SKIP - [] - >>> result.knownfail #doctest: +SKIP - [] - """ - - # cap verbosity at 3 because nose becomes *very* verbose beyond that - verbose = min(verbose, 3) - - from . import utils - utils.verbose = verbose - - if doctests: - print("Running unit tests and doctests for %s" % self.package_name) - else: - print("Running unit tests for %s" % self.package_name) - - self._show_system_info() - - # reset doctest state on every run - import doctest - doctest.master = None - - argv, plugins = self.prepare_test_args(label, verbose, extra_argv, - doctests, coverage) - from .noseclasses import NumpyTestProgram - t = NumpyTestProgram(argv=argv, exit=False, plugins=plugins) - return t.result - - def bench(self, label='fast', verbose=1, extra_argv=None): - """ - Run benchmarks for module using nose. - - Parameters - ---------- - label : {'fast', 'full', '', attribute identifier}, optional - Identifies the benchmarks to run. This can be a string to pass to - the nosetests executable with the '-A' option, or one of several - special values. Special values are: - * 'fast' - the default - which corresponds to the ``nosetests -A`` - option of 'not slow'. - * 'full' - fast (as above) and slow benchmarks as in the - 'no -A' option to nosetests - this is the same as ''. - * None or '' - run all tests. - attribute_identifier - string passed directly to nosetests as '-A'. - verbose : int, optional - Verbosity value for benchmark outputs, in the range 1-10. Default is 1. - extra_argv : list, optional - List with any extra arguments to pass to nosetests. - - Returns - ------- - success : bool - Returns True if running the benchmarks works, False if an error - occurred. - - Notes - ----- - Benchmarks are like tests, but have names starting with "bench" instead - of "test", and can be found under the "benchmarks" sub-directory of the - module. - - Each NumPy module exposes `bench` in its namespace to run all benchmarks - for it. - - Examples - -------- - >>> success = np.lib.bench() #doctest: +SKIP - Running benchmarks for numpy.lib - ... - using 562341 items: - unique: - 0.11 - unique1d: - 0.11 - ratio: 1.0 - nUnique: 56230 == 56230 - ... - OK - - >>> success #doctest: +SKIP - True - - """ - - print("Running benchmarks for %s" % self.package_name) - self._show_system_info() - - argv = self._test_argv(label, verbose, extra_argv) - argv += ['--match', r'(?:^|[\\b_\\.%s-])[Bb]ench' % os.sep] - - # import nose or make informative error - nose = import_nose() - - # get plugin to disable doctests - from .noseclasses import Unplugger - add_plugins = [Unplugger('doctest')] - - return nose.run(argv=argv, addplugins=add_plugins) diff --git a/nipype/fixes/numpy/testing/utils.py b/nipype/fixes/numpy/testing/utils.py deleted file mode 100644 index f50cb0bd2a..0000000000 --- a/nipype/fixes/numpy/testing/utils.py +++ /dev/null @@ -1,3 +0,0 @@ -# -*- coding: utf-8 -*- -# Allow numpy fixes noseclasses to do local import of utils -from numpy.testing.utils import * From c8ee538bca0285b8505688113b299e28fedecb97 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 18:11:40 -0500 Subject: [PATCH 15/33] removing coverage from travis (so the coverage is calculated from Circle CI only) --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6dbefbd97c..b62ecf142d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,14 +37,9 @@ install: rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && pip install -r requirements.txt && pip install -e .[$NIPYPE_EXTRAS] && - export COVERAGE_PROCESS_START=$(pwd)/.coveragerc && - export COVERAGE_DATA_FILE=$(pwd)/.coverage && - echo "data_file = ${COVERAGE_DATA_FILE}" >> ${COVERAGE_PROCESS_START}; } - travis_retry inst script: - py.test --doctest-modules --cov=nipype nipype -after_success: -- bash <(curl -s https://codecov.io/bash) -t ac172a50-8e66-42e5-8822-5373fcf54686 -cF unittests deploy: provider: pypi user: satra From 623f20f4328f03144cd3032278f570dfb859469b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 18:34:12 -0500 Subject: [PATCH 16/33] fixing the travis file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b62ecf142d..bf3d4beb73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ install: conda install -y nipype && rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && pip install -r requirements.txt && - pip install -e .[$NIPYPE_EXTRAS] && + pip install -e .[$NIPYPE_EXTRAS] ;} - travis_retry inst script: - py.test --doctest-modules --cov=nipype nipype From f353fbbe2a21b05a7f314348014150ae93b7636c Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 18:47:24 -0500 Subject: [PATCH 17/33] Removing nosetest from nipype/__init__.py; pytest version is missing for now --- nipype/__init__.py | 67 +++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/nipype/__init__.py b/nipype/__init__.py index b633736023..d6b6196ed2 100644 --- a/nipype/__init__.py +++ b/nipype/__init__.py @@ -11,7 +11,6 @@ STATUS as __status__, __version__) from .utils.config import NipypeConfig -from .fixes.numpy.testing import nosetester from .utils.logger import Logging from .refs import due @@ -24,48 +23,48 @@ config = NipypeConfig() logging = Logging(config) +#NOTE_dj: it has to be changed to python +#class _NoseTester(nosetester.NoseTester): +# """ Subclass numpy's NoseTester to add doctests by default +# """ -class _NoseTester(nosetester.NoseTester): - """ Subclass numpy's NoseTester to add doctests by default - """ +# def _get_custom_doctester(self): +# return None - def _get_custom_doctester(self): - return None +# def test(self, label='fast', verbose=1, extra_argv=['--exe'], +# doctests=True, coverage=False): +# """Run the full test suite +# +# Examples +# -------- +# This will run the test suite and stop at the first failing +# example +# >>> from nipype import test +# >>> test(extra_argv=['--exe', '-sx']) # doctest: +SKIP +# """ +# return super(_NoseTester, self).test(label=label, +# verbose=verbose, +# extra_argv=extra_argv, +# doctests=doctests, +# coverage=coverage) - def test(self, label='fast', verbose=1, extra_argv=['--exe'], - doctests=True, coverage=False): - """Run the full test suite - - Examples - -------- - This will run the test suite and stop at the first failing - example - >>> from nipype import test - >>> test(extra_argv=['--exe', '-sx']) # doctest: +SKIP - """ - return super(_NoseTester, self).test(label=label, - verbose=verbose, - extra_argv=extra_argv, - doctests=doctests, - coverage=coverage) - -try: - test = _NoseTester(raise_warnings="release").test -except TypeError: +#try: +# test = _NoseTester(raise_warnings="release").test +#except TypeError: # Older versions of numpy do not have a raise_warnings argument - test = _NoseTester().test -del nosetester +# test = _NoseTester().test +#del nosetester # Set up package information function -from .pkg_info import get_pkg_info as _get_pkg_info -get_info = lambda: _get_pkg_info(os.path.dirname(__file__)) +#from .pkg_info import get_pkg_info as _get_pkg_info +#get_info = lambda: _get_pkg_info(os.path.dirname(__file__)) # If this file is exec after being imported, the following lines will # fail -try: - del Tester -except: - pass +#try: +# del Tester +#except: +# pass from .pipeline import Node, MapNode, JoinNode, Workflow From d7c6a540b758ee99651b6d0751ea524cab355916 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 22:32:55 -0500 Subject: [PATCH 18/33] restore get_info in nipype/__init__.py --- nipype/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nipype/__init__.py b/nipype/__init__.py index d6b6196ed2..c7ce79c99e 100644 --- a/nipype/__init__.py +++ b/nipype/__init__.py @@ -56,8 +56,8 @@ #del nosetester # Set up package information function -#from .pkg_info import get_pkg_info as _get_pkg_info -#get_info = lambda: _get_pkg_info(os.path.dirname(__file__)) +from .pkg_info import get_pkg_info as _get_pkg_info +get_info = lambda: _get_pkg_info(os.path.dirname(__file__)) # If this file is exec after being imported, the following lines will # fail From 7d77343e475406c8340986b8afefeabfa3d81b55 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Dec 2016 22:55:38 -0500 Subject: [PATCH 19/33] removing workflows from the directories that are omitted --- .coveragerc | 1 - 1 file changed, 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index ea62a2eb46..c69c0eef3c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,6 +4,5 @@ source = nipype include = */nipype/* omit = */nipype/external/* - */nipype/workflows/* */nipype/fixes/* */setup.py From e981a53fd6c778278fc8eeaf137e8e3d081d5f27 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 16 Dec 2016 07:45:07 -0500 Subject: [PATCH 20/33] trying to fix coverage (codecov doesnt report anything), restoring some lines from travis --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf3d4beb73..5a428e654b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,10 +36,13 @@ install: conda install -y nipype && rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && pip install -r requirements.txt && - pip install -e .[$NIPYPE_EXTRAS] ;} + pip install -e .[$NIPYPE_EXTRAS] && + export COVERAGE_PROCESS_START=$(pwd)/.coveragerc && + export COVERAGE_DATA_FILE=$(pwd)/.coverage && + echo "data_file = ${COVERAGE_DATA_FILE}" >> ${COVERAGE_PROCESS_START}; } - travis_retry inst script: -- py.test --doctest-modules --cov=nipype nipype +- py.test --doctest-modules nipype deploy: provider: pypi user: satra From a3cf60d870f795e0640e879fa696e0065b66ce32 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 16 Dec 2016 09:02:45 -0500 Subject: [PATCH 21/33] it looks like codecov works at the end, removing COVERAGE_PROCESS_START and COVERAGE_DATA_FILE from travis again --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a428e654b..ca29323e5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,10 +36,7 @@ install: conda install -y nipype && rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype* && pip install -r requirements.txt && - pip install -e .[$NIPYPE_EXTRAS] && - export COVERAGE_PROCESS_START=$(pwd)/.coveragerc && - export COVERAGE_DATA_FILE=$(pwd)/.coverage && - echo "data_file = ${COVERAGE_DATA_FILE}" >> ${COVERAGE_PROCESS_START}; } + pip install -e .[$NIPYPE_EXTRAS]; } - travis_retry inst script: - py.test --doctest-modules nipype From 7d529e04ba06bab2e7287fc165212dfbbbba149a Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 16 Dec 2016 21:58:31 -0500 Subject: [PATCH 22/33] changing %s to format in test_maths (in all places that the strings are used only locally in test functions) --- nipype/interfaces/fsl/tests/test_maths.py | 90 +++++++++++------------ 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/nipype/interfaces/fsl/tests/test_maths.py b/nipype/interfaces/fsl/tests/test_maths.py index 40ba11b893..0d1425a345 100644 --- a/nipype/interfaces/fsl/tests/test_maths.py +++ b/nipype/interfaces/fsl/tests/test_maths.py @@ -77,23 +77,23 @@ def test_maths_base(create_files_in_directory): # Set an in file maths.inputs.in_file = "a.nii" - out_file = "a_maths%s" % out_ext + out_file = "a_maths{}".format(out_ext) # Now test the most basic command line - assert maths.cmdline == "fslmaths a.nii %s" % os.path.join(testdir, out_file) + assert maths.cmdline == "fslmaths a.nii {}".format(os.path.join(testdir, out_file)) # Now test that we can set the various data types dtypes = ["float", "char", "int", "short", "double", "input"] - int_cmdline = "fslmaths -dt %s a.nii " + os.path.join(testdir, out_file) - out_cmdline = "fslmaths a.nii " + os.path.join(testdir, out_file) + " -odt %s" - duo_cmdline = "fslmaths -dt %s a.nii " + os.path.join(testdir, out_file) + " -odt %s" + int_cmdline = "fslmaths -dt {} a.nii " + os.path.join(testdir, out_file) + out_cmdline = "fslmaths a.nii " + os.path.join(testdir, out_file) + " -odt {}" + duo_cmdline = "fslmaths -dt {} a.nii " + os.path.join(testdir, out_file) + " -odt {}" for dtype in dtypes: foo = fsl.MathsCommand(in_file="a.nii", internal_datatype=dtype) - assert foo.cmdline == int_cmdline % dtype + assert foo.cmdline == int_cmdline.format(dtype) bar = fsl.MathsCommand(in_file="a.nii", output_datatype=dtype) - assert bar.cmdline == out_cmdline % dtype + assert bar.cmdline == out_cmdline.format(dtype) foobar = fsl.MathsCommand(in_file="a.nii", internal_datatype=dtype, output_datatype=dtype) - assert foobar.cmdline == duo_cmdline % (dtype, dtype) + assert foobar.cmdline == duo_cmdline.format(dtype, dtype) # Test that we can ask for an outfile name maths.inputs.out_file = "b.nii" @@ -124,10 +124,10 @@ def test_changedt(create_files_in_directory): # Now test that we can set the various data types dtypes = ["float", "char", "int", "short", "double", "input"] - cmdline = "fslmaths a.nii b.nii -odt %s" + cmdline = "fslmaths a.nii b.nii -odt {}" for dtype in dtypes: foo = fsl.MathsCommand(in_file="a.nii", out_file="b.nii", output_datatype=dtype) - assert foo.cmdline == cmdline % dtype + assert foo.cmdline == cmdline.format(dtype) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -145,22 +145,22 @@ def test_threshold(create_files_in_directory): thresh.run() # Test the various opstrings - cmdline = "fslmaths a.nii %s b.nii" + cmdline = "fslmaths a.nii {} b.nii" for val in [0, 0., -1, -1.5, -0.5, 0.5, 3, 400, 400.5]: thresh.inputs.thresh = val - assert thresh.cmdline == cmdline % "-thr %.10f" % val + assert thresh.cmdline == cmdline.format("-thr {:.10f}".format(val)) - val = "%.10f" % 42 + val = "{:.10f}".format(42) thresh = fsl.Threshold(in_file="a.nii", out_file="b.nii", thresh=42, use_robust_range=True) - assert thresh.cmdline == cmdline % ("-thrp " + val) + assert thresh.cmdline == cmdline.format("-thrp " + val) thresh.inputs.use_nonzero_voxels = True - assert thresh.cmdline == cmdline % ("-thrP " + val) + assert thresh.cmdline == cmdline.format("-thrP " + val) thresh = fsl.Threshold(in_file="a.nii", out_file="b.nii", thresh=42, direction="above") - assert thresh.cmdline == cmdline % ("-uthr " + val) + assert thresh.cmdline == cmdline.format("-uthr " + val) thresh.inputs.use_robust_range = True - assert thresh.cmdline == cmdline % ("-uthrp " + val) + assert thresh.cmdline == cmdline.format("-uthrp " + val) thresh.inputs.use_nonzero_voxels = True - assert thresh.cmdline == cmdline % ("-uthrP " + val) + assert thresh.cmdline == cmdline.format("-uthrP " + val) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -177,14 +177,14 @@ def test_meanimage(create_files_in_directory): assert meaner.cmdline == "fslmaths a.nii -Tmean b.nii" # Test the other dimensions - cmdline = "fslmaths a.nii -%smean b.nii" + cmdline = "fslmaths a.nii -{}mean b.nii" for dim in ["X", "Y", "Z", "T"]: meaner.inputs.dimension = dim - assert meaner.cmdline == cmdline % dim + assert meaner.cmdline == cmdline.format(dim) # Test the auto naming meaner = fsl.MeanImage(in_file="a.nii") - assert meaner.cmdline == "fslmaths a.nii -Tmean %s" % os.path.join(testdir, "a_mean%s" % out_ext) + assert meaner.cmdline == "fslmaths a.nii -Tmean {}".format(os.path.join(testdir, "a_mean{}".format(out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -201,14 +201,14 @@ def test_stdimage(create_files_in_directory): assert stder.cmdline == "fslmaths a.nii -Tstd b.nii" # Test the other dimensions - cmdline = "fslmaths a.nii -%sstd b.nii" + cmdline = "fslmaths a.nii -{}std b.nii" for dim in ["X","Y","Z","T"]: stder.inputs.dimension=dim - assert stder.cmdline == cmdline%dim + assert stder.cmdline == cmdline.format(dim) # Test the auto naming stder = fsl.StdImage(in_file="a.nii", output_type='NIFTI') - assert stder.cmdline == "fslmaths a.nii -Tstd %s"%os.path.join(testdir, "a_std.nii") + assert stder.cmdline == "fslmaths a.nii -Tstd {}".format(os.path.join(testdir, "a_std.nii")) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -225,14 +225,14 @@ def test_maximage(create_files_in_directory): assert maxer.cmdline == "fslmaths a.nii -Tmax b.nii" # Test the other dimensions - cmdline = "fslmaths a.nii -%smax b.nii" + cmdline = "fslmaths a.nii -{}max b.nii" for dim in ["X", "Y", "Z", "T"]: maxer.inputs.dimension = dim - assert maxer.cmdline == cmdline % dim + assert maxer.cmdline == cmdline.format(dim) # Test the auto naming maxer = fsl.MaxImage(in_file="a.nii") - assert maxer.cmdline == "fslmaths a.nii -Tmax %s" % os.path.join(testdir, "a_max%s" % out_ext) + assert maxer.cmdline == "fslmaths a.nii -Tmax {}".format(os.path.join(testdir, "a_max{}".format(out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -250,17 +250,17 @@ def test_smooth(create_files_in_directory): smoother.run() # Test smoothing kernels - cmdline = "fslmaths a.nii -s %.5f b.nii" + cmdline = "fslmaths a.nii -s {:.5f} b.nii" for val in [0, 1., 1, 25, 0.5, 8 / 3.]: smoother = fsl.IsotropicSmooth(in_file="a.nii", out_file="b.nii", sigma=val) - assert smoother.cmdline == cmdline % val + assert smoother.cmdline == cmdline.format(val) smoother = fsl.IsotropicSmooth(in_file="a.nii", out_file="b.nii", fwhm=val) val = float(val) / np.sqrt(8 * np.log(2)) - assert smoother.cmdline == cmdline % val + assert smoother.cmdline == cmdline.format(val) # Test automatic naming smoother = fsl.IsotropicSmooth(in_file="a.nii", sigma=5) - assert smoother.cmdline == "fslmaths a.nii -s %.5f %s" % (5, os.path.join(testdir, "a_smooth%s" % out_ext)) + assert smoother.cmdline == "fslmaths a.nii -s {:.5f} {}".format(5, os.path.join(testdir, "a_smooth{}".format(out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -283,7 +283,7 @@ def test_mask(create_files_in_directory): # Test auto name generation masker = fsl.ApplyMask(in_file="a.nii", mask_file="b.nii") - assert masker.cmdline == "fslmaths a.nii -mas b.nii " + os.path.join(testdir, "a_masked%s" % out_ext) + assert masker.cmdline == "fslmaths a.nii -mas b.nii " + os.path.join(testdir, "a_masked{}".format(out_ext)) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -304,14 +304,14 @@ def test_dilation(create_files_in_directory): for op in ["mean", "modal", "max"]: cv = dict(mean="M", modal="D", max="F") diller.inputs.operation = op - assert diller.cmdline == "fslmaths a.nii -dil%s b.nii" % cv[op] + assert diller.cmdline == "fslmaths a.nii -dil{} b.nii".format(cv[op]) # Now test the different kernel options for k in ["3D", "2D", "box", "boxv", "gauss", "sphere"]: for size in [1, 1.5, 5]: diller.inputs.kernel_shape = k diller.inputs.kernel_size = size - assert diller.cmdline == "fslmaths a.nii -kernel %s %.4f -dilF b.nii" % (k, size) + assert diller.cmdline == "fslmaths a.nii -kernel {} {:.4f} -dilF b.nii".format(k, size) # Test that we can use a file kernel f = open("kernel.txt", "w").close() @@ -323,7 +323,7 @@ def test_dilation(create_files_in_directory): # Test that we don't need to request an out name dil = fsl.DilateImage(in_file="a.nii", operation="max") - assert dil.cmdline == "fslmaths a.nii -dilF %s" % os.path.join(testdir, "a_dil%s" % out_ext) + assert dil.cmdline == "fslmaths a.nii -dilF {}".format(os.path.join(testdir, "a_dil{}".format(out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -345,7 +345,7 @@ def test_erosion(create_files_in_directory): # Test that we don't need to request an out name erode = fsl.ErodeImage(in_file="a.nii") - assert erode.cmdline == "fslmaths a.nii -ero %s" % os.path.join(testdir, "a_ero%s" % out_ext) + assert erode.cmdline == "fslmaths a.nii -ero {}".format(os.path.join(testdir, "a_ero{}".format(out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -365,11 +365,11 @@ def test_spatial_filter(create_files_in_directory): # Test the different operations for op in ["mean", "meanu", "median"]: filter.inputs.operation = op - assert filter.cmdline == "fslmaths a.nii -f%s b.nii" % op + assert filter.cmdline == "fslmaths a.nii -f{} b.nii".format(op) # Test that we don't need to ask for an out name filter = fsl.SpatialFilter(in_file="a.nii", operation="mean") - assert filter.cmdline == "fslmaths a.nii -fmean %s" % os.path.join(testdir, "a_filt%s" % out_ext) + assert filter.cmdline == "fslmaths a.nii -fmean {}".format(os.path.join(testdir, "a_filt{}".format(out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -390,12 +390,12 @@ def test_unarymaths(create_files_in_directory): ops = ["exp", "log", "sin", "cos", "sqr", "sqrt", "recip", "abs", "bin", "index"] for op in ops: maths.inputs.operation = op - assert maths.cmdline == "fslmaths a.nii -%s b.nii" % op + assert maths.cmdline == "fslmaths a.nii -{} b.nii".format(op) # Test that we don't need to ask for an out file for op in ops: maths = fsl.UnaryMaths(in_file="a.nii", operation=op) - assert maths.cmdline == "fslmaths a.nii -%s %s" % (op, os.path.join(testdir, "a_%s%s" % (op, out_ext))) + assert maths.cmdline == "fslmaths a.nii -{} {}".format(op, os.path.join(testdir, "a_{}{}".format(op, out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -420,15 +420,15 @@ def test_binarymaths(create_files_in_directory): maths = fsl.BinaryMaths(in_file="a.nii", out_file="c.nii", operation=op) if ent == "b.nii": maths.inputs.operand_file = ent - assert maths.cmdline == "fslmaths a.nii -%s b.nii c.nii" % op + assert maths.cmdline == "fslmaths a.nii -{} b.nii c.nii".format(op) else: maths.inputs.operand_value = ent - assert maths.cmdline == "fslmaths a.nii -%s %.8f c.nii" % (op, ent) + assert maths.cmdline == "fslmaths a.nii -{} {:.8f} c.nii".format(op, ent) # Test that we don't need to ask for an out file for op in ops: maths = fsl.BinaryMaths(in_file="a.nii", operation=op, operand_file="b.nii") - assert maths.cmdline == "fslmaths a.nii -%s b.nii %s" % (op, os.path.join(testdir, "a_maths%s" % out_ext)) + assert maths.cmdline == "fslmaths a.nii -{} b.nii {}".format(op, os.path.join(testdir, "a_maths{}".format(out_ext))) @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") @@ -478,12 +478,12 @@ def test_tempfilt(create_files_in_directory): for win in windows: filt.inputs.highpass_sigma = win[0] filt.inputs.lowpass_sigma = win[1] - assert filt.cmdline == "fslmaths a.nii -bptf %.6f %.6f b.nii" % win + assert filt.cmdline == "fslmaths a.nii -bptf {:.6f} {:.6f} b.nii".format(win[0], win[1]) # Test that we don't need to ask for an out file filt = fsl.TemporalFilter(in_file="a.nii", highpass_sigma=64) assert filt.cmdline == \ - "fslmaths a.nii -bptf 64.000000 -1.000000 %s" % os.path.join(testdir, "a_filt%s" % out_ext) + "fslmaths a.nii -bptf 64.000000 -1.000000 {}".format(os.path.join(testdir, "a_filt{}".format(out_ext))) From 378664041b81d790d7fe5dace35de2763459e84e Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 18 Dec 2016 11:37:54 -0500 Subject: [PATCH 23/33] removing test_BEDPOSTX.py and test_XFibres.py (auto tests wont be created anyway) --- nipype/interfaces/fsl/tests/test_BEDPOSTX.py | 2 -- nipype/interfaces/fsl/tests/test_XFibres.py | 2 -- 2 files changed, 4 deletions(-) delete mode 100644 nipype/interfaces/fsl/tests/test_BEDPOSTX.py delete mode 100644 nipype/interfaces/fsl/tests/test_XFibres.py diff --git a/nipype/interfaces/fsl/tests/test_BEDPOSTX.py b/nipype/interfaces/fsl/tests/test_BEDPOSTX.py deleted file mode 100644 index c6ccb51c53..0000000000 --- a/nipype/interfaces/fsl/tests/test_BEDPOSTX.py +++ /dev/null @@ -1,2 +0,0 @@ -# to ensure that the autotest is not written -from nipype.interfaces.fsl.dti import BEDPOSTX diff --git a/nipype/interfaces/fsl/tests/test_XFibres.py b/nipype/interfaces/fsl/tests/test_XFibres.py deleted file mode 100644 index 032a27b986..0000000000 --- a/nipype/interfaces/fsl/tests/test_XFibres.py +++ /dev/null @@ -1,2 +0,0 @@ -# to ensure that the autotest is not written -from nipype.interfaces.fsl.dti import XFibres From 4fa120b1e3fc33f304193c8038d3f5a1c4e18f7d Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sun, 18 Dec 2016 15:04:36 -0500 Subject: [PATCH 24/33] fix: clean up some tests and remove dependency on raises_regexp --- nipype/algorithms/tests/test_compcor.py | 4 +-- nipype/algorithms/tests/test_mesh_ops.py | 6 ++-- .../interfaces/freesurfer/tests/test_model.py | 33 +++++++++---------- .../interfaces/fsl/tests/test_preprocess.py | 7 ++-- 4 files changed, 22 insertions(+), 28 deletions(-) diff --git a/nipype/algorithms/tests/test_compcor.py b/nipype/algorithms/tests/test_compcor.py index 54efe0f8b8..05a964ecfb 100644 --- a/nipype/algorithms/tests/test_compcor.py +++ b/nipype/algorithms/tests/test_compcor.py @@ -83,13 +83,13 @@ def test_compcor_bad_input_shapes(self): for data_shape in (shape_less_than, shape_more_than): data_file = utils.save_toy_nii(np.zeros(data_shape), 'temp.nii') interface = CompCor(realigned_file=data_file, mask_file=self.mask_file) - with pytest.raises_regexp(ValueError, "dimensions"): interface.run() + with pytest.raises(ValueError, message="Dimension mismatch"): interface.run() def test_tcompcor_bad_input_dim(self): bad_dims = (2, 2, 2) data_file = utils.save_toy_nii(np.zeros(bad_dims), 'temp.nii') interface = TCompCor(realigned_file=data_file) - with pytest.raises_regexp(ValueError, '4-D'): interface.run() + with pytest.raises(ValueError, message='Not a 4D file'): interface.run() def run_cc(self, ccinterface, expected_components, expected_header='CompCor'): # run diff --git a/nipype/algorithms/tests/test_mesh_ops.py b/nipype/algorithms/tests/test_mesh_ops.py index 9762b900b2..fa7ebebe54 100644 --- a/nipype/algorithms/tests/test_mesh_ops.py +++ b/nipype/algorithms/tests/test_mesh_ops.py @@ -34,8 +34,6 @@ def test_ident_distances(tmpdir): @pytest.mark.skipif(VTKInfo.no_tvtk(), reason="tvtk is not installed") def test_trans_distances(tmpdir): tempdir = str(tmpdir) - os.chdir(tempdir) - from ...interfaces.vtkbase import tvtk in_surf = example_data('surf01.vtk') @@ -57,10 +55,10 @@ def test_trans_distances(tmpdir): dist.inputs.surface2 = warped_surf dist.inputs.out_file = os.path.join(tempdir, 'distance.npy') res = dist.run() - npt.assert_almost_equal(res.outputs.distance, np.linalg.norm(inc), 4) + assert np.allclose(res.outputs.distance, np.linalg.norm(inc), 4) dist.inputs.weighting = 'area' res = dist.run() - npt.assert_almost_equal(res.outputs.distance, np.linalg.norm(inc), 4) + assert np.allclose(res.outputs.distance, np.linalg.norm(inc), 4) @pytest.mark.skipif(VTKInfo.no_tvtk(), reason="tvtk is not installed") diff --git a/nipype/interfaces/freesurfer/tests/test_model.py b/nipype/interfaces/freesurfer/tests/test_model.py index 41aa3b1197..d9da543154 100644 --- a/nipype/interfaces/freesurfer/tests/test_model.py +++ b/nipype/interfaces/freesurfer/tests/test_model.py @@ -3,8 +3,6 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: import os -import tempfile -import shutil import numpy as np import nibabel as nib @@ -14,12 +12,11 @@ @pytest.mark.skipif(no_freesurfer(), reason="freesurfer is not installed") -def test_concatenate(): - tmp_dir = os.path.realpath(tempfile.mkdtemp()) - cwd = os.getcwd() - os.chdir(tmp_dir) - in1 = os.path.join(tmp_dir, 'cont1.nii') - in2 = os.path.join(tmp_dir, 'cont2.nii') +def test_concatenate(tmpdir): + tempdir = str(tmpdir) + os.chdir(tempdir) + in1 = os.path.join(tempdir, 'cont1.nii') + in2 = os.path.join(tempdir, 'cont2.nii') out = 'bar.nii' data1 = np.zeros((3, 3, 3, 1), dtype=np.float32) @@ -32,27 +29,27 @@ def test_concatenate(): # Test default behavior res = model.Concatenate(in_files=[in1, in2]).run() - assert res.outputs.concatenated_file == os.path.join(tmp_dir, 'concat_output.nii.gz') - assert nib.load('concat_output.nii.gz').get_data() == out_data + assert res.outputs.concatenated_file == os.path.join(tempdir, 'concat_output.nii.gz') + assert np.allclose(nib.load('concat_output.nii.gz').get_data(), out_data) # Test specified concatenated_file res = model.Concatenate(in_files=[in1, in2], concatenated_file=out).run() - assert res.outputs.concatenated_file == os.path.join(tmp_dir, out) - assert nib.load(out).get_data() == out_data + assert res.outputs.concatenated_file == os.path.join(tempdir, out) + assert np.allclose(nib.load(out).get_data(), out_data) # Test in workflow - wf = pe.Workflow('test_concatenate', base_dir=tmp_dir) + wf = pe.Workflow('test_concatenate', base_dir=tempdir) concat = pe.Node(model.Concatenate(in_files=[in1, in2], concatenated_file=out), name='concat') wf.add_nodes([concat]) wf.run() - assert nib.load(os.path.join(tmp_dir, 'test_concatenate','concat', out)).get_data()== out_data + assert np.allclose(nib.load(os.path.join(tempdir, + 'test_concatenate', + 'concat', out)).get_data(), + out_data) # Test a simple statistic res = model.Concatenate(in_files=[in1, in2], concatenated_file=out, stats='mean').run() - assert nib.load(out).get_data() == mean_data - - os.chdir(cwd) - shutil.rmtree(tmp_dir) + assert np.allclose(nib.load(out).get_data(), mean_data) diff --git a/nipype/interfaces/fsl/tests/test_preprocess.py b/nipype/interfaces/fsl/tests/test_preprocess.py index f702842aeb..a3098ddb6a 100644 --- a/nipype/interfaces/fsl/tests/test_preprocess.py +++ b/nipype/interfaces/fsl/tests/test_preprocess.py @@ -169,10 +169,9 @@ def test_fast_list_outputs(setup_infile): def _run_and_test(opts, output_base): outputs = fsl.FAST(**opts)._list_outputs() for output in outputs.values(): - filenames = filename_to_list(output) - if filenames is not None: - for filename in filenames: - assert filename[:len(output_base)] == output_base + if output: + for filename in filename_to_list(output): + assert os.path.realpath(filename).startswith(os.path.realpath(output_base)) # set up tmp_infile, indir = setup_infile From 5b0e74c737f9e54fbab7d6165ae413c1c8ab9f72 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sun, 18 Dec 2016 15:09:20 -0500 Subject: [PATCH 25/33] fix: addresses issue #1446 (solution from @ashgillman in #1551) --- nipype/pipeline/engine/nodes.py | 3 +- nipype/pipeline/engine/tests/test_utils.py | 50 ++++++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 675646d56e..551abd18ae 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -1128,7 +1128,8 @@ def _node_runner(self, nodes, updatehash=False): err = None try: node.run(updatehash=updatehash) - except Exception as err: + except Exception as e: + err = str(e) if str2bool(self.config['execution']['stop_on_first_crash']): raise finally: diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 0134017f91..29b7871445 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -339,6 +339,7 @@ def test_provenance(tmpdir): def test_mapnode_crash(tmpdir): + """Test mapnode crash when stop_on_first_crash is True""" def myfunction(string): return string + 'meh' node = pe.MapNode(niu.Function(input_names=['WRONG'], @@ -350,14 +351,55 @@ def myfunction(string): node.inputs.WRONG = ['string' + str(i) for i in range(3)] node.config = deepcopy(config._sections) node.config['execution']['stop_on_first_crash'] = True - cwd = os.getcwd() - node.base_dir = tmpdir + node.base_dir = str(tmpdir) error_raised = False try: node.run() except TypeError as e: error_raised = True - os.chdir(cwd) - rmtree(node.base_dir) + assert error_raised + + +def test_mapnode_crash2(tmpdir): + """Test mapnode crash when stop_on_first_crash is False""" + def myfunction(string): + return string + 'meh' + node = pe.MapNode(niu.Function(input_names=['WRONG'], + output_names=['newstring'], + function=myfunction), + iterfield=['WRONG'], + name='myfunc') + + node.inputs.WRONG = ['string' + str(i) for i in range(3)] + node.base_dir = str(tmpdir) + + error_raised = False + try: + node.run() + except Exception as e: + error_raised = True + assert error_raised + + +def test_mapnode_crash3(tmpdir): + """Test mapnode crash when mapnode is embedded in a workflow""" + def myfunction(string): + return string + 'meh' + node = pe.MapNode(niu.Function(input_names=['WRONG'], + output_names=['newstring'], + function=myfunction), + iterfield=['WRONG'], + name='myfunc') + + node.inputs.WRONG = ['string' + str(i) for i in range(3)] + wf = pe.Workflow('test_mapnode_crash') + wf.add_nodes([node]) + wf.base_dir = str(tmpdir) + + error_raised = False + try: + wf.run() + except RuntimeError as e: + error_raised = True assert error_raised From f290b7ef86375cd529004e6bf7812dfa9d989a0b Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sun, 18 Dec 2016 15:10:56 -0500 Subject: [PATCH 26/33] enh: use platform.node instead of getfqdn to improve DNS resolution times --- nipype/interfaces/base.py | 3 +-- nipype/utils/provenance.py | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nipype/interfaces/base.py b/nipype/interfaces/base.py index b9f66e1f3b..2e134842fd 100644 --- a/nipype/interfaces/base.py +++ b/nipype/interfaces/base.py @@ -22,7 +22,6 @@ import os import re import platform -from socket import getfqdn from string import Template import select import subprocess @@ -1079,7 +1078,7 @@ def run(self, **inputs): startTime=dt.isoformat(dt.utcnow()), endTime=None, platform=platform.platform(), - hostname=getfqdn(), + hostname=platform.node(), version=self.version) try: runtime = self._run_wrapper(runtime) diff --git a/nipype/utils/provenance.py b/nipype/utils/provenance.py index 0fdf72e02a..066cbb9a57 100644 --- a/nipype/utils/provenance.py +++ b/nipype/utils/provenance.py @@ -11,7 +11,7 @@ from pickle import dumps import os import getpass -from socket import getfqdn +import platform from uuid import uuid1 import simplejson as json @@ -133,7 +133,9 @@ def safe_encode(x, as_literal=True): if isinstance(x, bytes): x = str(x, 'utf-8') if os.path.exists(x): - value = 'file://{}{}'.format(getfqdn(), x) + if x[0] != os.pathsep: + x = os.path.abspath(x) + value = 'file://{}{}'.format(platform.node().lower(), x) if not as_literal: return value try: From 657094910e267bed327596b88c9c2b74d37fa685 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sun, 18 Dec 2016 15:12:03 -0500 Subject: [PATCH 27/33] fix: ensure io classes derived from IOBase but not in nipype allow appropriate connections --- nipype/pipeline/engine/tests/test_engine.py | 34 +++++++++++++++++++++ nipype/pipeline/engine/workflows.py | 9 ++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/nipype/pipeline/engine/tests/test_engine.py b/nipype/pipeline/engine/tests/test_engine.py index 76ed8af965..cc367e8c45 100644 --- a/nipype/pipeline/engine/tests/test_engine.py +++ b/nipype/pipeline/engine/tests/test_engine.py @@ -734,3 +734,37 @@ def test_deep_nested_write_graph_runs(tmpdir): os.remove('graph_detailed.dot') except OSError: pass + + +def test_io_subclass(): + """Ensure any io subclass allows dynamic traits""" + from nipype.interfaces.io import IOBase + from nipype.interfaces.base import DynamicTraitedSpec + + class TestKV(IOBase): + _always_run = True + output_spec = DynamicTraitedSpec + + def _list_outputs(self): + outputs = {} + outputs['test'] = 1 + outputs['foo'] = 'bar' + return outputs + + wf = pe.Workflow('testkv') + + def testx2(test): + return test * 2 + + kvnode = pe.Node(TestKV(), name='testkv') + from nipype.interfaces.utility import Function + func = pe.Node( + Function(input_names=['test'], output_names=['test2'], function=testx2), + name='func') + exception_not_raised = True + try: + wf.connect(kvnode, 'test', func, 'test') + except Exception as e: + if 'Module testkv has no output called test' in e: + exception_not_raised = False + assert exception_not_raised diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py index 2e8c7cae21..5a8c5eed56 100644 --- a/nipype/pipeline/engine/workflows.py +++ b/nipype/pipeline/engine/workflows.py @@ -200,11 +200,16 @@ def connect(self, *args, **kwargs): connected. """ % (srcnode, source, destnode, dest, dest, destnode)) if not (hasattr(destnode, '_interface') and - '.io' in str(destnode._interface.__class__)): + ('.io' in str(destnode._interface.__class__) or + any(['.io' in str(val) for val in + destnode._interface.__class__.__bases__])) + ): if not destnode._check_inputs(dest): not_found.append(['in', destnode.name, dest]) if not (hasattr(srcnode, '_interface') and - '.io' in str(srcnode._interface.__class__)): + ('.io' in str(srcnode._interface.__class__) + or any(['.io' in str(val) for val in + srcnode._interface.__class__.__bases__]))): if isinstance(source, tuple): # handles the case that source is specified # with a function From daba01cc3f46452c5c74e6393cc5ab411ed58d11 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sun, 18 Dec 2016 15:13:33 -0500 Subject: [PATCH 28/33] enh: updated dependencies and tests for nipype --- nipype/__init__.py | 57 ++++++++++++------------------------- nipype/info.py | 2 +- nipype/tests/__init__.py | 0 nipype/tests/test_nipype.py | 9 ++++++ requirements.txt | 3 +- rtd_requirements.txt | 3 +- 6 files changed, 30 insertions(+), 44 deletions(-) create mode 100644 nipype/tests/__init__.py create mode 100644 nipype/tests/test_nipype.py diff --git a/nipype/__init__.py b/nipype/__init__.py index c7ce79c99e..f761a8ef09 100644 --- a/nipype/__init__.py +++ b/nipype/__init__.py @@ -13,6 +13,7 @@ from .utils.config import NipypeConfig from .utils.logger import Logging from .refs import due +from .pkg_info import get_pkg_info as _get_pkg_info try: import faulthandler @@ -23,49 +24,27 @@ config = NipypeConfig() logging = Logging(config) -#NOTE_dj: it has to be changed to python -#class _NoseTester(nosetester.NoseTester): -# """ Subclass numpy's NoseTester to add doctests by default -# """ - -# def _get_custom_doctester(self): -# return None -# def test(self, label='fast', verbose=1, extra_argv=['--exe'], -# doctests=True, coverage=False): -# """Run the full test suite -# -# Examples -# -------- -# This will run the test suite and stop at the first failing -# example -# >>> from nipype import test -# >>> test(extra_argv=['--exe', '-sx']) # doctest: +SKIP -# """ -# return super(_NoseTester, self).test(label=label, -# verbose=verbose, -# extra_argv=extra_argv, -# doctests=doctests, -# coverage=coverage) +class NipypeTester(object): + def __call__(self, doctests=True): + try: + import pytest + except: + raise RuntimeError('py.test not installed, run: pip install pytest') + params = {'args': []} + if doctests: + params['args'].append('--doctest-modules') + nipype_path = os.path.dirname(__file__) + params['args'].extend(['-x', '--ignore={}/external'.format(nipype_path), + nipype_path]) + pytest.main(**params) -#try: -# test = _NoseTester(raise_warnings="release").test -#except TypeError: - # Older versions of numpy do not have a raise_warnings argument -# test = _NoseTester().test -#del nosetester - -# Set up package information function -from .pkg_info import get_pkg_info as _get_pkg_info -get_info = lambda: _get_pkg_info(os.path.dirname(__file__)) +test = NipypeTester() -# If this file is exec after being imported, the following lines will -# fail -#try: -# del Tester -#except: -# pass +def get_info(): + """Returns package information""" + return _get_pkg_info(os.path.dirname(__file__)) from .pipeline import Node, MapNode, JoinNode, Workflow from .interfaces import (DataGrabber, DataSink, SelectFiles, diff --git a/nipype/info.py b/nipype/info.py index ec60053b3e..d98faae782 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -146,11 +146,11 @@ def get_nipype_gitversion(): 'xvfbwrapper', 'funcsigs', 'configparser', + 'pytest>=%s' % PYTEST_MIN_VERSION ] TESTS_REQUIRES = [ 'pytest>=%s' % PYTEST_MIN_VERSION, - 'pytest-raisesregexp', 'pytest-cov', 'mock', 'codecov', diff --git a/nipype/tests/__init__.py b/nipype/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nipype/tests/test_nipype.py b/nipype/tests/test_nipype.py new file mode 100644 index 0000000000..5c1b714617 --- /dev/null +++ b/nipype/tests/test_nipype.py @@ -0,0 +1,9 @@ +from .. import get_info + +def test_nipype_info(): + exception_not_raised = True + try: + get_info() + except Exception as e: + exception_not_raised = False + assert exception_not_raised \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4da64e8f98..c06bfbfee5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ networkx>=1.7 traits>=4.3 python-dateutil>=1.5 nibabel>=2.0.1 -future==0.15.2 +future>=0.15.2 simplejson>=3.8.0 prov>=1.4.0 click>=6.6.0 @@ -13,5 +13,4 @@ psutil funcsigs configparser pytest>=3.0 -pytest-raisesregexp pytest-cov diff --git a/rtd_requirements.txt b/rtd_requirements.txt index 11b7fcfad1..a8b426ea8a 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -5,9 +5,8 @@ traits>=4.3 python-dateutil>=1.5 nibabel>=2.0.1 pytest>=3.0 -pytest-raisesregexp pytest-cov -future==0.15.2 +future>=0.15.2 simplejson>=3.8.0 prov>=1.4.0 xvfbwrapper From fe60dfc2043789c2276abfc3410a71cde66ce1c1 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sun, 18 Dec 2016 16:10:40 -0500 Subject: [PATCH 29/33] removed references to nose and fixed exception discrepancy between Py2 and Py3 --- nipype/pipeline/engine/nodes.py | 5 ++++- nipype/pipeline/engine/tests/__init__.py | 3 --- nipype/pipeline/engine/tests/test_utils.py | 4 +++- nipype/testing/README | 6 ------ nipype/testing/__init__.py | 14 ++------------ nipype/testing/utils.py | 13 ------------- 6 files changed, 9 insertions(+), 36 deletions(-) delete mode 100644 nipype/testing/README diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 551abd18ae..7daf26f09d 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -1129,7 +1129,10 @@ def _node_runner(self, nodes, updatehash=False): try: node.run(updatehash=updatehash) except Exception as e: - err = str(e) + if sys.version < 3: + err = e + else: + err = str(e) if str2bool(self.config['execution']['stop_on_first_crash']): raise finally: diff --git a/nipype/pipeline/engine/tests/__init__.py b/nipype/pipeline/engine/tests/__init__.py index 81bf04cc92..99fb243f19 100644 --- a/nipype/pipeline/engine/tests/__init__.py +++ b/nipype/pipeline/engine/tests/__init__.py @@ -1,6 +1,3 @@ # -*- coding: utf-8 -*- # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: - -from nipype.testing import skip_if_no_package -skip_if_no_package('networkx', '1.0') diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 29b7871445..5e629df5ab 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -384,6 +384,7 @@ def myfunction(string): def test_mapnode_crash3(tmpdir): """Test mapnode crash when mapnode is embedded in a workflow""" + def myfunction(string): return string + 'meh' node = pe.MapNode(niu.Function(input_names=['WRONG'], @@ -393,7 +394,8 @@ def myfunction(string): name='myfunc') node.inputs.WRONG = ['string' + str(i) for i in range(3)] - wf = pe.Workflow('test_mapnode_crash') + wf = pe.Workflow('testmapnodecrash') + wf.config['crashdump_dir'] = str(tmpdir) wf.add_nodes([node]) wf.base_dir = str(tmpdir) diff --git a/nipype/testing/README b/nipype/testing/README deleted file mode 100644 index 6a6d0dca7b..0000000000 --- a/nipype/testing/README +++ /dev/null @@ -1,6 +0,0 @@ -The numpytesting directory contains a copy of all the files from -numpy/testing for numpy version 1.3. This provides all the test -integration with the nose test framework we need to run the nipype -tests. By including these files, nipype can now run on systems that -only have numpy 1.1, like Debian Lenny. This feature was added by -Yaroslav Halchenko. diff --git a/nipype/testing/__init__.py b/nipype/testing/__init__.py index 996fb0ac01..74217aacf8 100644 --- a/nipype/testing/__init__.py +++ b/nipype/testing/__init__.py @@ -2,17 +2,7 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """The testing directory contains a small set of imaging files to be -used for doctests only. More thorough tests and example data will be -stored in a nipy data packages that you can download separately. - -.. note: - - We use the ``nose`` testing framework for tests. - - Nose is a dependency for the tests, but should not be a dependency - for running the algorithms in the NIPY library. This file should - import without nose being present on the python path. - +used for doctests only. """ import os @@ -28,7 +18,7 @@ from . import decorators as dec -from .utils import skip_if_no_package, package_check, TempFATFS +from .utils import package_check, TempFATFS skipif = dec.skipif diff --git a/nipype/testing/utils.py b/nipype/testing/utils.py index 95d8045b78..7c03ca6d04 100644 --- a/nipype/testing/utils.py +++ b/nipype/testing/utils.py @@ -13,7 +13,6 @@ import subprocess from subprocess import CalledProcessError from tempfile import mkdtemp -from nose import SkipTest from future.utils import raise_from from ..utils.misc import package_check @@ -22,18 +21,6 @@ import numpy as np import nibabel as nb -def skip_if_no_package(*args, **kwargs): - """Raise SkipTest if package_check fails - - Parameters - ---------- - *args Positional parameters passed to `package_check` - *kwargs Keyword parameters passed to `package_check` - """ - package_check(exc_failed_import=SkipTest, - exc_failed_check=SkipTest, - *args, **kwargs) - class TempFATFS(object): def __init__(self, size_in_mbytes=8, delay=0.5): From 49d1a95f8e82da640716f527f4b7cea4841f5323 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sun, 18 Dec 2016 17:06:04 -0500 Subject: [PATCH 30/33] simplified test construct --- nipype/pipeline/engine/nodes.py | 7 ++----- nipype/pipeline/engine/tests/test_utils.py | 24 +++++----------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 7daf26f09d..fcbcca7c4c 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -1128,11 +1128,8 @@ def _node_runner(self, nodes, updatehash=False): err = None try: node.run(updatehash=updatehash) - except Exception as e: - if sys.version < 3: - err = e - else: - err = str(e) + except Exception as this_err: + err = this_err if str2bool(self.config['execution']['stop_on_first_crash']): raise finally: diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 5e629df5ab..221d99395c 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -9,6 +9,7 @@ import os from copy import deepcopy from shutil import rmtree +import pytest from ... import engine as pe from ....interfaces import base as nib @@ -352,13 +353,8 @@ def myfunction(string): node.config = deepcopy(config._sections) node.config['execution']['stop_on_first_crash'] = True node.base_dir = str(tmpdir) - - error_raised = False - try: + with pytest.raises(TypeError): node.run() - except TypeError as e: - error_raised = True - assert error_raised def test_mapnode_crash2(tmpdir): @@ -374,12 +370,8 @@ def myfunction(string): node.inputs.WRONG = ['string' + str(i) for i in range(3)] node.base_dir = str(tmpdir) - error_raised = False - try: + with pytest.raises(Exception): node.run() - except Exception as e: - error_raised = True - assert error_raised def test_mapnode_crash3(tmpdir): @@ -395,13 +387,7 @@ def myfunction(string): node.inputs.WRONG = ['string' + str(i) for i in range(3)] wf = pe.Workflow('testmapnodecrash') - wf.config['crashdump_dir'] = str(tmpdir) wf.add_nodes([node]) wf.base_dir = str(tmpdir) - - error_raised = False - try: - wf.run() - except RuntimeError as e: - error_raised = True - assert error_raised + with pytest.raises(RuntimeError): + wf.run(plugin='Linear') From 8ada360fb6b7b4b65f68789874df1c2ef1ebf401 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 20 Dec 2016 22:28:25 -0500 Subject: [PATCH 31/33] ensure node crashes switch back directory --- nipype/pipeline/engine/tests/test_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 221d99395c..1d892606dd 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -338,9 +338,9 @@ def test_provenance(tmpdir): assert len(psg.bundles) == 2 assert len(psg.get_records()) == 7 - def test_mapnode_crash(tmpdir): """Test mapnode crash when stop_on_first_crash is True""" + cwd = os.getcwd() def myfunction(string): return string + 'meh' node = pe.MapNode(niu.Function(input_names=['WRONG'], @@ -348,30 +348,31 @@ def myfunction(string): function=myfunction), iterfield=['WRONG'], name='myfunc') - node.inputs.WRONG = ['string' + str(i) for i in range(3)] node.config = deepcopy(config._sections) node.config['execution']['stop_on_first_crash'] = True node.base_dir = str(tmpdir) with pytest.raises(TypeError): node.run() - + os.chdir(cwd) def test_mapnode_crash2(tmpdir): """Test mapnode crash when stop_on_first_crash is False""" + cwd = os.getcwd() def myfunction(string): return string + 'meh' + node = pe.MapNode(niu.Function(input_names=['WRONG'], output_names=['newstring'], function=myfunction), iterfield=['WRONG'], name='myfunc') - node.inputs.WRONG = ['string' + str(i) for i in range(3)] node.base_dir = str(tmpdir) with pytest.raises(Exception): node.run() + os.chdir(cwd) def test_mapnode_crash3(tmpdir): @@ -384,7 +385,6 @@ def myfunction(string): function=myfunction), iterfield=['WRONG'], name='myfunc') - node.inputs.WRONG = ['string' + str(i) for i in range(3)] wf = pe.Workflow('testmapnodecrash') wf.add_nodes([node]) From 2ec0b0fd688c62d953983b8f20566b23e77238ff Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Tue, 20 Dec 2016 22:32:42 -0500 Subject: [PATCH 32/33] change to simplify debugging --- nipype/pipeline/engine/nodes.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index fcbcca7c4c..2f03843f85 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -1275,9 +1275,12 @@ def _run_interface(self, execute=True, updatehash=False): nitems = len(filename_to_list(getattr(self.inputs, self.iterfield[0]))) nodenames = ['_' + self.name + str(i) for i in range(nitems)] + nodes = list(self._make_nodes(cwd)) + node_results = list(self._node_runner(nodes)) + self._collate_results(node_results) # map-reduce formulation - self._collate_results(self._node_runner(self._make_nodes(cwd), - updatehash=updatehash)) + #self._collate_results(self._node_runner(self._make_nodes(cwd), + # updatehash=updatehash)) self._save_results(self._result, cwd) # remove any node directories no longer required dirs2remove = [] From 75300a5d2b47b50477084ab23459758f0e5afb78 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 24 Dec 2016 11:10:16 -0500 Subject: [PATCH 33/33] fix: tests for mapnode crash --- nipype/pipeline/engine/nodes.py | 14 +++++------- nipype/pipeline/engine/tests/test_utils.py | 26 ++++++++++------------ 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 2f03843f85..efaa286e94 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -36,7 +36,7 @@ from hashlib import sha1 from ... import config, logging -from ...utils.misc import (flatten, unflatten, package_check, str2bool) +from ...utils.misc import (flatten, unflatten, str2bool) from ...utils.filemanip import (save_json, FileNotFoundError, filename_to_list, list_to_filename, copyfiles, fnames_presuffix, loadpkl, @@ -54,7 +54,6 @@ get_print_name, merge_dict, evaluate_connect_function) from .base import EngineBase -package_check('networkx', '1.3') logger = logging.getLogger('workflow') class Node(EngineBase): @@ -607,6 +606,7 @@ def _run_command(self, execute, copyfiles=True): try: result = self._interface.run() except Exception as msg: + self._save_results(result, cwd) self._result.runtime.stderr = msg raise @@ -1124,6 +1124,7 @@ def _make_nodes(self, cwd=None): yield i, node def _node_runner(self, nodes, updatehash=False): + old_cwd = os.getcwd() for i, node in nodes: err = None try: @@ -1133,6 +1134,7 @@ def _node_runner(self, nodes, updatehash=False): if str2bool(self.config['execution']['stop_on_first_crash']): raise finally: + os.chdir(old_cwd) yield i, node, err def _collate_results(self, nodes): @@ -1275,12 +1277,8 @@ def _run_interface(self, execute=True, updatehash=False): nitems = len(filename_to_list(getattr(self.inputs, self.iterfield[0]))) nodenames = ['_' + self.name + str(i) for i in range(nitems)] - nodes = list(self._make_nodes(cwd)) - node_results = list(self._node_runner(nodes)) - self._collate_results(node_results) - # map-reduce formulation - #self._collate_results(self._node_runner(self._make_nodes(cwd), - # updatehash=updatehash)) + self._collate_results(self._node_runner(self._make_nodes(cwd), + updatehash=updatehash)) self._save_results(self._result, cwd) # remove any node directories no longer required dirs2remove = [] diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 1d892606dd..f755ebc886 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -338,17 +338,20 @@ def test_provenance(tmpdir): assert len(psg.bundles) == 2 assert len(psg.get_records()) == 7 + +def dummy_func(value): + return value + 1 + + def test_mapnode_crash(tmpdir): """Test mapnode crash when stop_on_first_crash is True""" cwd = os.getcwd() - def myfunction(string): - return string + 'meh' node = pe.MapNode(niu.Function(input_names=['WRONG'], output_names=['newstring'], - function=myfunction), + function=dummy_func), iterfield=['WRONG'], name='myfunc') - node.inputs.WRONG = ['string' + str(i) for i in range(3)] + node.inputs.WRONG = ['string{}'.format(i) for i in range(3)] node.config = deepcopy(config._sections) node.config['execution']['stop_on_first_crash'] = True node.base_dir = str(tmpdir) @@ -356,18 +359,16 @@ def myfunction(string): node.run() os.chdir(cwd) + def test_mapnode_crash2(tmpdir): """Test mapnode crash when stop_on_first_crash is False""" cwd = os.getcwd() - def myfunction(string): - return string + 'meh' - node = pe.MapNode(niu.Function(input_names=['WRONG'], output_names=['newstring'], - function=myfunction), + function=dummy_func), iterfield=['WRONG'], name='myfunc') - node.inputs.WRONG = ['string' + str(i) for i in range(3)] + node.inputs.WRONG = ['string{}'.format(i) for i in range(3)] node.base_dir = str(tmpdir) with pytest.raises(Exception): @@ -377,15 +378,12 @@ def myfunction(string): def test_mapnode_crash3(tmpdir): """Test mapnode crash when mapnode is embedded in a workflow""" - - def myfunction(string): - return string + 'meh' node = pe.MapNode(niu.Function(input_names=['WRONG'], output_names=['newstring'], - function=myfunction), + function=dummy_func), iterfield=['WRONG'], name='myfunc') - node.inputs.WRONG = ['string' + str(i) for i in range(3)] + node.inputs.WRONG = ['string{}'.format(i) for i in range(3)] wf = pe.Workflow('testmapnodecrash') wf.add_nodes([node]) wf.base_dir = str(tmpdir)