From c451b69455838d8f8f84cf737967ac25090fabc7 Mon Sep 17 00:00:00 2001 From: dPys Date: Mon, 6 Jan 2020 12:33:00 -0600 Subject: [PATCH 1/5] [FIX] immunize shutil.rmtree to node non-existence for remove_node_directories=True in the case that stop_on_first_crash=False *Add regression test --- nipype/pipeline/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/base.py b/nipype/pipeline/plugins/base.py index 4be8eb232b..599db29418 100644 --- a/nipype/pipeline/plugins/base.py +++ b/nipype/pipeline/plugins/base.py @@ -461,7 +461,7 @@ def _remove_node_dirs(self): ) % (self.procs[idx]._id, outdir) ) - shutil.rmtree(outdir) + shutil.rmtree(outdir, ignore_errors=True) class SGELikeBatchManagerBase(DistributedPluginBase): From b6474db3ff6a662e495ecd396bd5a055f7ee9249 Mon Sep 17 00:00:00 2001 From: dPys Date: Mon, 6 Jan 2020 18:39:57 -0600 Subject: [PATCH 2/5] Add reg test --- nipype/conftest.py | 1 - nipype/pipeline/plugins/base.py | 2 +- nipype/pipeline/plugins/tests/test_base.py | 34 +++++++++++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/nipype/conftest.py b/nipype/conftest.py index b099fd0078..52b2f84c49 100644 --- a/nipype/conftest.py +++ b/nipype/conftest.py @@ -12,7 +12,6 @@ data_dir = os.path.join(temp_folder, "data") shutil.copytree(NIPYPE_DATADIR, data_dir) - @pytest.fixture(autouse=True) def add_np(doctest_namespace): doctest_namespace["np"] = numpy diff --git a/nipype/pipeline/plugins/base.py b/nipype/pipeline/plugins/base.py index 599db29418..4be8eb232b 100644 --- a/nipype/pipeline/plugins/base.py +++ b/nipype/pipeline/plugins/base.py @@ -461,7 +461,7 @@ def _remove_node_dirs(self): ) % (self.procs[idx]._id, outdir) ) - shutil.rmtree(outdir, ignore_errors=True) + shutil.rmtree(outdir) class SGELikeBatchManagerBase(DistributedPluginBase): diff --git a/nipype/pipeline/plugins/tests/test_base.py b/nipype/pipeline/plugins/tests/test_base.py index fddcfa2368..b53429ee0a 100644 --- a/nipype/pipeline/plugins/tests/test_base.py +++ b/nipype/pipeline/plugins/tests/test_base.py @@ -3,10 +3,11 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """Tests for the engine module """ +import pytest +from pathlib import Path import numpy as np import scipy.sparse as ssp - def test_scipy_sparse(): foo = ssp.lil_matrix(np.eye(3, k=1)) goo = foo.getrowview(0) @@ -39,3 +40,34 @@ def func(arg1): wf.run(plugin='MultiProc') """ + +@pytest.mark.regression +def test_remove_nodes(tmpdir): + import os + import nipype.interfaces.utility as niu + import nipype.pipeline.engine as pe + + wf = pe.Workflow(name='test') + + def func(arg1): + try: + if arg1 == 2: + raise Exception('arg cannot be ' + str(arg1)) + except: + pass + return arg1 + + funkynode = pe.MapNode(niu.Function(function=func, input_names=['arg1'], + output_names=['out']), + iterfield=['arg1'], + name = 'functor') + funkynode.inputs.arg1 = [1,2] + + wf.add_nodes([funkynode]) + wf.base_dir = tmpdir + wf.config['execution']['remove_node_directories'] = 'true' + wf.config['execution']['stop_on_first_crash'] = 'false' + wf.config['logging']['crashdump_dir'] = tmpdir + res = wf.run(plugin='MultiProc') + + assert os.path.isdir(Path(tmpdir)/'test/functor') is False From a6f03b5c7618ab3f4524178c34a63fa2526664ca Mon Sep 17 00:00:00 2001 From: Derek Pisner <16432683+dPys@users.noreply.github.com> Date: Tue, 7 Jan 2020 09:08:54 -0600 Subject: [PATCH 3/5] Update nipype/pipeline/plugins/tests/test_base.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_base.py b/nipype/pipeline/plugins/tests/test_base.py index b53429ee0a..4796b45a36 100644 --- a/nipype/pipeline/plugins/tests/test_base.py +++ b/nipype/pipeline/plugins/tests/test_base.py @@ -42,7 +42,7 @@ def func(arg1): """ @pytest.mark.regression -def test_remove_nodes(tmpdir): +def test_remove_nodes(tmp_path): import os import nipype.interfaces.utility as niu import nipype.pipeline.engine as pe From 94000dde6d45a58411b335644e8393288cf20033 Mon Sep 17 00:00:00 2001 From: Derek Pisner <16432683+dPys@users.noreply.github.com> Date: Tue, 7 Jan 2020 09:09:01 -0600 Subject: [PATCH 4/5] Update nipype/pipeline/plugins/tests/test_base.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_base.py b/nipype/pipeline/plugins/tests/test_base.py index 4796b45a36..226ee6941f 100644 --- a/nipype/pipeline/plugins/tests/test_base.py +++ b/nipype/pipeline/plugins/tests/test_base.py @@ -67,7 +67,7 @@ def func(arg1): wf.base_dir = tmpdir wf.config['execution']['remove_node_directories'] = 'true' wf.config['execution']['stop_on_first_crash'] = 'false' - wf.config['logging']['crashdump_dir'] = tmpdir + wf.config['logging']['crashdump_dir'] = str(tmp_path) res = wf.run(plugin='MultiProc') assert os.path.isdir(Path(tmpdir)/'test/functor') is False From b473b6be0b5c9726b479b1c6b18cb6ca2f36874d Mon Sep 17 00:00:00 2001 From: Derek Pisner <16432683+dPys@users.noreply.github.com> Date: Tue, 7 Jan 2020 09:09:08 -0600 Subject: [PATCH 5/5] Update nipype/pipeline/plugins/tests/test_base.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_base.py b/nipype/pipeline/plugins/tests/test_base.py index 226ee6941f..a1d62ce235 100644 --- a/nipype/pipeline/plugins/tests/test_base.py +++ b/nipype/pipeline/plugins/tests/test_base.py @@ -70,4 +70,4 @@ def func(arg1): wf.config['logging']['crashdump_dir'] = str(tmp_path) res = wf.run(plugin='MultiProc') - assert os.path.isdir(Path(tmpdir)/'test/functor') is False + assert not Path.is_dir(tmp_path / "test" / "functor")