From f568d782f06c8e6a26f735bd1d4784bc7a70f574 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 11 Jul 2018 22:55:57 -0400
Subject: [PATCH 01/55] Adding auxiliary and state from nipype2_tmp

---
 nipype/pipeline/engine/auxiliary.py           | 206 ++++++++++++++++++
 nipype/pipeline/engine/state.py               |  85 ++++++++
 .../pipeline/engine/tests/test_auxiliary.py   |  73 +++++++
 3 files changed, 364 insertions(+)
 create mode 100644 nipype/pipeline/engine/auxiliary.py
 create mode 100644 nipype/pipeline/engine/state.py
 create mode 100644 nipype/pipeline/engine/tests/test_auxiliary.py

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
new file mode 100644
index 0000000000..b477801004
--- /dev/null
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -0,0 +1,206 @@
+import pdb
+import inspect
+from .. import config, logging
+logger = logging.getLogger('workflow')
+
+
+# Function to change user provided mapper to "reverse polish notation" used in State
+def mapper2rpn(mapper):
+    """ Functions that translate mapper to "reverse polish notation."""
+    global output_mapper
+    output_mapper = []
+    _ordering(mapper, i=0)
+    return output_mapper
+
+
+def _ordering(el, i, current_sign=None):
+    """ Used in the mapper2rpn to get a proper order of fields and signs. """
+    global output_mapper
+    if type(el) is tuple:
+        _iterate_list(el, ".")
+    elif type(el) is list:
+        _iterate_list(el, "*")
+    elif type(el) is str:
+        output_mapper.append(el)
+    else:
+        raise Exception("mapper has to be a string, a tuple or a list")
+    
+    if i > 0:
+        output_mapper.append(current_sign)
+
+
+def _iterate_list(element, sign):
+    """ Used in the mapper2rpn to get recursion. """
+    for i, el in enumerate(element):
+        _ordering(el, i, current_sign=sign)
+
+
+# functions used in State to know which element should be used for a specific axis
+
+def mapping_axis(state_inputs, mapper_rpn):
+    """Having inputs and mapper (in rpn notation), functions returns the axes of output for every input."""
+    axis_for_input = {}
+    stack = []
+    current_axis = None
+    current_shape = None
+
+    for el in mapper_rpn:
+        if el == ".":
+            right = stack.pop()
+            left = stack.pop()
+            if left == "OUT":
+                if state_inputs[right].shape == current_shape: #todo:should we allow for one-element array? 
+                    axis_for_input[right] = current_axis
+                else:
+                    raise Exception("arrays for scalar operations should have the same size")
+
+            elif right == "OUT":
+                if state_inputs[left].shape == current_shape:
+                    axis_for_input[left] = current_axis
+                else:
+                    raise Exception("arrays for scalar operations should have the same size")
+
+            else:
+                if state_inputs[right].shape == state_inputs[left].shape:
+                    current_axis = list(range(state_inputs[right].ndim))
+                    current_shape = state_inputs[left].shape
+                    axis_for_input[left] = current_axis
+                    axis_for_input[right] = current_axis
+                else:
+                    raise Exception("arrays for scalar operations should have the same size")
+                
+            stack.append("OUT")
+
+        elif el == "*":
+            right = stack.pop()
+            left = stack.pop()
+            if left == "OUT":
+                axis_for_input[right] = [i + 1 + current_axis[-1]
+                                         for i in range(state_inputs[right].ndim)]
+                current_axis = current_axis + axis_for_input[right]
+                current_shape = tuple([i for i in current_shape + state_inputs[right].shape])
+            elif right == "OUT":
+                for key in axis_for_input:
+                    axis_for_input[key] = [i + state_inputs[left].ndim
+                                           for i in axis_for_input[key]]
+
+                axis_for_input[left] = [i - len(current_shape) + current_axis[-1] + 1
+                                        for i in range(state_inputs[left].ndim)]
+                current_axis = current_axis + [i + 1 + current_axis[-1]
+                                               for i in range(state_inputs[left].ndim)]
+                current_shape = tuple([i for i in state_inputs[left].shape + current_shape])
+            else:
+                axis_for_input[left] = list(range(state_inputs[left].ndim))
+                axis_for_input[right] = [i + state_inputs[left].ndim
+                                         for i in range(state_inputs[right].ndim)]
+                current_axis = axis_for_input[left] + axis_for_input[right]
+                current_shape = tuple([i for i in 
+                                       state_inputs[left].shape + state_inputs[right].shape])
+            stack.append("OUT")
+
+        else:
+            stack.append(el)
+
+    if len(stack) == 0:
+        pass
+    elif len(stack) > 1:
+        raise Exception("exception from mapping_axis")
+    elif stack[0] != "OUT":
+        current_axis = [i for i in range(state_inputs[stack[0]].ndim)]
+        axis_for_input[stack[0]] = current_axis
+
+    if current_axis:
+        ndim = max(current_axis) + 1
+    else:
+        ndim = 0
+    return axis_for_input, ndim
+
+
+def converting_axis2input(state_inputs, axis_for_input, ndim):
+    """ Having axes for all the input fields, the function returns fields for each axis. """
+    input_for_axis = []
+    shape = []
+    for i in range(ndim):
+        input_for_axis.append([])
+        shape.append(0)
+        
+    for inp, axis in axis_for_input.items():
+        for (i, ax) in enumerate(axis):
+            input_for_axis[ax].append(inp)
+            shape[ax] = state_inputs[inp].shape[i]
+            
+    return input_for_axis, shape
+
+
+# used in the Node to change names in a mapper
+
+def change_mapper(mapper, name):
+    """changing names of mapper: adding names of the node"""
+    if isinstance(mapper, str):
+        if "-" in mapper:
+            return mapper
+        else:
+            return "{}-{}".format(name, mapper)
+    elif isinstance(mapper, list):
+        return _add_name(mapper, name)
+    elif isinstance(mapper, tuple):
+        mapper_l = list(mapper)
+        return tuple(_add_name(mapper_l, name))
+
+
+def _add_name(mlist, name):
+    for i, elem in enumerate(mlist):
+        if isinstance(elem, str):
+            if "-" in elem:
+                pass
+            else:
+                mlist[i] = "{}-{}".format(name, mlist[i])
+        elif isinstance(elem, list):
+            mlist[i] = _add_name(elem, name)
+        elif isinstance(elem, tuple):
+            mlist[i] = list(elem)
+            mlist[i] = _add_name(mlist[i], name)
+            mlist[i] = tuple(mlist[i])
+    return mlist
+
+
+#Function interface
+
+class Function_Interface(object):
+    """ A new function interface """
+    def __init__(self, function, output_nm, input_map=None):
+        self.function = function
+        if type(output_nm) is list:
+            self._output_nm = output_nm
+        else:
+            raise Exception("output_nm should be a list")
+        if not input_map:
+            self.input_map = {}
+        # TODO use signature
+        for key in inspect.getargspec(function)[0]:
+            if key not in self.input_map.keys():
+                self.input_map[key] = key
+
+
+    def run(self, input):
+        self.output = {}
+        if self.input_map:
+            for (key_fun, key_inp) in self.input_map.items():
+                try:
+                    input[key_fun] = input.pop(key_inp)
+                except KeyError:
+                    raise Exception("no {} in the input dictionary".format(key_inp))
+        fun_output = self.function(**input)
+        logger.debug("Function Interf, input={}, fun_out={}".format(input, fun_output))
+        if type(fun_output) is tuple:
+            if len(self._output_nm) == len(fun_output):
+                for i, out in enumerate(fun_output):
+                    self.output[self._output_nm[i]] = out
+            else:
+                raise Exception("length of output_nm doesnt match length of the function output")
+        elif len(self._output_nm)==1:
+            self.output[self._output_nm[0]] = fun_output
+        else:
+            raise Exception("output_nm doesnt match length of the function output")
+
+        return fun_output
diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
new file mode 100644
index 0000000000..68d82137f0
--- /dev/null
+++ b/nipype/pipeline/engine/state.py
@@ -0,0 +1,85 @@
+from collections import OrderedDict
+
+from . import auxiliary as aux
+
+class State(object):
+    def __init__(self, state_inputs, node_name, mapper=None):
+        self.state_inputs = state_inputs
+
+        self._mapper = mapper
+        self.node_name = node_name
+        if self._mapper:
+            # changing mapper (as in rpn), so I can read from left to right
+            # e.g. if mapper=('d', ['e', 'r']), _mapper_rpn=['d', 'e', 'r', '*', '.']
+            self._mapper_rpn = aux.mapper2rpn(self._mapper)
+            self._input_names_mapper = [i for i in self._mapper_rpn if i not in ["*", "."]]
+        else:
+            self._mapper_rpn = []
+            self._input_names_mapper = []
+        # not all input field have to be use in the mapper, can be an extra scalar
+        self._input_names = list(self.state_inputs.keys())
+
+        # dictionary[key=input names] = list of axes related to
+        # e.g. {'r': [1], 'e': [0], 'd': [0, 1]}
+        # ndim - int, number of dimension for the "final array" (that is not created)
+        self._axis_for_input, self._ndim = aux.mapping_axis(self.state_inputs, self._mapper_rpn)
+
+        # list of inputs variable for each axis
+        # e.g. [['e', 'd'], ['r', 'd']]
+        # shape - list, e.g. [2,3]
+        self._input_for_axis, self._shape = aux.converting_axis2input(self.state_inputs,
+                                                                      self._axis_for_input, self._ndim)
+
+        # list of all possible indexes in each dim, will be use to iterate
+        # e.g. [[0, 1], [0, 1, 2]]
+        self._all_elements = [range(i) for i in self._shape]
+
+
+    def __getitem__(self, key):
+        if type(key) is int:
+            key = (key,)
+        return self.state_values(key)
+
+    @property
+    def all_elements(self):
+        return self._all_elements
+
+    # not used?
+    #@property
+    #def mapper(self):
+    #    return self._mapper
+
+
+    @property
+    def ndim(self):
+        return self._ndim
+
+
+    @property
+    def shape(self):
+        return self._shape
+
+
+    def state_values(self, ind):
+        if len(ind) > self._ndim:
+            raise IndexError("too many indices")
+
+        for ii, index in enumerate(ind):
+            if index > self._shape[ii] - 1:
+                raise IndexError("index {} is out of bounds for axis {} with size {}".format(index, ii, self._shape[ii]))
+
+        state_dict = {}
+        for input, ax in self._axis_for_input.items():
+            # checking which axes are important for the input
+            sl_ax = slice(ax[0], ax[-1]+1)
+            # taking the indexes for the axes
+            ind_inp = ind[sl_ax]
+            state_dict[input] = self.state_inputs[input][ind_inp]
+
+        # adding values from input that are not used in the mapper
+        for input in set(self._input_names) - set(self._input_names_mapper):
+            state_dict[input] = self.state_inputs[input]
+
+        # in py3.7 we can skip OrderedDict
+        # returning a named tuple?
+        return OrderedDict(sorted(state_dict.items(), key=lambda t: t[0]))
\ No newline at end of file
diff --git a/nipype/pipeline/engine/tests/test_auxiliary.py b/nipype/pipeline/engine/tests/test_auxiliary.py
new file mode 100644
index 0000000000..d5d3ffb536
--- /dev/null
+++ b/nipype/pipeline/engine/tests/test_auxiliary.py
@@ -0,0 +1,73 @@
+from .. import auxiliary as aux
+
+import numpy as np
+import pytest
+
+@pytest.mark.parametrize("mapper, rpn",
+                        [
+                            ("a",                      ["a"]),
+                            (("a", "b"),               ["a", "b", "."]),
+                            (["a", "b"],               ["a", "b", "*"]),
+                            (["a", ("b", "c")],        ["a", "b", "c", ".", "*"]),
+                            ([("a", "b"), "c"],        ["a", "b", ".", "c", "*"]),
+                            (["a", ("b", ["c", "d"])], ["a", "b", "c", "d", "*", ".", "*"])
+                        ])
+def test_mapper2rpn(mapper, rpn):
+    assert aux.mapper2rpn(mapper) == rpn
+
+
+@pytest.mark.parametrize("mapper, mapper_changed",
+                        [
+                            ("a",               "Node-a"),
+                            (["a", ("b", "c")], ["Node-a", ("Node-b", "Node-c")]),
+                            (("a", ["b", "c"]), ("Node-a", ["Node-b", "Node-c"]))
+                        ])
+def test_change_mapper(mapper, mapper_changed):
+    assert aux.change_mapper(mapper, "Node") == mapper_changed
+
+
+@pytest.mark.parametrize("inputs, rpn, expected",
+                         [
+                             ({"a": np.array([1, 2])}, ["a"], {"a": [0]}),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4])}, ["a", "b", "."], {"a": [0], "b": [0]}),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4, 1])}, ["a", "b", "*"], {"a": [0], "b": [1]}),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4]), "c": np.array([1, 2, 3])}, ["a", "b", ".", "c", "*"],
+                                                        {"a": [0], "b": [0], "c": [1]}),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4]), "c": np.array([1, 2, 3])},
+                              ["c", "a", "b", ".", "*"], {"a": [1], "b": [1], "c": [0]}),
+                             ({"a": np.array([[1, 2], [1, 2]]), "b": np.array([[3, 4], [3, 3]]), "c": np.array([1, 2, 3])},
+                                                        ["a", "b", ".", "c", "*"], {"a": [0, 1], "b": [0, 1], "c": [2]}),
+                             ({"a": np.array([[1, 2], [1, 2]]), "b": np.array([[3, 4], [3, 3]]),
+                               "c": np.array([1, 2, 3])}, ["c", "a", "b", ".", "*"], {"a": [1, 2], "b": [1, 2], "c": [0]})
+                         ])
+def test_mapping_axis(inputs, rpn, expected):
+    res = aux.mapping_axis(inputs, rpn)[0]
+    print(res)
+    for key in inputs.keys():
+        assert res[key] == expected[key]
+
+
+def test_mapping_axis_error():
+    with pytest.raises(Exception):
+        aux.mapping_axis({"a": np.array([1, 2]), "b": np.array([3, 4, 5])}, ["a", "b", "."])
+
+
+@pytest.mark.parametrize("inputs, axis_inputs, ndim, expected",
+                         [
+                             ({"a": np.array([1, 2])}, {"a": [0]}, 1, [["a"]]),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4])}, {"a": [0], "b": [0]}, 1,
+                              [["a", "b"]]),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4, 1])}, {"a": [0], "b": [1]}, 2,
+                              [["a"], ["b"]]),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4]), "c": np.array([1, 2, 3])},
+                                                        {"a": [0], "b": [0], "c": [1]}, 2, [["a", "b"]]),
+                             ({"a": np.array([1, 2]), "b": np.array([3, 4]), "c": np.array([1, 2, 3])},
+                              {"a": [1], "b": [1], "c": [0]}, 2, [["c"], ["a", "b"]]),
+                             ({"a": np.array([[1, 2], [1, 2]]), "b": np.array([[3, 4], [3, 3]]), "c": np.array([1, 2, 3])},
+                                                {"a": [0, 1], "b": [0, 1], "c": [2]}, 3, [["a", "b"], ["a", "b"], ["c"]]),
+                             ({"a": np.array([[1, 2], [1, 2]]), "b": np.array([[3, 4], [3, 3]]),
+                               "c": np.array([1, 2, 3])}, {"a": [1, 2], "b": [1, 2], "c": [0]}, 3,
+                              [["c"], ["a", "b"], ["a", "b"]])
+                         ])
+def test_converting_axis2input(inputs, axis_inputs, ndim, expected):
+    aux.converting_axis2input(inputs, axis_inputs, ndim)[0] == expected
\ No newline at end of file

From 682248a7f31e51a85541b8355e4c6f58a10a53cb Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Fri, 13 Jul 2018 12:00:16 -0400
Subject: [PATCH 02/55] updating exceptions and __init__; updating logger to
 the current version in nipype

---
 nipype/exceptions.py                | 4 ++++
 nipype/pipeline/engine/__init__.py  | 2 +-
 nipype/pipeline/engine/auxiliary.py | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/nipype/exceptions.py b/nipype/exceptions.py
index 19880a22ab..bb914cc4e0 100644
--- a/nipype/exceptions.py
+++ b/nipype/exceptions.py
@@ -2,6 +2,10 @@ class NipypeError(Exception):
     pass
 
 
+class EngineError(Exception):
+    pass
+
+
 class PipelineError(NipypeError):
     pass
 
diff --git a/nipype/pipeline/engine/__init__.py b/nipype/pipeline/engine/__init__.py
index e950086307..2a8ca8850f 100644
--- a/nipype/pipeline/engine/__init__.py
+++ b/nipype/pipeline/engine/__init__.py
@@ -9,6 +9,6 @@
 
 from __future__ import absolute_import
 __docformat__ = 'restructuredtext'
-from .workflows import Workflow
+from .workflows import Workflow, NewNode, NewWorkflow
 from .nodes import Node, MapNode, JoinNode
 from .utils import generate_expanded_graph
diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index b477801004..16e15f9906 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -1,7 +1,7 @@
 import pdb
 import inspect
-from .. import config, logging
-logger = logging.getLogger('workflow')
+from ... import config, logging
+logger = logging.getLogger('nipype.workflow')
 
 
 # Function to change user provided mapper to "reverse polish notation" used in State

From 7b3bba66a87135177d9b90b6c54be6aeb14baca3 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Fri, 13 Jul 2018 12:05:19 -0400
Subject: [PATCH 03/55] starting updating NewNode using mapper and state from
 nipype2_tmp; changing State.state_value so it returns values (and not arrays)

---
 nipype/pipeline/engine/state.py     |  6 +--
 nipype/pipeline/engine/workflows.py | 66 +++++++++++++++++++++++++++--
 2 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
index 68d82137f0..ec96dab04c 100644
--- a/nipype/pipeline/engine/state.py
+++ b/nipype/pipeline/engine/state.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import pdb
 
 from . import auxiliary as aux
 
@@ -73,13 +74,12 @@ def state_values(self, ind):
             # checking which axes are important for the input
             sl_ax = slice(ax[0], ax[-1]+1)
             # taking the indexes for the axes
-            ind_inp = ind[sl_ax]
+            ind_inp = tuple(ind[sl_ax]) #used to be list
             state_dict[input] = self.state_inputs[input][ind_inp]
-
         # adding values from input that are not used in the mapper
         for input in set(self._input_names) - set(self._input_names_mapper):
             state_dict[input] = self.state_inputs[input]
 
         # in py3.7 we can skip OrderedDict
         # returning a named tuple?
-        return OrderedDict(sorted(state_dict.items(), key=lambda t: t[0]))
\ No newline at end of file
+        return OrderedDict(sorted(state_dict.items(), key=lambda t: t[0]))
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index c0e253c0e3..ee2fa93c98 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -35,6 +35,10 @@
 
 from .base import EngineBase
 from .nodes import MapNode, Node
+from . import state
+from . import auxiliary as aux
+
+
 
 # Py2 compat: http://python-future.org/compatible_idioms.html#collections-counter-and-ordereddict
 from future import standard_library
@@ -1066,16 +1070,70 @@ class MapState(object):
     pass
 
 class NewNode(EngineBase):
-    def __init__(self, inputs={}, map_on=None, join_by=None,
+    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
                  *args, **kwargs):
-        self._mappers = {}
+        self.name = name
+        # dj: do I need a state_input and state_mapper??
+        # dj: reading the input from files should be added
+        if inputs:
+            # adding name of the node to the input name
+            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                                for (key, val) in self._inputs.items())
+        else:
+            self._inputs = {}
+        if mapper:
+            # adding name of the node to the input name within the mapper
+            mapper = aux.change_mapper(mapper, self.name)
+        self._mapper = mapper
+        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
+        self.state = state.State(state_inputs=self._inputs, mapper=self._mapper, node_name=self.name)
+
+        # adding interface: i'm using Function Interface from aux that has input_map that can change the name of arguments
+        self._interface = interface
+        self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
+                                         for (key, value) in self._interface.input_map.items())
+
         self._joiners = {}
 
-    def map(self, field, values=None):
+
+    @property
+    def mapper(self):
+        return self._mapper
+
+
+    @property
+    def inputs(self):
+        return self._inputs
+
+    #@inputs.setter
+    #def inputs(self, inputs):
+    #    self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+    #    self.state_inputs = self._inputs.copy()
+
+
+    def map(self, mapper, inputs=None):
+        if self._mapper:
+            raise Exception("mapper is already set")
+        else:
+            self._mapper = aux.change_mapper(mapper, self.name)
+
+        if inputs:
+            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                          for (key, val) in inputs.items())
+            self._inputs.update(inputs)
+
+        self.state = state.State(state_inputs=self._inputs, mapper=self._mapper, node_name=self.name)
+
+
+
+    def map_orig(self, field, values=None):
         if isinstance(field, list):
-            for field_
+            #for field_ #dj
             if values is not None:
                 if len(values != len(field)):
+                    pass #dj
         elif isinstance(field, tuple):
             pass
         if values is None:

From c9446851df93305767c705b4228e7872edc27b77 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Sat, 14 Jul 2018 08:38:20 -0400
Subject: [PATCH 04/55] adding simple tests

---
 nipype/pipeline/engine/tests/test_newnode.py | 61 ++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 nipype/pipeline/engine/tests/test_newnode.py

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
new file mode 100644
index 0000000000..98fab21630
--- /dev/null
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -0,0 +1,61 @@
+from .. import NewNode
+from ..auxiliary import Function_Interface
+
+import numpy as np
+import pytest, pdb
+
+def fun_addtwo(a):
+    return a + 2
+
+interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+
+def test_node_1():
+    """Node with only mandatory arguments"""
+    nn = NewNode(name="N_A", interface=interf_addtwo)
+    assert nn.mapper is None
+    assert nn.inputs == {}
+    assert nn.state._mapper is None
+
+
+def test_node_2():
+    """Node with interface and inputs"""
+    nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": 3})
+    assert nn.mapper is None
+    assert nn.inputs == {"N_A-a": 3}
+    assert nn.state._mapper is None
+
+
+def test_node_3():
+    """Node with interface and inputs"""
+    nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": [3, 5]}, mapper="a")
+    assert nn.mapper == "N_A-a"
+    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+    
+    assert nn.state._mapper == "N_A-a"
+    assert nn.state.state_values([0]) == {"N_A-a": 3}
+    assert nn.state.state_values([1]) == {"N_A-a": 5}
+
+
+def test_node_4():
+    """Node with interface and inputs"""
+    nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": [3, 5]})
+    nn.map(mapper="a")
+    assert nn.mapper == "N_A-a"
+    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+
+    assert nn.state._mapper == "N_A-a"
+    assert nn.state.state_values([0]) == {"N_A-a": 3}
+    assert nn.state.state_values([1]) == {"N_A-a": 5}
+
+
+def test_node_5():
+    """Node with interface and inputs"""
+    nn = NewNode(name="N_A", interface=interf_addtwo)
+    nn.map(mapper="a", inputs={"a": [3, 5]})
+    assert nn.mapper == "N_A-a"
+    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+
+    assert nn.state._mapper == "N_A-a"
+    assert nn.state.state_values([0]) == {"N_A-a": 3}
+    assert nn.state.state_values([1]) == {"N_A-a": 5}
+    pdb.set_trace()

From 4da873728545fcca2a5f1d038feb5074dc0c1b1c Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Sat, 14 Jul 2018 23:00:15 -0400
Subject: [PATCH 05/55] adding run_interface method for NewNode (still have to
 add run method)

---
 nipype/pipeline/engine/tests/test_newnode.py | 15 +++-
 nipype/pipeline/engine/workflows.py          | 75 ++++++++++++++++----
 2 files changed, 74 insertions(+), 16 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 98fab21630..4a908fe03c 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -58,4 +58,17 @@ def test_node_5():
     assert nn.state._mapper == "N_A-a"
     assert nn.state.state_values([0]) == {"N_A-a": 3}
     assert nn.state.state_values([1]) == {"N_A-a": 5}
-    pdb.set_trace()
+
+
+def test_node_6():
+    """Node with interface and inputs, running interface"""
+    nn = NewNode(name="N_A", interface=interf_addtwo, base_dir="test6")
+    nn.map(mapper="a", inputs={"a": [3, 5]})
+    assert nn.mapper == "N_A-a"
+    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+
+    assert nn.state._mapper == "N_A-a"
+
+    nn.run_interface_el(0, (0,))
+
+
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index ee2fa93c98..ab68627dcf 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1071,8 +1071,10 @@ class MapState(object):
 
 class NewNode(EngineBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 *args, **kwargs):
-        self.name = name
+                 base_dir=None, *args, **kwargs):
+        super(NewNode, self).__init__(name=name, base_dir=base_dir)
+        # dj: should be changed for wf
+        self.nodedir = self.base_dir
         # dj: do I need a state_input and state_mapper??
         # dj: reading the input from files should be added
         if inputs:
@@ -1094,6 +1096,7 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
         self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
                                          for (key, value) in self._interface.input_map.items())
 
+        self.needed_outputs = []
         self._joiners = {}
 
 
@@ -1128,24 +1131,66 @@ def map(self, mapper, inputs=None):
 
 
 
-    def map_orig(self, field, values=None):
-        if isinstance(field, list):
-            #for field_ #dj
-            if values is not None:
-                if len(values != len(field)):
-                    pass #dj
-        elif isinstance(field, tuple):
-            pass
-        if values is None:
-            values = getattr(self._inputs, field)
-            if values is None:
-                raise MappingError('Cannot map unassigned input field')
-        self._mappers[field] = values
+#    def map_orig(self, field, values=None):
+#        if isinstance(field, list):
+#            for field_
+#            if values is not None:
+#                if len(values != len(field)):
+#        elif isinstance(field, tuple):
+#            pass
+#        if values is None:
+#            values = getattr(self._inputs, field)
+#            if values is None:
+#                raise MappingError('Cannot map unassigned input field')
+#        self._mappers[field] = values
 
+    # TBD
     def join(self, field):
         pass
 
 
+    def run_interface_el(self, i, ind, single_node=False):
+        """ running interface one element generated from node_state."""
+        logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
+        if not single_node: # if we run a single node, we don't have to collect output
+            state_dict, inputs_dict = self._collecting_input_el(ind)
+        logger.debug("Run interface el, name={}, inputs_dict={}, state_dict={}".format(
+                                                            self.name, inputs_dict, state_dict))
+        res = self._interface.run(inputs_dict)
+        #pdb.set_trace()
+        output = self._interface.output
+        logger.debug("Run interface el, output={}".format(output))
+        dir_nm_el = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())])
+        # TODO when join
+        #if self._joinByKey:
+        #    dir_join = "join_" + "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items()) if i not in self._joinByKey])
+        #elif self._join:
+        #    dir_join = "join_"
+        #if self._joinByKey or self._join:
+        #    os.makedirs(os.path.join(self.nodedir, dir_join), exist_ok=True)
+        #    dir_nm_el = os.path.join(dir_join, dir_nm_el)
+        os.makedirs(os.path.join(self.nodedir, dir_nm_el), exist_ok=True)
+        for key_out in list(output.keys()):
+            with open(os.path.join(self.nodedir, dir_nm_el, key_out+".txt"), "w") as fout:
+                fout.write(str(output[key_out]))
+        return res
+
+    # dj: this is not used for a single node
+    def _collecting_input_el(self, ind):
+        state_dict = self.state.state_values(ind)
+        inputs_dict = {k: state_dict[k] for k in self._inputs.keys()}
+        # reading extra inputs that come from previous nodes
+        for (from_node, from_socket, to_socket) in self.needed_outputs:
+            dir_nm_el_from = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())
+                                       if i in list(from_node.state_inputs.keys())])
+            file_from = os.path.join(from_node.nodedir, dir_nm_el_from, from_socket+".txt")
+            with open(file_from) as f:
+                inputs_dict["{}-{}".format(self.name, to_socket)] = eval(f.readline())
+        return state_dict, inputs_dict
+
+
+
+
 class NewWorkflow(NewNode):
     def __init__(self, inputs={}, *args, **kwargs):
         super(NewWorkflow, self).__init__(*args, **kwargs)

From 7ff7722c61d6376293de75843989ae9ae5b2cbe4 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Sun, 15 Jul 2018 00:26:35 -0400
Subject: [PATCH 06/55] copy submitter and workers from nipype2_tmp; adding
 SubmitterNode and SubmitterWorkflow class; simple test for NewNode.run using
 SubmitterNode

---
 nipype/pipeline/engine/submitter.py          | 150 +++++++++++++++++++
 nipype/pipeline/engine/tests/test_newnode.py |   5 +-
 nipype/pipeline/engine/workers.py            |  99 ++++++++++++
 nipype/pipeline/engine/workflows.py          |   9 +-
 4 files changed, 259 insertions(+), 4 deletions(-)
 create mode 100644 nipype/pipeline/engine/submitter.py
 create mode 100644 nipype/pipeline/engine/workers.py

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
new file mode 100644
index 0000000000..3035b79f47
--- /dev/null
+++ b/nipype/pipeline/engine/submitter.py
@@ -0,0 +1,150 @@
+from __future__ import print_function, division, unicode_literals, absolute_import
+from builtins import object
+from collections import defaultdict
+
+from future import standard_library
+standard_library.install_aliases()
+
+import os, pdb, time, glob
+import itertools, collections
+import queue
+
+from .workers import MpWorker, SerialWorker, DaskWorker, ConcurrentFuturesWorker
+
+from ... import config, logging
+logger = logging.getLogger('nipype.workflow')
+
+class Submitter(object):
+    def __init__(self, plugin):
+        self.plugin = plugin
+        self.node_line = []
+        if self.plugin == "mp":
+            self.worker = MpWorker()
+        elif self.plugin == "serial":
+            self.worker = SerialWorker()
+        elif self.plugin == "dask":
+            self.worker = DaskWorker()
+        elif self.plugin == "cf":
+            self.worker = ConcurrentFuturesWorker()
+        else:
+            raise Exception("plugin {} not available".format(self.plugin))
+
+
+    def submit_work(self, node):
+        for (i, ind) in enumerate(itertools.product(*node.state.all_elements)):
+            self._submit_work_el(node, i, ind)
+
+    def _submit_work_el(self, node, i, ind):
+        logger.debug("SUBMIT WORKER, node: {}, ind: {}".format(node, ind))
+        self.worker.run_el(node.run_interface_el, (i, ind))
+
+
+    def close(self):
+        self.worker.close()
+
+
+
+class SubmitterNode(Submitter):
+    def __init__(self, plugin, node):
+        super(SubmitterNode, self).__init__(plugin)
+        self.node = node
+
+    def run_node(self):
+        self.submit_work(self.node)
+
+
+class SubmitterWorkflow(Submitter):
+    def __init__(self, graph, plugin):
+        super(SubmitterWorkflow, self).__init_(plugin)
+        self.graph = graph
+        logger.debug('Initialize Submitter, graph: {}'.format(graph))
+        self._to_finish = list(self.graph)
+
+
+    def run_workflow(self):
+        for (i_n, node) in enumerate(self.graph):
+            # submitting all the nodes who are self sufficient (self.graph is already sorted)
+            if node.sufficient:
+                self.submit_work(node)
+            # if its not, its been added to a line
+            else:
+                break
+
+            # in case there is no element in the graph that goes to the break
+            # i want to be sure that not calculating the last node again in the next for loop
+            if i_n == len(self.graph) - 1:
+                i_n += 1
+
+            # adding task for reducer
+            if node._join_interface:
+                # decided to add it as one task, since I have to wait for everyone before  can start it anyway
+                self.node_line.append((node, "join", None))
+
+
+        # all nodes that are not self sufficient will go to the line
+        # iterating over all elements
+        # (i think ordered list work well here, since it's more efficient to check within a specific order)
+        for nn in self.graph[i_n:]:
+            for (i, ind) in enumerate(itertools.product(*nn.state.all_elements)):
+                self.node_line.append((nn, i, ind))
+            if nn._join_interface:
+                # decided to add it as one task, since I have to wait for everyone before can start it anyway
+                self.node_line.append((nn, "join", None))
+
+
+        # this parts submits nodes that are waiting to be run
+        # it should stop when nothing is waiting
+        while self._nodes_check():
+            logger.debug("Submitter, in while, node_line: {}".format(self.node_line))
+            time.sleep(3)
+
+        # TODO(?): combining two while together
+        # this part simply waiting for all "last nodes" to finish
+        while self._output_check():
+            logger.debug("Submitter, in while, to_finish: {}".format(self._to_finish))
+            time.sleep(3)
+
+
+    # for now without callback, so checking all nodes(with ind) in some order
+    def _nodes_check(self):
+        _to_remove = []
+        for (to_node, i, ind) in self.node_line:
+            if i == "join":
+                if to_node.global_done: #have to check if interface has finished
+                    self.submit_join_work(to_node)
+                    _to_remove.append((to_node, i, ind))
+                else:
+                    pass
+            else:
+                if to_node.checking_input_el(ind):
+                    self._submit_work_el(to_node, i, ind)
+                    _to_remove.append((to_node, i, ind))
+                else:
+                    pass
+        # can't remove during iterating
+        for rn in _to_remove:
+            self.node_line.remove(rn)
+        return self.node_line
+
+
+    # this I believe can be done for entire node
+    def _output_check(self):
+        _to_remove = []
+        for node in self._to_finish:
+            print("_output check node", node,node.global_done, node._join_interface, node._global_done_join )
+            if node.global_done:
+                if node._join_interface:
+                    if node.global_done_join:
+                        _to_remove.append(node)
+                else:
+                    _to_remove.append(node)
+        for rn in _to_remove:
+            self._to_finish.remove(rn)
+        return self._to_finish
+
+
+    def submit_join_work(self, node):
+        logger.debug("SUBMIT JOIN WORKER, node: {}".format(node))
+        for (state_redu, res_redu) in node.result[node._join_interface_input]: # TODO, should be more general than out
+            res_redu_l = [i[1] for i in res_redu]
+            self.worker.run_el(node.run_interface_join_el, (state_redu, res_redu_l))
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 4a908fe03c..a2b59ff45e 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -69,6 +69,5 @@ def test_node_6():
 
     assert nn.state._mapper == "N_A-a"
 
-    nn.run_interface_el(0, (0,))
-
-
+    # testing if the run method works
+    nn.run()
diff --git a/nipype/pipeline/engine/workers.py b/nipype/pipeline/engine/workers.py
new file mode 100644
index 0000000000..2e7b83608e
--- /dev/null
+++ b/nipype/pipeline/engine/workers.py
@@ -0,0 +1,99 @@
+from __future__ import print_function, division, unicode_literals, absolute_import
+from builtins import object
+from collections import defaultdict
+
+from future import standard_library
+standard_library.install_aliases()
+
+from copy import deepcopy
+import re, os, pdb, time
+import multiprocessing as mp
+#import multiprocess as mp
+import itertools
+
+#from pycon_utils import make_cluster
+from dask.distributed import Client
+
+import concurrent.futures as cf
+
+from ... import config, logging
+logger = logging.getLogger('nipype.workflow')
+
+
+class Worker(object):
+    def __init__(self):
+        logger.debug("Initialize Worker")
+        pass
+
+    def run_el(self):
+        raise NotImplementedError
+
+    def close(self):
+        raise NotImplementedError
+
+
+class MpWorker(Worker):
+    def __init__(self, nr_proc=4): #should be none
+        self.nr_proc = nr_proc
+        self.pool = mp.Pool(processes=self.nr_proc)
+        logger.debug('Initialize MpWorker')
+
+    def run_el(self, interface, inp):
+        self.pool.apply_async(interface, (inp[0], inp[1]))
+
+    def close(self):
+        # added this method since I was having somtetimes problem with reading results from (existing) files
+        # i thought that pool.close() should work, but still was getting some errors, so testing terminate
+        self.pool.terminate()
+
+
+class SerialWorker(Worker):
+    def __init__(self):
+        logger.debug("Initialize SerialWorker")
+        pass
+
+    def run_el(self, interface, inp):
+        interface(inp[0], inp[1])
+
+    def close(self):
+        pass
+
+
+class ConcurrentFuturesWorker(Worker):
+    def __init__(self, nr_proc=4):
+        self.nr_proc = nr_proc
+        self.pool = cf.ProcessPoolExecutor(self.nr_proc)
+        logger.debug('Initialize ConcurrentFuture')
+
+    def run_el(self, interface, inp):
+        x = self.pool.submit(interface, inp[0], inp[1])
+        #print("X, DONE", x.done())
+        x.add_done_callback(lambda x: print("DONE ", interface, inp, x.done))
+        #print("DIR", x.result())
+
+    def close(self):
+        self.pool.shutdown()
+
+
+class DaskWorker(Worker):
+    def __init__(self):
+        from distributed.deploy.local import LocalCluster
+        logger.debug("Initialize Dask Worker")
+        #self.cluster = LocalCluster()
+        self.client = Client()#self.cluster)
+        print("BOKEH", self.client.scheduler_info()["address"] + ":" + str(self.client.scheduler_info()["services"]["bokeh"]))
+
+
+    def run_el(self, interface, inp):
+        print("DASK, run_el: ", interface, inp)
+        x = self.client.submit(interface, inp[0], inp[1])
+        print("DASK, status: ", x.status)
+        # this important, otherwise dask will not finish the job
+        x.add_done_callback(lambda x: print("DONE ", interface, inp))
+        #print("res", x.result())
+
+
+    def close(self):
+        #self.cluster.close()
+        self.client.close()
+
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index ab68627dcf..913375108c 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -37,7 +37,7 @@
 from .nodes import MapNode, Node
 from . import state
 from . import auxiliary as aux
-
+from . import submitter as sub
 
 
 # Py2 compat: http://python-future.org/compatible_idioms.html#collections-counter-and-ordereddict
@@ -1149,6 +1149,13 @@ def join(self, field):
         pass
 
 
+    def run(self, plugin="serial"):
+        self.sub = sub.SubmitterNode(plugin, node=self) #dj: ?
+        self.sub.run_node()
+        self.sub.close()
+
+
+
     def run_interface_el(self, i, ind, single_node=False):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))

From 502f6170363a3bb3175b6d64862ae81802ee4f4f Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 16 Jul 2018 09:27:21 -0400
Subject: [PATCH 07/55] fixing run method for node (previously it didn work for
 dask and cf): had to create a new class and have now NewNodeBase and NewNode
 that is a wrapper with the run method

---
 nipype/pipeline/engine/submitter.py          |  4 +
 nipype/pipeline/engine/tests/test_newnode.py | 21 +++--
 nipype/pipeline/engine/workflows.py          | 92 ++++++++++++++++----
 3 files changed, 95 insertions(+), 22 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 3035b79f47..d04c29b550 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -52,6 +52,10 @@ def __init__(self, plugin, node):
     def run_node(self):
         self.submit_work(self.node)
 
+        while not self.node.global_done:
+            logger.debug("Submitter, in while, to_finish: {}".format(self.node))
+            time.sleep(3)
+
 
 class SubmitterWorkflow(Submitter):
     def __init__(self, graph, plugin):
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index a2b59ff45e..04b20d9983 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -7,10 +7,10 @@
 def fun_addtwo(a):
     return a + 2
 
-interf_addtwo = Function_Interface(fun_addtwo, ["out"])
 
 def test_node_1():
     """Node with only mandatory arguments"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="N_A", interface=interf_addtwo)
     assert nn.mapper is None
     assert nn.inputs == {}
@@ -19,6 +19,7 @@ def test_node_1():
 
 def test_node_2():
     """Node with interface and inputs"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": 3})
     assert nn.mapper is None
     assert nn.inputs == {"N_A-a": 3}
@@ -27,6 +28,7 @@ def test_node_2():
 
 def test_node_3():
     """Node with interface and inputs"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": [3, 5]}, mapper="a")
     assert nn.mapper == "N_A-a"
     assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
@@ -38,6 +40,7 @@ def test_node_3():
 
 def test_node_4():
     """Node with interface and inputs"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": [3, 5]})
     nn.map(mapper="a")
     assert nn.mapper == "N_A-a"
@@ -50,6 +53,7 @@ def test_node_4():
 
 def test_node_5():
     """Node with interface and inputs"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="N_A", interface=interf_addtwo)
     nn.map(mapper="a", inputs={"a": [3, 5]})
     assert nn.mapper == "N_A-a"
@@ -60,14 +64,17 @@ def test_node_5():
     assert nn.state.state_values([1]) == {"N_A-a": 5}
 
 
-def test_node_6():
+Plugins = ["mp", "serial", "cf", "dask"] 
+
+@pytest.mark.parametrize("plugin", Plugins)
+def test_node_6(plugin):
     """Node with interface and inputs, running interface"""
-    nn = NewNode(name="N_A", interface=interf_addtwo, base_dir="test6")
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    nn = NewNode(name="N_A", interface=interf_addtwo, base_dir="test6_{}".format(plugin))
     nn.map(mapper="a", inputs={"a": [3, 5]})
+
     assert nn.mapper == "N_A-a"
     assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
 
-    assert nn.state._mapper == "N_A-a"
-
-    # testing if the run method works
-    nn.run()
+    # testing if the node runs properly
+    nn.run(plugin=plugin)
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 913375108c..a93ceca885 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -20,6 +20,7 @@
 
 import numpy as np
 import networkx as nx
+import itertools
 
 from ... import config, logging
 from ...exceptions import NodeError, WorkflowError, MappingError, JoinError
@@ -39,6 +40,7 @@
 from . import auxiliary as aux
 from . import submitter as sub
 
+import pdb
 
 # Py2 compat: http://python-future.org/compatible_idioms.html#collections-counter-and-ordereddict
 from future import standard_library
@@ -1069,10 +1071,10 @@ class Join(Node):
 class MapState(object):
     pass
 
-class NewNode(EngineBase):
+class NewNodeBase(EngineBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
                  base_dir=None, *args, **kwargs):
-        super(NewNode, self).__init__(name=name, base_dir=base_dir)
+        super(NewNodeBase, self).__init__(name=name, base_dir=base_dir)
         # dj: should be changed for wf
         self.nodedir = self.base_dir
         # dj: do I need a state_input and state_mapper??
@@ -1097,6 +1099,9 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
                                          for (key, value) in self._interface.input_map.items())
 
         self.needed_outputs = []
+        self._out_nm = self._interface._output_nm
+        self._global_done = False
+
         self._joiners = {}
 
 
@@ -1109,11 +1114,6 @@ def mapper(self):
     def inputs(self):
         return self._inputs
 
-    #@inputs.setter
-    #def inputs(self, inputs):
-    #    self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-    #    self.state_inputs = self._inputs.copy()
-
 
     def map(self, mapper, inputs=None):
         if self._mapper:
@@ -1126,7 +1126,6 @@ def map(self, mapper, inputs=None):
             inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
                           for (key, val) in inputs.items())
             self._inputs.update(inputs)
-
         self.state = state.State(state_inputs=self._inputs, mapper=self._mapper, node_name=self.name)
 
 
@@ -1149,13 +1148,6 @@ def join(self, field):
         pass
 
 
-    def run(self, plugin="serial"):
-        self.sub = sub.SubmitterNode(plugin, node=self) #dj: ?
-        self.sub.run_node()
-        self.sub.close()
-
-
-
     def run_interface_el(self, i, ind, single_node=False):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
@@ -1197,6 +1189,76 @@ def _collecting_input_el(self, ind):
 
 
 
+    # checking if all outputs are saved
+    @property
+    def global_done(self):
+        # once _global_done os True, this should not change
+        logger.debug('global_done {}'.format(self._global_done))
+        if self._global_done:
+            return self._global_done
+        else:
+            return self._check_all_results()
+
+    # dj: version without join
+    def _check_all_results(self):
+        # checking if all files that should be created are present
+        for ind in itertools.product(*self.state._all_elements):
+            state_dict = self.state.state_values(ind)
+            dir_nm_el = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())])
+            for key_out in self._out_nm:
+                if not os.path.isfile(os.path.join(self.nodedir, dir_nm_el, key_out+".txt")):
+                    return False
+        self._global_done = True
+        return True
+
+
+
+
+class NewNode(object):
+    """wrapper around NewNodeBase, mostly have run method """
+    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
+                 base_dir=None, *args, **kwargs):
+        self.node = NewNodeBase(name, interface, inputs, mapper, join_by,
+                                base_dir, *args, **kwargs)
+        # dj: might want to use a one element graph
+        #self.graph = nx.DiGraph()
+        #self.graph.add_nodes_from([self.node])
+
+
+    def map(self, mapper, inputs=None):
+         self.node.map(mapper, inputs)
+
+    @property
+    def state(self):
+        return self.node.state
+
+
+    @property
+    def mapper(self):
+        return self.node.mapper
+
+
+    @property
+    def inputs(self):
+        return self.node._inputs
+
+    @property
+    def global_done(self):
+        return self.node._global_done
+
+
+    @property
+    def outputs(self):
+        return self.node.outputs
+
+
+    def run(self, plugin="serial"):
+        self.sub = sub.SubmitterNode(plugin, node=self.node)
+        self.sub.run_node()
+        self.sub.close()
+
+
+
 
 class NewWorkflow(NewNode):
     def __init__(self, inputs={}, *args, **kwargs):

From 784ec2269881914e32ec9cbfeedc502f3cc7a4f7 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 16 Jul 2018 10:06:52 -0400
Subject: [PATCH 08/55] adding reading results; changing node names (node name
 shouldn have _)

---
 nipype/pipeline/engine/tests/test_newnode.py | 54 +++++++++++---------
 nipype/pipeline/engine/workflows.py          | 40 ++++++++++++++-
 2 files changed, 68 insertions(+), 26 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 04b20d9983..935a1c414d 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -11,7 +11,7 @@ def fun_addtwo(a):
 def test_node_1():
     """Node with only mandatory arguments"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="N_A", interface=interf_addtwo)
+    nn = NewNode(name="NA", interface=interf_addtwo)
     assert nn.mapper is None
     assert nn.inputs == {}
     assert nn.state._mapper is None
@@ -20,48 +20,48 @@ def test_node_1():
 def test_node_2():
     """Node with interface and inputs"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": 3})
+    nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": 3})
     assert nn.mapper is None
-    assert nn.inputs == {"N_A-a": 3}
+    assert nn.inputs == {"NA-a": 3}
     assert nn.state._mapper is None
 
 
 def test_node_3():
     """Node with interface and inputs"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": [3, 5]}, mapper="a")
-    assert nn.mapper == "N_A-a"
-    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+    nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]}, mapper="a")
+    assert nn.mapper == "NA-a"
+    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
     
-    assert nn.state._mapper == "N_A-a"
-    assert nn.state.state_values([0]) == {"N_A-a": 3}
-    assert nn.state.state_values([1]) == {"N_A-a": 5}
+    assert nn.state._mapper == "NA-a"
+    assert nn.state.state_values([0]) == {"NA-a": 3}
+    assert nn.state.state_values([1]) == {"NA-a": 5}
 
 
 def test_node_4():
     """Node with interface and inputs"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="N_A", interface=interf_addtwo, inputs={"a": [3, 5]})
+    nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]})
     nn.map(mapper="a")
-    assert nn.mapper == "N_A-a"
-    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+    assert nn.mapper == "NA-a"
+    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
 
-    assert nn.state._mapper == "N_A-a"
-    assert nn.state.state_values([0]) == {"N_A-a": 3}
-    assert nn.state.state_values([1]) == {"N_A-a": 5}
+    assert nn.state._mapper == "NA-a"
+    assert nn.state.state_values([0]) == {"NA-a": 3}
+    assert nn.state.state_values([1]) == {"NA-a": 5}
 
 
 def test_node_5():
     """Node with interface and inputs"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="N_A", interface=interf_addtwo)
+    nn = NewNode(name="NA", interface=interf_addtwo)
     nn.map(mapper="a", inputs={"a": [3, 5]})
-    assert nn.mapper == "N_A-a"
-    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+    assert nn.mapper == "NA-a"
+    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
 
-    assert nn.state._mapper == "N_A-a"
-    assert nn.state.state_values([0]) == {"N_A-a": 3}
-    assert nn.state.state_values([1]) == {"N_A-a": 5}
+    assert nn.state._mapper == "NA-a"
+    assert nn.state.state_values([0]) == {"NA-a": 3}
+    assert nn.state.state_values([1]) == {"NA-a": 5}
 
 
 Plugins = ["mp", "serial", "cf", "dask"] 
@@ -70,11 +70,17 @@ def test_node_5():
 def test_node_6(plugin):
     """Node with interface and inputs, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="N_A", interface=interf_addtwo, base_dir="test6_{}".format(plugin))
+    nn = NewNode(name="NA", interface=interf_addtwo, base_dir="test6_{}".format(plugin))
     nn.map(mapper="a", inputs={"a": [3, 5]})
 
-    assert nn.mapper == "N_A-a"
-    assert (nn.inputs["N_A-a"] == np.array([3, 5])).all()
+    assert nn.mapper == "NA-a"
+    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
 
     # testing if the node runs properly
     nn.run(plugin=plugin)
+
+    # checking teh results
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    for i, res in enumerate(expected):
+        assert nn.result["out"][i][0] == res[0]
+        assert nn.result["out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index a93ceca885..fb85343d72 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -10,7 +10,7 @@
                         absolute_import)
 from builtins import str, bytes, open
 
-import os
+import os, glob
 import os.path as op
 import sys
 from datetime import datetime
@@ -20,7 +20,7 @@
 
 import numpy as np
 import networkx as nx
-import itertools
+import itertools, collections
 
 from ... import config, logging
 from ...exceptions import NodeError, WorkflowError, MappingError, JoinError
@@ -1101,6 +1101,7 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
         self.needed_outputs = []
         self._out_nm = self._interface._output_nm
         self._global_done = False
+        self._result = {}
 
         self._joiners = {}
 
@@ -1212,6 +1213,36 @@ def _check_all_results(self):
         return True
 
 
+    # reading results (without join for now)
+    @property
+    def result(self):
+        if not self._result:
+            self._reading_results()
+        return self._result
+
+
+    def _reading_results(self):
+        """
+        reading results from file,
+        doesn't check if everything is ready, i.e. if self.global_done"""
+        for key_out in self._out_nm:
+            self._result[key_out] = []
+            if self.inputs: #self.state_inputs:
+                files = [name for name in glob.glob("{}/*/{}.txt".format(self.nodedir, key_out))]
+                for file in files:
+                    st_el = file.split(os.sep)[-2].split("_")
+                    st_dict = collections.OrderedDict([(el.split(".")[0], eval(el.split(".")[1]))
+                                                            for el in st_el])
+                    with open(file) as fout:
+                        logger.debug('Reading Results: file={}, st_dict={}'.format(file, st_dict))
+                        self._result[key_out].append((st_dict, eval(fout.readline())))
+            # for nodes without input
+            else:
+                files = [name for name in glob.glob("{}/{}.txt".format(self.nodedir, key_out))]
+                with open(files[0]) as fout:
+                    self._result[key_out].append(({}, eval(fout.readline())))
+
+
 
 
 class NewNode(object):
@@ -1252,6 +1283,11 @@ def outputs(self):
         return self.node.outputs
 
 
+    @property
+    def result(self):
+        return self.node.result
+
+
     def run(self, plugin="serial"):
         self.sub = sub.SubmitterNode(plugin, node=self.node)
         self.sub.run_node()

From 18a617612467a7680aad56b78ead962611ad374c Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 16 Jul 2018 13:04:30 -0400
Subject: [PATCH 09/55] adding dask to requirements

---
 nipype/info.py       | 1 +
 requirements.txt     | 1 +
 rtd_requirements.txt | 1 +
 3 files changed, 3 insertions(+)

diff --git a/nipype/info.py b/nipype/info.py
index a371b70e2e..c8e384f7b0 100644
--- a/nipype/info.py
+++ b/nipype/info.py
@@ -148,6 +148,7 @@ def get_nipype_gitversion():
     'pydot>=%s' % PYDOT_MIN_VERSION,
     'packaging',
     'futures; python_version == "2.7"',
+    'dask',
 ]
 
 if sys.version_info <= (3, 4):
diff --git a/requirements.txt b/requirements.txt
index 5ef00ec98b..602be4c50a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,3 +17,4 @@ mock
 pydotplus
 pydot>=1.2.3
 packaging
+dask
diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index 68a366bbdf..6a84e7c2aa 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -17,3 +17,4 @@ psutil
 matplotlib
 packaging
 numpydoc
+dask

From 601c15bcd4f27b236f34259a4c40cfd51a1c67c1 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 16 Jul 2018 17:50:10 -0400
Subject: [PATCH 10/55] adding distributed package

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 602be4c50a..6daeee21ab 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,3 +18,4 @@ pydotplus
 pydot>=1.2.3
 packaging
 dask
+distributed

From d5ceb2481db73670e951cf5ccb69378990691ee4 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 16 Jul 2018 18:37:41 -0400
Subject: [PATCH 11/55] skipping tests for py2; sorting results (not sure if
 the node should be keep the order)

---
 nipype/pipeline/engine/tests/test_newnode.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 935a1c414d..501e2fd871 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1,9 +1,14 @@
 from .. import NewNode
 from ..auxiliary import Function_Interface
 
+import sys
 import numpy as np
 import pytest, pdb
 
+python3_only = pytest.mark.skipif(sys.version_info < (3, 0),
+                                  reason="requires Python3")
+
+
 def fun_addtwo(a):
     return a + 2
 
@@ -67,6 +72,7 @@ def test_node_5():
 Plugins = ["mp", "serial", "cf", "dask"] 
 
 @pytest.mark.parametrize("plugin", Plugins)
+@python3_only
 def test_node_6(plugin):
     """Node with interface and inputs, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -81,6 +87,10 @@ def test_node_6(plugin):
 
     # checking teh results
     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    # to be sure that there is the same order (not sure if node itself should keep the order)
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    nn.result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
     for i, res in enumerate(expected):
         assert nn.result["out"][i][0] == res[0]
         assert nn.result["out"][i][1] == res[1]

From 692bc1b305f3d510f07bb9ba376bed006c4ce53d Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 16 Jul 2018 21:52:56 -0400
Subject: [PATCH 12/55] removing printing the bokeh address (doesnt work with
 travis)

---
 nipype/pipeline/engine/workers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nipype/pipeline/engine/workers.py b/nipype/pipeline/engine/workers.py
index 2e7b83608e..d20a12725c 100644
--- a/nipype/pipeline/engine/workers.py
+++ b/nipype/pipeline/engine/workers.py
@@ -81,7 +81,7 @@ def __init__(self):
         logger.debug("Initialize Dask Worker")
         #self.cluster = LocalCluster()
         self.client = Client()#self.cluster)
-        print("BOKEH", self.client.scheduler_info()["address"] + ":" + str(self.client.scheduler_info()["services"]["bokeh"]))
+        #print("BOKEH", self.client.scheduler_info()["address"] + ":" + str(self.client.scheduler_info()["services"]["bokeh"]))
 
 
     def run_el(self, interface, inp):

From ccf89264960a4b8a78b21140ef8d7a10ace3aebc Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 17 Jul 2018 16:32:46 -0400
Subject: [PATCH 13/55] adding tests for node mappers

---
 nipype/pipeline/engine/tests/test_newnode.py | 58 ++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 501e2fd871..d6c79f4b30 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -13,6 +13,11 @@ def fun_addtwo(a):
     return a + 2
 
 
+def fun_addvar(a, b):
+    return a + b
+
+
+
 def test_node_1():
     """Node with only mandatory arguments"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -94,3 +99,56 @@ def test_node_6(plugin):
     for i, res in enumerate(expected):
         assert nn.result["out"][i][0] == res[0]
         assert nn.result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python3_only
+def test_node_7(plugin):
+    """Node with interface and inputs, running interface"""
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test7_{}".format(plugin))
+    nn.map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [2, 1]})
+
+    assert nn.mapper == ("NA-a", "NA-b")
+    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
+    assert (nn.inputs["NA-b"] == np.array([2, 1])).all()
+
+    # testing if the node runs properly
+    nn.run(plugin=plugin)
+
+    # checking teh results
+    expected = [({"NA-a": 3, "NA-b": 2}, 5), ({"NA-a": 5, "NA-b": 1}, 6)]
+    # to be sure that there is the same order (not sure if node itself should keep the order)
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    nn.result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert nn.result["out"][i][0] == res[0]
+        assert nn.result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python3_only
+def test_node_8(plugin):
+    """Node with interface and inputs, running interface"""
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test8_{}".format(plugin))
+    nn.map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [2, 1]})
+
+    assert nn.mapper == ["NA-a", "NA-b"]
+    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
+    assert (nn.inputs["NA-b"] == np.array([2, 1])).all()
+
+    # testing if the node runs properly
+    nn.run(plugin=plugin)
+
+    # checking teh results
+    expected = [({"NA-a": 3, "NA-b": 1}, 4), ({"NA-a": 3, "NA-b": 2}, 5),
+                ({"NA-a": 5, "NA-b": 1}, 6), ({"NA-a": 5, "NA-b": 2}, 7)]
+    # to be sure that there is the same order (not sure if node itself should keep the order)
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    nn.result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert nn.result["out"][i][0] == res[0]
+        assert nn.result["out"][i][1] == res[1]

From 8300a98d3cd286534f3008ff7d066082f64d3848 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 17 Jul 2018 16:34:41 -0400
Subject: [PATCH 14/55] changing requirments to py>3.4 (fails with py3.4, not
 sure if we want to support it anyway)

---
 nipype/pipeline/engine/tests/test_newnode.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index d6c79f4b30..7a0c83be33 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -5,8 +5,8 @@
 import numpy as np
 import pytest, pdb
 
-python3_only = pytest.mark.skipif(sys.version_info < (3, 0),
-                                  reason="requires Python3")
+python35_only = pytest.mark.skipif(sys.version_info < (3, 5),
+                                   reason="requires Python>3.4")
 
 
 def fun_addtwo(a):
@@ -77,7 +77,7 @@ def test_node_5():
 Plugins = ["mp", "serial", "cf", "dask"] 
 
 @pytest.mark.parametrize("plugin", Plugins)
-@python3_only
+@python35_only
 def test_node_6(plugin):
     """Node with interface and inputs, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -102,7 +102,7 @@ def test_node_6(plugin):
 
 
 @pytest.mark.parametrize("plugin", Plugins)
-@python3_only
+@python35_only
 def test_node_7(plugin):
     """Node with interface and inputs, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
@@ -128,7 +128,7 @@ def test_node_7(plugin):
 
 
 @pytest.mark.parametrize("plugin", Plugins)
-@python3_only
+@python35_only
 def test_node_8(plugin):
     """Node with interface and inputs, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])

From 46684a4dc0f8ed9939c646d83c46d76deab6d10e Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Sun, 22 Jul 2018 15:09:31 -0400
Subject: [PATCH 15/55] moving many attributes from NewNodeBase/NewNodeCore to
 NewNode, so I can use it for NewWorkflow; creating prepare_state_input in
 State from the __init__, so I can initialize State before having input

---
 nipype/pipeline/engine/state.py              |  13 +-
 nipype/pipeline/engine/submitter.py          |   1 -
 nipype/pipeline/engine/tests/test_newnode.py |   4 +
 nipype/pipeline/engine/workflows.py          | 149 +++++++++----------
 4 files changed, 83 insertions(+), 84 deletions(-)

diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
index ec96dab04c..7d6fc688dd 100644
--- a/nipype/pipeline/engine/state.py
+++ b/nipype/pipeline/engine/state.py
@@ -4,9 +4,7 @@
 from . import auxiliary as aux
 
 class State(object):
-    def __init__(self, state_inputs, node_name, mapper=None):
-        self.state_inputs = state_inputs
-
+    def __init__(self, node_name, mapper=None):
         self._mapper = mapper
         self.node_name = node_name
         if self._mapper:
@@ -17,6 +15,15 @@ def __init__(self, state_inputs, node_name, mapper=None):
         else:
             self._mapper_rpn = []
             self._input_names_mapper = []
+
+
+    def prepare_state_input(self, state_inputs):
+        """prepare all inputs, should be called once all input is available"""
+
+        # dj TOTHINK: I actually stopped using state_inputs for now, since people wanted to have mapper not only
+        # for state inputs. Might have to come back....
+        self.state_inputs = state_inputs
+
         # not all input field have to be use in the mapper, can be an extra scalar
         self._input_names = list(self.state_inputs.keys())
 
diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index d04c29b550..78aeefb83d 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -51,7 +51,6 @@ def __init__(self, plugin, node):
 
     def run_node(self):
         self.submit_work(self.node)
-
         while not self.node.global_done:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 7a0c83be33..f9b7338222 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -44,6 +44,8 @@ def test_node_3():
     assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
     
     assert nn.state._mapper == "NA-a"
+
+    nn.prepare_state_input()
     assert nn.state.state_values([0]) == {"NA-a": 3}
     assert nn.state.state_values([1]) == {"NA-a": 5}
 
@@ -56,6 +58,7 @@ def test_node_4():
     assert nn.mapper == "NA-a"
     assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
 
+    nn.prepare_state_input()
     assert nn.state._mapper == "NA-a"
     assert nn.state.state_values([0]) == {"NA-a": 3}
     assert nn.state.state_values([1]) == {"NA-a": 5}
@@ -70,6 +73,7 @@ def test_node_5():
     assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
 
     assert nn.state._mapper == "NA-a"
+    nn.prepare_state_input()
     assert nn.state.state_values([0]) == {"NA-a": 3}
     assert nn.state.state_values([1]) == {"NA-a": 5}
 
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index fb85343d72..8e54fb250d 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1071,29 +1071,14 @@ class Join(Node):
 class MapState(object):
     pass
 
-class NewNodeBase(EngineBase):
-    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, *args, **kwargs):
-        super(NewNodeBase, self).__init__(name=name, base_dir=base_dir)
-        # dj: should be changed for wf
-        self.nodedir = self.base_dir
-        # dj: do I need a state_input and state_mapper??
-        # dj: reading the input from files should be added
-        if inputs:
-            # adding name of the node to the input name
-            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                                for (key, val) in self._inputs.items())
-        else:
-            self._inputs = {}
-        if mapper:
-            # adding name of the node to the input name within the mapper
-            mapper = aux.change_mapper(mapper, self.name)
-        self._mapper = mapper
-        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
-        self.state = state.State(state_inputs=self._inputs, mapper=self._mapper, node_name=self.name)
-
+# dj ??: should I use EngineBase?
+class NewNodeCore(object):
+    def __init__(self, name, interface, state, inputs=None, base_dir=None, *args, **kwargs):
         # adding interface: i'm using Function Interface from aux that has input_map that can change the name of arguments
+        self.nodedir = base_dir
+        self.name = name
+        self.state = state
+        self._inputs = inputs
         self._interface = interface
         self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
                                          for (key, value) in self._interface.input_map.items())
@@ -1103,12 +1088,10 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
         self._global_done = False
         self._result = {}
 
-        self._joiners = {}
-
 
     @property
-    def mapper(self):
-        return self._mapper
+    def interface(self):
+        return self._interface
 
 
     @property
@@ -1116,39 +1099,6 @@ def inputs(self):
         return self._inputs
 
 
-    def map(self, mapper, inputs=None):
-        if self._mapper:
-            raise Exception("mapper is already set")
-        else:
-            self._mapper = aux.change_mapper(mapper, self.name)
-
-        if inputs:
-            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                          for (key, val) in inputs.items())
-            self._inputs.update(inputs)
-        self.state = state.State(state_inputs=self._inputs, mapper=self._mapper, node_name=self.name)
-
-
-
-#    def map_orig(self, field, values=None):
-#        if isinstance(field, list):
-#            for field_
-#            if values is not None:
-#                if len(values != len(field)):
-#        elif isinstance(field, tuple):
-#            pass
-#        if values is None:
-#            values = getattr(self._inputs, field)
-#            if values is None:
-#                raise MappingError('Cannot map unassigned input field')
-#        self._mappers[field] = values
-
-    # TBD
-    def join(self, field):
-        pass
-
-
     def run_interface_el(self, i, ind, single_node=False):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
@@ -1246,50 +1196,89 @@ def _reading_results(self):
 
 
 class NewNode(object):
-    """wrapper around NewNodeBase, mostly have run method """
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
                  base_dir=None, *args, **kwargs):
-        self.node = NewNodeBase(name, interface, inputs, mapper, join_by,
-                                base_dir, *args, **kwargs)
-        # dj: might want to use a one element graph
-        #self.graph = nx.DiGraph()
-        #self.graph.add_nodes_from([self.node])
+        # dj: should be changed for wf
+        self.name = name
+        self.nodedir = base_dir
+        self._interface = interface
+        # dj: do I need a state_input and state_mapper??
+        # dj: reading the input from files should be added
+        if inputs:
+            # adding name of the node to the input name
+            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                                for (key, val) in self._inputs.items())
+        else:
+            self._inputs = {}
+        if mapper:
+            # adding name of the node to the input name within the mapper
+            mapper = aux.change_mapper(mapper, self.name)
+        self._mapper = mapper
+        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
+        self._state = state.State(mapper=self._mapper, node_name=self.name)
 
 
     def map(self, mapper, inputs=None):
-         self.node.map(mapper, inputs)
+        if self._mapper:
+            raise Exception("mapper is already set")
+        else:
+            self._mapper = aux.change_mapper(mapper, self.name)
+
+        if inputs:
+            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                          for (key, val) in inputs.items())
+            self._inputs.update(inputs)
+        if mapper:
+            # updating state if we have a new mapper
+            self._state = state.State(mapper=self._mapper, node_name=self.name)
+
+#    def map_orig(self, field, values=None):
+#        if isinstance(field, list):
+#            for field_
+#            if values is not None:
+#                if len(values != len(field)):
+#        elif isinstance(field, tuple):
+#            pass
+#        if values is None:
+#            values = getattr(self._inputs, field)
+#            if values is None:
+#                raise MappingError('Cannot map unassigned input field')
+#        self._mappers[field] = values
+
+    # TBD
+    def join(self, field):
+        pass
 
     @property
     def state(self):
-        return self.node.state
+        return self._state
 
 
     @property
     def mapper(self):
-        return self.node.mapper
+        return self._mapper
 
 
     @property
     def inputs(self):
-        return self.node._inputs
-
-    @property
-    def global_done(self):
-        return self.node._global_done
-
-
-    @property
-    def outputs(self):
-        return self.node.outputs
+        return self._inputs
 
 
     @property
     def result(self):
-        return self.node.result
+        return self.nodecore.result
+
 
+    def prepare_state_input(self):
+        self._state.prepare_state_input(state_inputs=self._inputs)
 
     def run(self, plugin="serial"):
-        self.sub = sub.SubmitterNode(plugin, node=self.node)
+        self.prepare_state_input()
+        self.nodecore = NewNodeCore(name=self.name, base_dir=self.nodedir, interface=self._interface,
+                                    inputs=self._inputs, state=self._state)
+        self.sub = sub.SubmitterNode(plugin, node=self.nodecore)
         self.sub.run_node()
         self.sub.close()
 

From 26496ae2fda6f0106ba50db538397af30dca0ed8 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 24 Jul 2018 10:46:40 -0400
Subject: [PATCH 16/55] starting NewWorkflow class that inherits from NewNode
 (for now only add_nodes, that takes nodes with mapper, tested; didnt work on
 map method; resume using state_inputs (since this is the inputs that I have
 before a workflow run and I can specify state_values based on it, etc.)

---
 nipype/pipeline/engine/submitter.py          |  46 +----
 nipype/pipeline/engine/tests/test_newnode.py | 145 ++++++++++++++-
 nipype/pipeline/engine/workflows.py          | 186 ++++++++++++++++---
 3 files changed, 307 insertions(+), 70 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 78aeefb83d..0f5e4b3b14 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -58,7 +58,7 @@ def run_node(self):
 
 class SubmitterWorkflow(Submitter):
     def __init__(self, graph, plugin):
-        super(SubmitterWorkflow, self).__init_(plugin)
+        super(SubmitterWorkflow, self).__init__(plugin)
         self.graph = graph
         logger.debug('Initialize Submitter, graph: {}'.format(graph))
         self._to_finish = list(self.graph)
@@ -68,7 +68,7 @@ def run_workflow(self):
         for (i_n, node) in enumerate(self.graph):
             # submitting all the nodes who are self sufficient (self.graph is already sorted)
             if node.sufficient:
-                self.submit_work(node)
+                self.submit_work(node.nodecore)
             # if its not, its been added to a line
             else:
                 break
@@ -78,22 +78,12 @@ def run_workflow(self):
             if i_n == len(self.graph) - 1:
                 i_n += 1
 
-            # adding task for reducer
-            if node._join_interface:
-                # decided to add it as one task, since I have to wait for everyone before  can start it anyway
-                self.node_line.append((node, "join", None))
-
-
         # all nodes that are not self sufficient will go to the line
         # iterating over all elements
         # (i think ordered list work well here, since it's more efficient to check within a specific order)
-        for nn in self.graph[i_n:]:
+        for nn in list(self.graph)[i_n:]:
             for (i, ind) in enumerate(itertools.product(*nn.state.all_elements)):
                 self.node_line.append((nn, i, ind))
-            if nn._join_interface:
-                # decided to add it as one task, since I have to wait for everyone before can start it anyway
-                self.node_line.append((nn, "join", None))
-
 
         # this parts submits nodes that are waiting to be run
         # it should stop when nothing is waiting
@@ -112,18 +102,12 @@ def run_workflow(self):
     def _nodes_check(self):
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
-            if i == "join":
-                if to_node.global_done: #have to check if interface has finished
-                    self.submit_join_work(to_node)
-                    _to_remove.append((to_node, i, ind))
-                else:
-                    pass
+            print("NODE LINE", self.node_line)
+            if to_node.nodecore.checking_input_el(ind):
+                self._submit_work_el(to_node.nodecore, i, ind)
+                _to_remove.append((to_node, i, ind))
             else:
-                if to_node.checking_input_el(ind):
-                    self._submit_work_el(to_node, i, ind)
-                    _to_remove.append((to_node, i, ind))
-                else:
-                    pass
+                pass
         # can't remove during iterating
         for rn in _to_remove:
             self.node_line.remove(rn)
@@ -134,20 +118,10 @@ def _nodes_check(self):
     def _output_check(self):
         _to_remove = []
         for node in self._to_finish:
-            print("_output check node", node,node.global_done, node._join_interface, node._global_done_join )
+            print("_output check node", node, node.global_done)
             if node.global_done:
-                if node._join_interface:
-                    if node.global_done_join:
-                        _to_remove.append(node)
-                else:
-                    _to_remove.append(node)
+                _to_remove.append(node)
         for rn in _to_remove:
             self._to_finish.remove(rn)
         return self._to_finish
 
-
-    def submit_join_work(self, node):
-        logger.debug("SUBMIT JOIN WORKER, node: {}".format(node))
-        for (state_redu, res_redu) in node.result[node._join_interface_input]: # TODO, should be more general than out
-            res_redu_l = [i[1] for i in res_redu]
-            self.worker.run_el(node.run_interface_join_el, (state_redu, res_redu_l))
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index f9b7338222..8a75464042 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1,7 +1,7 @@
-from .. import NewNode
+from .. import NewNode, NewWorkflow
 from ..auxiliary import Function_Interface
 
-import sys
+import sys, time
 import numpy as np
 import pytest, pdb
 
@@ -10,6 +10,7 @@
 
 
 def fun_addtwo(a):
+    time.sleep(3)
     return a + 2
 
 
@@ -80,9 +81,9 @@ def test_node_5():
 
 Plugins = ["mp", "serial", "cf", "dask"] 
 
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_node_6(plugin):
+#@pytest.mark.parametrize("plugin", Plugins)
+#@python35_only
+def test_node_6(plugin="serial"):
     """Node with interface and inputs, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, base_dir="test6_{}".format(plugin))
@@ -156,3 +157,137 @@ def test_node_8(plugin):
     for i, res in enumerate(expected):
         assert nn.result["out"][i][0] == res[0]
         assert nn.result["out"][i][1] == res[1]
+
+
+# tests for workflows that set mapper to node that are later added to a workflow
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_1(plugin):
+    """one node with a mapper"""
+    wf = NewWorkflow(name="wf1", workingdir="test_wf1_".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add_nodes([na])
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_2(plugin):
+    """workflow with 2 nodes, second node without mapper"""
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2_".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, base_dir="nb")
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_2a(plugin):
+    """workflow with 2 nodes, second node with a scalar mapper"""
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2a_".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_2b(plugin):
+    """workflow with 2 nodes, second node with a vector mapper"""
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2b_".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb.map(mapper=["NA-a", "b"], inputs={"b": [2, 1]})
+
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+    expected_B = [({"NA-a": 3, "NB-b": 1}, 6), ({"NA-a": 3, "NB-b": 2}, 7),
+                  ({"NA-a": 5, "NB-b": 1}, 8), ({"NA-a": 5, "NB-b": 2}, 9)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 8e54fb250d..1245475465 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1073,12 +1073,14 @@ class MapState(object):
 
 # dj ??: should I use EngineBase?
 class NewNodeCore(object):
-    def __init__(self, name, interface, state, inputs=None, base_dir=None, *args, **kwargs):
+    def __init__(self, name, interface, state, inputs=None, state_inputs=None, base_dir=None, *args, **kwargs):
         # adding interface: i'm using Function Interface from aux that has input_map that can change the name of arguments
         self.nodedir = base_dir
         self.name = name
+        #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
         self.state = state
         self._inputs = inputs
+        self._state_inputs = state_inputs
         self._interface = interface
         self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
                                          for (key, value) in self._interface.input_map.items())
@@ -1099,11 +1101,10 @@ def inputs(self):
         return self._inputs
 
 
-    def run_interface_el(self, i, ind, single_node=False):
+    def run_interface_el(self, i, ind):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
-        if not single_node: # if we run a single node, we don't have to collect output
-            state_dict, inputs_dict = self._collecting_input_el(ind)
+        state_dict, inputs_dict = self._collecting_input_el(ind)
         logger.debug("Run interface el, name={}, inputs_dict={}, state_dict={}".format(
                                                             self.name, inputs_dict, state_dict))
         res = self._interface.run(inputs_dict)
@@ -1132,13 +1133,20 @@ def _collecting_input_el(self, ind):
         # reading extra inputs that come from previous nodes
         for (from_node, from_socket, to_socket) in self.needed_outputs:
             dir_nm_el_from = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())
-                                       if i in list(from_node.state_inputs.keys())])
+                                       if i in list(from_node._state_inputs.keys())])
             file_from = os.path.join(from_node.nodedir, dir_nm_el_from, from_socket+".txt")
             with open(file_from) as f:
                 inputs_dict["{}-{}".format(self.name, to_socket)] = eval(f.readline())
         return state_dict, inputs_dict
 
 
+    def checking_input_el(self, ind):
+        try:
+            self._collecting_input_el(ind)
+            return True
+        except: #TODO specify
+            return False
+
 
     # checking if all outputs are saved
     @property
@@ -1177,7 +1185,7 @@ def _reading_results(self):
         doesn't check if everything is ready, i.e. if self.global_done"""
         for key_out in self._out_nm:
             self._result[key_out] = []
-            if self.inputs: #self.state_inputs:
+            if self._state_inputs:
                 files = [name for name in glob.glob("{}/*/{}.txt".format(self.nodedir, key_out))]
                 for file in files:
                     st_el = file.split(os.sep)[-2].split("_")
@@ -1197,10 +1205,10 @@ def _reading_results(self):
 
 class NewNode(object):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, *args, **kwargs):
+                 base_dir=None, singlenode=True, *args, **kwargs):
         # dj: should be changed for wf
         self.name = name
-        self.nodedir = base_dir
+        self._nodedir = base_dir
         self._interface = interface
         # dj: do I need a state_input and state_mapper??
         # dj: reading the input from files should be added
@@ -1209,15 +1217,20 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
             self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
             self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
                                 for (key, val) in self._inputs.items())
+            self._state_inputs = self._inputs.copy()
         else:
             self._inputs = {}
+            self._state_inputs = {}
         if mapper:
             # adding name of the node to the input name within the mapper
             mapper = aux.change_mapper(mapper, self.name)
         self._mapper = mapper
         # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
         self._state = state.State(mapper=self._mapper, node_name=self.name)
-
+        if singlenode:
+            self.nodecore = NewNodeCore(name=self.name, base_dir=self.nodedir, interface=self._interface,
+            inputs=self._inputs, state_inputs=self._state_inputs, state=self._state)
+        self.sufficient = True
 
     def map(self, mapper, inputs=None):
         if self._mapper:
@@ -1230,6 +1243,7 @@ def map(self, mapper, inputs=None):
             inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
                           for (key, val) in inputs.items())
             self._inputs.update(inputs)
+            self._state_inputs.update(inputs)
         if mapper:
             # updating state if we have a new mapper
             self._state = state.State(mapper=self._mapper, node_name=self.name)
@@ -1255,16 +1269,42 @@ def join(self, field):
     def state(self):
         return self._state
 
+    @property
+    def nodedir(self):
+        return self._nodedir
+
+    @nodedir.setter
+    def nodedir(self, nodedir):
+        self._nodedir = nodedir
+        self.nodecore.nodedir = nodedir
+
 
     @property
     def mapper(self):
         return self._mapper
 
+    @mapper.setter
+    def mapper(self, mapper):
+        self._mapper = mapper
+        #updating state
+        self._state = state.State(mapper=self._mapper, node_name=self.name)
 
     @property
     def inputs(self):
         return self._inputs
 
+    @property
+    def state_inputs(self):
+        return self.nodecore._state_inputs
+
+    @property
+    def global_done(self):
+        return self.nodecore.global_done
+
+
+    @property
+    def needed_outputs(self):
+        return self.nodecore.needed_outputs
 
     @property
     def result(self):
@@ -1272,37 +1312,125 @@ def result(self):
 
 
     def prepare_state_input(self):
-        self._state.prepare_state_input(state_inputs=self._inputs)
+        self._state.prepare_state_input(state_inputs=self.state_inputs)
+        # updating node
+        self.nodecore.state = self._state
+        self.nodecore._inputs = self._inputs
+        self.nodecore._state_inputs = self._state_inputs
 
     def run(self, plugin="serial"):
         self.prepare_state_input()
-        self.nodecore = NewNodeCore(name=self.name, base_dir=self.nodedir, interface=self._interface,
-                                    inputs=self._inputs, state=self._state)
         self.sub = sub.SubmitterNode(plugin, node=self.nodecore)
         self.sub.run_node()
         self.sub.close()
 
 
+class NewWorkflow(NewNode):
+    def __init__(self, inputs=None, mapper=None, #join_by=None,
+                 base_dir=None, nodes=None, workingdir=None, *args, **kwargs):
+        # dj: workflow can't have interface...
+        super(NewWorkflow, self).__init__(inputs=inputs, mapper=mapper, interface=None,
+                 base_dir=base_dir, singlenode=False, *args, **kwargs)
 
+        self.graph = nx.DiGraph()
+        self._nodes = []
+        self.connected_var = {}
+        if nodes:
+            self.add_nodes(nodes)
+        for nn in self._nodes:
+            self.connected_var[nn] = {}
 
-class NewWorkflow(NewNode):
-    def __init__(self, inputs={}, *args, **kwargs):
-        super(NewWorkflow, self).__init__(*args, **kwargs)
-
-        self._nodes = {}
-
-        mro = self.__class__.mro()
-        wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
-        items = {}
-        for klass in wf_klasses:
-            items.update(klass.__dict__)
-        for name, runnable in items.items():
-            if name in ('__module__', '__doc__'):
-                continue
+        # dj: I have nodedir and workingdir...
+        self.workingdir = os.path.join(os.getcwd(), workingdir)
+
+        # dj not sure what was the motivation, wf_klasses gives an empty list
+        #mro = self.__class__.mro()
+        #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
+        #items = {}
+        #for klass in wf_klasses:
+        #    items.update(klass.__dict__)
+        #for name, runnable in items.items():
+        #    if name in ('__module__', '__doc__'):
+        #        continue
+
+        #    self.add(name, value)
+
+
+
+    @property
+    def nodes(self):
+        return self._nodes
+
+    def add_nodes(self, nodes):
+        """adding nodes without defining connections"""
+        self._nodes = nodes
+        self.graph.add_nodes_from(nodes)
+        for nn in nodes:
+            #self._inputs.update(nn.inputs)
+            self.connected_var[nn] = {}
+            nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
+
+
+    def connect(self, from_node, from_socket, to_node, to_socket):
+        self.graph.add_edges_from([(from_node, to_node)])
+        if not to_node in self.nodes:
+            self.add_nodes(to_node)
+        self.connected_var[to_node][to_socket] = (from_node, from_socket)
+        # from_node.sending_output.append((from_socket, to_node, to_socket))
+
+        logger.debug('connecting {} and {}'.format(from_node, to_node))
+
+
+    def _preparing(self):
+        """preparing nodes which are connected: setting the final mapper and state_inputs"""
+        self.graph_sorted = list(nx.topological_sort(self.graph))
+        logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
+        for nn in self.graph_sorted:
+            nn.wfdir = self.workingdir
+            try:
+                for inp, (out_node, out_var) in self.connected_var[nn].items():
+                    nn.sufficient = False #it has some history (doesnt have to be in the loop)
+                    nn.state_inputs.update(out_node.state_inputs)
+                    nn.needed_outputs.append((out_node, out_var, inp))
+                    #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
+                    if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
+                        nn.mapper = out_node.mapper
+                        #nn._mapper = inp #not used
+                    elif not out_node.mapper: # we shouldn't change anything
+                        pass
+                    # when the mapper from previous node is used in the current node (it has to be the same syntax)
+                    elif nn.mapper and out_node.mapper in nn.mapper:  # state_mapper or _mapper?? TODO
+                        #dj: if I use the syntax with state_inp name than I don't have to change the mapper...
+                        #if type(nn._mapper) is tuple:
+                        #    nn._mapper = tuple([inp if x == out_node.state_mapper else x for x in list(nn._mapper)])
+                        # TODO: not sure if I'll have to implement more
+                        pass
+
+                    #TODO: implement inner mapper
+                    # TODO: if nn.mapper is a string and inp can be a letter that exists in nn.mapper
+                    #elif nn.mapper and inp in nn.mapper:
+                    #    raise Exception("{} can be in the mapper only together with {}, i.e. {})".format(inp, out[1],
+                    #                                                                                    [out[1], inp]))
+                    else:
+                        raise Exception("worflow._preparing: should I implement something more?")
+                pass
+            except(KeyError):
+                # tmp: we don't care about nn that are not in self.connected_var
+                pass
+
+            #nn.preparing_node() #dj: only one this needed?:
+            # do i need it at all?
+            nn.prepare_state_input()
+
+
+    def run(self, plugin="serial"):
+        self._preparing()
+        self.sub = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
+        self.sub.run_workflow()
+        self.sub.close()
 
-            self.add(name, value)
 
-    def add(self, name, runnable):
+    def add_orig(self, name, runnable):
         if is_function(runnable):
             node = Node(Function(function=runnable), name=name)
         elif is_interface(runnable):
@@ -1315,7 +1443,7 @@ def add(self, name, runnable):
         self._nodes[name] = node
         self._last_added = name
 
-    def map(self, field, node=None, values=None):
+    def map_orig(self, field, node=None, values=None):
         if node is None:
             if '.' in field:
                 node, field = field.rsplit('.', 1)

From 40562a80af87b130361215a4e5973575d95a6e01 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 24 Jul 2018 16:22:35 -0400
Subject: [PATCH 17/55] adding add and method methods to NewWorkflow

---
 nipype/pipeline/engine/tests/test_newnode.py | 226 ++++++++++++++++++-
 nipype/pipeline/engine/workflows.py          |  75 +++---
 2 files changed, 270 insertions(+), 31 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 8a75464042..a65093ba10 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -165,7 +165,7 @@ def test_node_8(plugin):
 @python35_only
 def test_workflow_1(plugin):
     """one node with a mapper"""
-    wf = NewWorkflow(name="wf1", workingdir="test_wf1_".format(plugin))
+    wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
@@ -186,7 +186,7 @@ def test_workflow_1(plugin):
 @python35_only
 def test_workflow_2(plugin):
     """workflow with 2 nodes, second node without mapper"""
-    wf = NewWorkflow(name="wf2", workingdir="test_wf2_".format(plugin))
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
@@ -222,7 +222,7 @@ def test_workflow_2(plugin):
 @python35_only
 def test_workflow_2a(plugin):
     """workflow with 2 nodes, second node with a scalar mapper"""
-    wf = NewWorkflow(name="wf2", workingdir="test_wf2a_".format(plugin))
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
@@ -235,6 +235,7 @@ def test_workflow_2a(plugin):
     wf.connect(na, "out", nb, "a")
 
     assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[1].mapper == ("NA-a", "NB-b")
     wf.run(plugin=plugin)
 
     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
@@ -258,7 +259,7 @@ def test_workflow_2a(plugin):
 @python35_only
 def test_workflow_2b(plugin):
     """workflow with 2 nodes, second node with a vector mapper"""
-    wf = NewWorkflow(name="wf2", workingdir="test_wf2b_".format(plugin))
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
@@ -272,6 +273,7 @@ def test_workflow_2b(plugin):
     wf.connect(na, "out", nb, "a")
 
     assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[1].mapper == ["NA-a", "NB-b"]
     wf.run(plugin=plugin)
 
     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
@@ -291,3 +293,219 @@ def test_workflow_2b(plugin):
     for i, res in enumerate(expected_B):
         assert wf.nodes[1].result["out"][i][0] == res[0]
         assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# using add method to add nodes
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_3(plugin):
+    """using add(node) method"""
+    wf = NewWorkflow(name="wf3", workingdir="test_wf3_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    wf.add(na)
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_3a(plugin):
+    """using add(interface) method"""
+    wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+
+    wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_3b(plugin):
+    """using add(interface) method"""
+    wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
+
+    wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_4(plugin):
+    """using add(node) method"""
+    wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(na)
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    wf.add(nb)
+
+    wf.connect(na, "out", nb, "a")
+
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# using map after add method
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_5(plugin):
+    """using a map method for one node"""
+    wf = NewWorkflow(name="wf5", workingdir="test_wf5_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_5a(plugin):
+    """using a map method for one node (using add and map in one chain)"""
+    wf = NewWorkflow(name="wf5a", workingdir="test_wf5a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_6(plugin):
+    """using a map method for two nodes (using last added node as default)"""
+    wf = NewWorkflow(name="wf6", workingdir="test_wf6_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+
+    wf.add(na)
+    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(nb)
+    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    wf.connect(na, "out", nb, "a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_6a(plugin):
+    """using a map method for two nodes (specifying the node)"""
+    wf = NewWorkflow(name="wf6a", workingdir="test_wf6a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+
+    wf.add(na)
+    wf.add(nb)
+    wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
+    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
+    wf.connect(na, "out", nb, "a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 1245475465..5028ffc939 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1363,9 +1363,9 @@ def nodes(self):
 
     def add_nodes(self, nodes):
         """adding nodes without defining connections"""
-        self._nodes = nodes
         self.graph.add_nodes_from(nodes)
         for nn in nodes:
+            self._nodes.append(nn)
             #self._inputs.update(nn.inputs)
             self.connected_var[nn] = {}
             nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
@@ -1429,36 +1429,58 @@ def run(self, plugin="serial"):
         self.sub.run_workflow()
         self.sub.close()
 
-
-    def add_orig(self, name, runnable):
+#zamienic na node a pozniej add_node
+    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
+        # dj TODO: should I move this if checks to NewNode __init__?
         if is_function(runnable):
-            node = Node(Function(function=runnable), name=name)
+            if not output_nm:
+                output_nm = ["out"]
+            interface = aux.Function_Interface(function=runnable, output_nm=output_nm)
+            if not name:
+                raise Exception("you have to specify name for the node")
+            if not base_dir:
+                base_dir = name
+            node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
         elif is_interface(runnable):
-            node = Node(runnable, name=name)
+            if not name:
+                raise Exception("you have to specify name for the node")
+            if not base_dir:
+                base_dir = name
+            node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
         elif is_node(runnable):
-            node = runnable if runnable.name == name else runnable.clone(name=name)
+            node = runnable
+            #dj: dont have clonning right now
+            #node = runnable if runnable.name == name else runnable.clone(name=name)
         else:
             raise ValueError("Unknown workflow element: {!r}".format(runnable))
-        setattr(self, name, node)
-        self._nodes[name] = node
-        self._last_added = name
-
-    def map_orig(self, field, node=None, values=None):
-        if node is None:
-            if '.' in field:
-                node, field = field.rsplit('.', 1)
-            else:
-                node = self._last_added
-
-        if '.' in node:
-            subwf, node = node.split('.', 1)
-            self._nodes[subwf].map(field, node, values)
-            return
-
-        if node in self._mappers:
+        self.add_nodes([node])
+        # dj: i'm using name as a name of a workflow
+        #setattr(self, name, node)
+        #self._nodes[name] = node
+        self._last_added = node #name
+        #dj: so I can call map right away
+        return self
+
+
+    def map(self, mapper, node=None, inputs=None):
+        # if node is None:
+        #     if '.' in field:
+        #         node, field = field.rsplit('.', 1)
+        #     else:
+        #         node = self._last_added
+        #
+        # if '.' in node:
+        #     subwf, node = node.split('.', 1)
+        #     self._nodes[subwf].map(field, node, values)
+        #     return
+
+        if not node:
+            node = self._last_added
+
+        if node.mapper:
             raise WorkflowError("Cannot assign two mappings to the same input")
+        node.map(mapper=mapper, inputs=inputs)
 
-        self._mappers[node] = (field, values)
 
     def join(self, field, node=None):
         pass
@@ -1469,9 +1491,8 @@ def is_function(obj):
 
 
 def is_interface(obj):
-    return all(hasattr(obj, protocol)
-               for protocol in ('input_spec', 'output_spec', 'run'))
+    return type(obj) is aux.Function_Interface
 
 
 def is_node(obj):
-    return hasattr(obj, itername)
+    return type(obj) is NewNode

From 9fb54c3960ccbd0a917041103746c746860086ff Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 24 Jul 2018 16:25:41 -0400
Subject: [PATCH 18/55] fixing one test

---
 nipype/pipeline/engine/tests/test_newnode.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index a65093ba10..8d658eaa36 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -8,6 +8,7 @@
 python35_only = pytest.mark.skipif(sys.version_info < (3, 5),
                                    reason="requires Python>3.4")
 
+Plugins = ["mp", "serial", "cf", "dask"]
 
 def fun_addtwo(a):
     time.sleep(3)
@@ -79,11 +80,9 @@ def test_node_5():
     assert nn.state.state_values([1]) == {"NA-a": 5}
 
 
-Plugins = ["mp", "serial", "cf", "dask"] 
-
-#@pytest.mark.parametrize("plugin", Plugins)
-#@python35_only
-def test_node_6(plugin="serial"):
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_node_6(plugin):
     """Node with interface and inputs, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, base_dir="test6_{}".format(plugin))

From 39e6b5002135598a0a60c851cc23d7167f716521 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 24 Jul 2018 19:22:55 -0400
Subject: [PATCH 19/55] adding add method to NewWOrkflow

---
 nipype/pipeline/engine/tests/test_newnode.py | 106 +++++++++++++++++++
 nipype/pipeline/engine/workflows.py          |  26 +++--
 2 files changed, 126 insertions(+), 6 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 8d658eaa36..d8a4b52a57 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -508,3 +508,109 @@ def test_workflow_6a(plugin):
     for i, res in enumerate(expected_B):
         assert wf.nodes[1].result["out"][i][0] == res[0]
         assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# tests for a workflow that have its own input
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_7(plugin):
+    """using inputs for workflow and connect_workflow"""
+    wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.connect_workflow(na, "wf_a","a")
+    wf.map(mapper="a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_7a(plugin):
+    """using inputs for workflow and connect(None...)"""
+    wf = NewWorkflow(name="wf7a", inputs={"wf_a": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.connect(None, "wf_a", na, "a")
+    wf.map(mapper="a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_8(plugin):
+    """using inputs for workflow and connect_workflow for the second node"""
+    wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+    wf.connect_workflow(nb, "b", "b")
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# tests for a workflow that have its own input and mapper
+
+#@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+@pytest.mark.xfail(reason="mapper in workflow still not impplemented")
+def test_workflow_9(plugin="serial"):
+    """using inputs for workflow and connect_workflow"""
+    wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf9_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.connect_workflow(na, "wf_a","a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 5028ffc939..838386a5fd 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1343,6 +1343,10 @@ def __init__(self, inputs=None, mapper=None, #join_by=None,
         # dj: I have nodedir and workingdir...
         self.workingdir = os.path.join(os.getcwd(), workingdir)
 
+        if self.mapper:
+            #dj: TODO have to implement mapper for workflow. should I create as many workflows??
+            pass
+
         # dj not sure what was the motivation, wf_klasses gives an empty list
         #mro = self.__class__.mro()
         #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
@@ -1372,13 +1376,23 @@ def add_nodes(self, nodes):
 
 
     def connect(self, from_node, from_socket, to_node, to_socket):
-        self.graph.add_edges_from([(from_node, to_node)])
-        if not to_node in self.nodes:
-            self.add_nodes(to_node)
-        self.connected_var[to_node][to_socket] = (from_node, from_socket)
-        # from_node.sending_output.append((from_socket, to_node, to_socket))
+        if from_node:
+            self.graph.add_edges_from([(from_node, to_node)])
+            if not to_node in self.nodes:
+                self.add_nodes(to_node)
+            self.connected_var[to_node][to_socket] = (from_node, from_socket)
+            # from_node.sending_output.append((from_socket, to_node, to_socket))
+            logger.debug('connecting {} and {}'.format(from_node, to_node))
+        else:
+            self.connect_workflow(to_node, from_socket, to_socket)
 
-        logger.debug('connecting {} and {}'.format(from_node, to_node))
+
+    def connect_workflow(self, node, inp_wf, inp_nd):
+        if "{}-{}".format(self.name, inp_wf) in self.inputs:
+            node.state_inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
+            node.inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
+        else:
+            raise Exception("{} not in the workflow inputs".format(inp_wf))
 
 
     def _preparing(self):

From e7959614be8c64e29dec0dbf713fca891aed7ca6 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 24 Jul 2018 19:25:42 -0400
Subject: [PATCH 20/55] adding comments to the tests

---
 nipype/pipeline/engine/tests/test_newnode.py | 22 ++++++++++----------
 nipype/pipeline/engine/workflows.py          |  1 -
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index d8a4b52a57..e07bccbfe6 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -21,7 +21,7 @@ def fun_addvar(a, b):
 
 
 def test_node_1():
-    """Node with only mandatory arguments"""
+    """Node with mandatory arguments only"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo)
     assert nn.mapper is None
@@ -39,7 +39,7 @@ def test_node_2():
 
 
 def test_node_3():
-    """Node with interface and inputs"""
+    """Node with interface, inputs and mapper"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]}, mapper="a")
     assert nn.mapper == "NA-a"
@@ -53,7 +53,7 @@ def test_node_3():
 
 
 def test_node_4():
-    """Node with interface and inputs"""
+    """Node with interface and inputs. mapper set using map method"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]})
     nn.map(mapper="a")
@@ -67,7 +67,7 @@ def test_node_4():
 
 
 def test_node_5():
-    """Node with interface and inputs"""
+    """Node with interface, mapper and inputs set with the map method"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo)
     nn.map(mapper="a", inputs={"a": [3, 5]})
@@ -83,7 +83,7 @@ def test_node_5():
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_node_6(plugin):
-    """Node with interface and inputs, running interface"""
+    """Node with interface, inputs and the simplest mapper, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, base_dir="test6_{}".format(plugin))
     nn.map(mapper="a", inputs={"a": [3, 5]})
@@ -108,7 +108,7 @@ def test_node_6(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_node_7(plugin):
-    """Node with interface and inputs, running interface"""
+    """Node with interface, inputs and scalar mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nn = NewNode(name="NA", interface=interf_addvar, base_dir="test7_{}".format(plugin))
     nn.map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [2, 1]})
@@ -134,7 +134,7 @@ def test_node_7(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_node_8(plugin):
-    """Node with interface and inputs, running interface"""
+    """Node with interface, inputs and vector mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nn = NewNode(name="NA", interface=interf_addvar, base_dir="test8_{}".format(plugin))
     nn.map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [2, 1]})
@@ -163,7 +163,7 @@ def test_node_8(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_1(plugin):
-    """one node with a mapper"""
+    """workflow with one node with a mapper"""
     wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
@@ -184,7 +184,7 @@ def test_workflow_1(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_2(plugin):
-    """workflow with 2 nodes, second node without mapper"""
+    """workflow with two nodes, second node without mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
@@ -220,7 +220,7 @@ def test_workflow_2(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_2a(plugin):
-    """workflow with 2 nodes, second node with a scalar mapper"""
+    """workflow with two nodes, second node with a scalar mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
@@ -257,7 +257,7 @@ def test_workflow_2a(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_2b(plugin):
-    """workflow with 2 nodes, second node with a vector mapper"""
+    """workflow with two nodes, second node with a vector mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 838386a5fd..a1babb2023 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1443,7 +1443,6 @@ def run(self, plugin="serial"):
         self.sub.run_workflow()
         self.sub.close()
 
-#zamienic na node a pozniej add_node
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
         # dj TODO: should I move this if checks to NewNode __init__?
         if is_function(runnable):

From 15b4ba8552feda36f3ae718bc609d02802ddabef Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Thu, 26 Jul 2018 09:09:56 -0400
Subject: [PATCH 21/55] [wip] rearranging classes - showing the problem with
 cf/dask when passisng node=self to SumitterNode

---
 nipype/pipeline/engine/submitter.py          |   6 +-
 nipype/pipeline/engine/tests/test_newnode.py | 908 +++++++++----------
 nipype/pipeline/engine/workflows.py          | 576 ++++++------
 3 files changed, 737 insertions(+), 753 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 0f5e4b3b14..f11a276a5a 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -68,7 +68,7 @@ def run_workflow(self):
         for (i_n, node) in enumerate(self.graph):
             # submitting all the nodes who are self sufficient (self.graph is already sorted)
             if node.sufficient:
-                self.submit_work(node.nodecore)
+                self.submit_work(node)
             # if its not, its been added to a line
             else:
                 break
@@ -103,8 +103,8 @@ def _nodes_check(self):
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
             print("NODE LINE", self.node_line)
-            if to_node.nodecore.checking_input_el(ind):
-                self._submit_work_el(to_node.nodecore, i, ind)
+            if to_node.checking_input_el(ind):
+                self._submit_work_el(to_node, i, ind)
                 _to_remove.append((to_node, i, ind))
             else:
                 pass
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index e07bccbfe6..535322f769 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -160,457 +160,457 @@ def test_node_8(plugin):
 
 # tests for workflows that set mapper to node that are later added to a workflow
 
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_1(plugin):
-    """workflow with one node with a mapper"""
-    wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    na.map(mapper="a", inputs={"a": [3, 5]})
-    wf.add_nodes([na])
-    assert wf.nodes[0].mapper == "NA-a"
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_2(plugin):
-    """workflow with two nodes, second node without mapper"""
-    wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    na.map(mapper="a", inputs={"a": [3, 5]})
-
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, base_dir="nb")
-
-    wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
-
-    assert wf.nodes[0].mapper == "NA-a"
-    wf.run(plugin=plugin)
-
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected_A[0][0].keys())
-    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_A):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
-    key_sort = list(expected_B[0][0].keys())
-    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_B):
-        assert wf.nodes[1].result["out"][i][0] == res[0]
-        assert wf.nodes[1].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_2a(plugin):
-    """workflow with two nodes, second node with a scalar mapper"""
-    wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    na.map(mapper="a", inputs={"a": [3, 5]})
-
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
-
-    wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
-
-    assert wf.nodes[0].mapper == "NA-a"
-    assert wf.nodes[1].mapper == ("NA-a", "NB-b")
-    wf.run(plugin=plugin)
-
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected_A[0][0].keys())
-    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_A):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-    key_sort = list(expected_B[0][0].keys())
-    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_B):
-        assert wf.nodes[1].result["out"][i][0] == res[0]
-        assert wf.nodes[1].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_2b(plugin):
-    """workflow with two nodes, second node with a vector mapper"""
-    wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    na.map(mapper="a", inputs={"a": [3, 5]})
-
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-    nb.map(mapper=["NA-a", "b"], inputs={"b": [2, 1]})
-
-
-    wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
-
-    assert wf.nodes[0].mapper == "NA-a"
-    assert wf.nodes[1].mapper == ["NA-a", "NB-b"]
-    wf.run(plugin=plugin)
-
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected_A[0][0].keys())
-    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_A):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-    expected_B = [({"NA-a": 3, "NB-b": 1}, 6), ({"NA-a": 3, "NB-b": 2}, 7),
-                  ({"NA-a": 5, "NB-b": 1}, 8), ({"NA-a": 5, "NB-b": 2}, 9)]
-    key_sort = list(expected_B[0][0].keys())
-    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_B):
-        assert wf.nodes[1].result["out"][i][0] == res[0]
-        assert wf.nodes[1].result["out"][i][1] == res[1]
-
-
-# using add method to add nodes
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_3(plugin):
-    """using add(node) method"""
-    wf = NewWorkflow(name="wf3", workingdir="test_wf3_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    na.map(mapper="a", inputs={"a": [3, 5]})
-
-    wf.add(na)
-
-    assert wf.nodes[0].mapper == "NA-a"
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_3a(plugin):
-    """using add(interface) method"""
-    wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-
-    wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
-
-    assert wf.nodes[0].mapper == "NA-a"
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_3b(plugin):
-    """using add(interface) method"""
-    wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
-
-    wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
-
-    assert wf.nodes[0].mapper == "NA-a"
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_4(plugin):
-    """using add(node) method"""
-    wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    na.map(mapper="a", inputs={"a": [3, 5]})
-    wf.add(na)
-
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
-    wf.add(nb)
-
-    wf.connect(na, "out", nb, "a")
-
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-    key_sort = list(expected_B[0][0].keys())
-    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_B):
-        assert wf.nodes[1].result["out"][i][0] == res[0]
-        assert wf.nodes[1].result["out"][i][1] == res[1]
-
-
-# using map after add method
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_5(plugin):
-    """using a map method for one node"""
-    wf = NewWorkflow(name="wf5", workingdir="test_wf5_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    wf.add(na)
-    wf.map(mapper="a", inputs={"a": [3, 5]})
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_5a(plugin):
-    """using a map method for one node (using add and map in one chain)"""
-    wf = NewWorkflow(name="wf5a", workingdir="test_wf5a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_6(plugin):
-    """using a map method for two nodes (using last added node as default)"""
-    wf = NewWorkflow(name="wf6", workingdir="test_wf6_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-
-    wf.add(na)
-    wf.map(mapper="a", inputs={"a": [3, 5]})
-    wf.add(nb)
-    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
-    wf.connect(na, "out", nb, "a")
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-    key_sort = list(expected_B[0][0].keys())
-    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_B):
-        assert wf.nodes[1].result["out"][i][0] == res[0]
-        assert wf.nodes[1].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_6a(plugin):
-    """using a map method for two nodes (specifying the node)"""
-    wf = NewWorkflow(name="wf6a", workingdir="test_wf6a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-
-    wf.add(na)
-    wf.add(nb)
-    wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
-    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
-    wf.connect(na, "out", nb, "a")
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-    key_sort = list(expected_B[0][0].keys())
-    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_B):
-        assert wf.nodes[1].result["out"][i][0] == res[0]
-        assert wf.nodes[1].result["out"][i][1] == res[1]
-
-
-# tests for a workflow that have its own input
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_7(plugin):
-    """using inputs for workflow and connect_workflow"""
-    wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    wf.add(na)
-    wf.connect_workflow(na, "wf_a","a")
-    wf.map(mapper="a")
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_7a(plugin):
-    """using inputs for workflow and connect(None...)"""
-    wf = NewWorkflow(name="wf7a", inputs={"wf_a": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    wf.add(na)
-    wf.connect(None, "wf_a", na, "a")
-    wf.map(mapper="a")
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_8(plugin):
-    """using inputs for workflow and connect_workflow for the second node"""
-    wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    na.map(mapper="a", inputs={"a": [3, 5]})
-
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-
-    wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
-    wf.connect_workflow(nb, "b", "b")
-    assert wf.nodes[0].mapper == "NA-a"
-    wf.run(plugin=plugin)
-
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected_A[0][0].keys())
-    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_A):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
-    key_sort = list(expected_B[0][0].keys())
-    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected_B):
-        assert wf.nodes[1].result["out"][i][0] == res[0]
-        assert wf.nodes[1].result["out"][i][1] == res[1]
-
-
-# tests for a workflow that have its own input and mapper
-
-#@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-@pytest.mark.xfail(reason="mapper in workflow still not impplemented")
-def test_workflow_9(plugin="serial"):
-    """using inputs for workflow and connect_workflow"""
-    wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf9_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    wf.add(na)
-    wf.connect_workflow(na, "wf_a","a")
-    wf.run(plugin=plugin)
-
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_1(plugin):
+#     """workflow with one node with a mapper"""
+#     wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#     na.map(mapper="a", inputs={"a": [3, 5]})
+#     wf.add_nodes([na])
+#     assert wf.nodes[0].mapper == "NA-a"
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_2(plugin):
+#     """workflow with two nodes, second node without mapper"""
+#     wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#     na.map(mapper="a", inputs={"a": [3, 5]})
+#
+#     interf_addvar = Function_Interface(fun_addvar, ["out"])
+#     nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, base_dir="nb")
+#
+#     wf.add_nodes([na, nb])
+#     wf.connect(na, "out", nb, "a")
+#
+#     assert wf.nodes[0].mapper == "NA-a"
+#     wf.run(plugin=plugin)
+#
+#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected_A[0][0].keys())
+#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_A):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+#     expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+#     key_sort = list(expected_B[0][0].keys())
+#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_B):
+#         assert wf.nodes[1].result["out"][i][0] == res[0]
+#         assert wf.nodes[1].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_2a(plugin):
+#     """workflow with two nodes, second node with a scalar mapper"""
+#     wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#     na.map(mapper="a", inputs={"a": [3, 5]})
+#
+#     interf_addvar = Function_Interface(fun_addvar, ["out"])
+#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+#     nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+#
+#     wf.add_nodes([na, nb])
+#     wf.connect(na, "out", nb, "a")
+#
+#     assert wf.nodes[0].mapper == "NA-a"
+#     assert wf.nodes[1].mapper == ("NA-a", "NB-b")
+#     wf.run(plugin=plugin)
+#
+#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected_A[0][0].keys())
+#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_A):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+#     key_sort = list(expected_B[0][0].keys())
+#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_B):
+#         assert wf.nodes[1].result["out"][i][0] == res[0]
+#         assert wf.nodes[1].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_2b(plugin):
+#     """workflow with two nodes, second node with a vector mapper"""
+#     wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#     na.map(mapper="a", inputs={"a": [3, 5]})
+#
+#     interf_addvar = Function_Interface(fun_addvar, ["out"])
+#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+#     nb.map(mapper=["NA-a", "b"], inputs={"b": [2, 1]})
+#
+#
+#     wf.add_nodes([na, nb])
+#     wf.connect(na, "out", nb, "a")
+#
+#     assert wf.nodes[0].mapper == "NA-a"
+#     assert wf.nodes[1].mapper == ["NA-a", "NB-b"]
+#     wf.run(plugin=plugin)
+#
+#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected_A[0][0].keys())
+#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_A):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+#     expected_B = [({"NA-a": 3, "NB-b": 1}, 6), ({"NA-a": 3, "NB-b": 2}, 7),
+#                   ({"NA-a": 5, "NB-b": 1}, 8), ({"NA-a": 5, "NB-b": 2}, 9)]
+#     key_sort = list(expected_B[0][0].keys())
+#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_B):
+#         assert wf.nodes[1].result["out"][i][0] == res[0]
+#         assert wf.nodes[1].result["out"][i][1] == res[1]
+#
+#
+# # using add method to add nodes
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_3(plugin):
+#     """using add(node) method"""
+#     wf = NewWorkflow(name="wf3", workingdir="test_wf3_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#     na.map(mapper="a", inputs={"a": [3, 5]})
+#
+#     wf.add(na)
+#
+#     assert wf.nodes[0].mapper == "NA-a"
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_3a(plugin):
+#     """using add(interface) method"""
+#     wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#
+#     wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+#
+#     assert wf.nodes[0].mapper == "NA-a"
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_3b(plugin):
+#     """using add(interface) method"""
+#     wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
+#
+#     wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+#
+#     assert wf.nodes[0].mapper == "NA-a"
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_4(plugin):
+#     """using add(node) method"""
+#     wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#     na.map(mapper="a", inputs={"a": [3, 5]})
+#     wf.add(na)
+#
+#     interf_addvar = Function_Interface(fun_addvar, ["out"])
+#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+#     nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+#     wf.add(nb)
+#
+#     wf.connect(na, "out", nb, "a")
+#
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+#     key_sort = list(expected_B[0][0].keys())
+#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_B):
+#         assert wf.nodes[1].result["out"][i][0] == res[0]
+#         assert wf.nodes[1].result["out"][i][1] == res[1]
+#
+#
+# # using map after add method
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_5(plugin):
+#     """using a map method for one node"""
+#     wf = NewWorkflow(name="wf5", workingdir="test_wf5_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#
+#     wf.add(na)
+#     wf.map(mapper="a", inputs={"a": [3, 5]})
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_5a(plugin):
+#     """using a map method for one node (using add and map in one chain)"""
+#     wf = NewWorkflow(name="wf5a", workingdir="test_wf5a_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#
+#     wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_6(plugin):
+#     """using a map method for two nodes (using last added node as default)"""
+#     wf = NewWorkflow(name="wf6", workingdir="test_wf6_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#
+#     interf_addvar = Function_Interface(fun_addvar, ["out"])
+#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+#
+#     wf.add(na)
+#     wf.map(mapper="a", inputs={"a": [3, 5]})
+#     wf.add(nb)
+#     wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+#     wf.connect(na, "out", nb, "a")
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+#     key_sort = list(expected_B[0][0].keys())
+#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_B):
+#         assert wf.nodes[1].result["out"][i][0] == res[0]
+#         assert wf.nodes[1].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_6a(plugin):
+#     """using a map method for two nodes (specifying the node)"""
+#     wf = NewWorkflow(name="wf6a", workingdir="test_wf6a_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#
+#     interf_addvar = Function_Interface(fun_addvar, ["out"])
+#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+#
+#     wf.add(na)
+#     wf.add(nb)
+#     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
+#     wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
+#     wf.connect(na, "out", nb, "a")
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+#     key_sort = list(expected_B[0][0].keys())
+#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_B):
+#         assert wf.nodes[1].result["out"][i][0] == res[0]
+#         assert wf.nodes[1].result["out"][i][1] == res[1]
+#
+#
+# # tests for a workflow that have its own input
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_7(plugin):
+#     """using inputs for workflow and connect_workflow"""
+#     wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#
+#     wf.add(na)
+#     wf.connect_workflow(na, "wf_a","a")
+#     wf.map(mapper="a")
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_7a(plugin):
+#     """using inputs for workflow and connect(None...)"""
+#     wf = NewWorkflow(name="wf7a", inputs={"wf_a": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#
+#     wf.add(na)
+#     wf.connect(None, "wf_a", na, "a")
+#     wf.map(mapper="a")
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+# @pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# def test_workflow_8(plugin):
+#     """using inputs for workflow and connect_workflow for the second node"""
+#     wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#     na.map(mapper="a", inputs={"a": [3, 5]})
+#
+#     interf_addvar = Function_Interface(fun_addvar, ["out"])
+#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+#
+#     wf.add_nodes([na, nb])
+#     wf.connect(na, "out", nb, "a")
+#     wf.connect_workflow(nb, "b", "b")
+#     assert wf.nodes[0].mapper == "NA-a"
+#     wf.run(plugin=plugin)
+#
+#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected_A[0][0].keys())
+#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_A):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
+#
+#
+#     expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+#     key_sort = list(expected_B[0][0].keys())
+#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected_B):
+#         assert wf.nodes[1].result["out"][i][0] == res[0]
+#         assert wf.nodes[1].result["out"][i][1] == res[1]
+#
+#
+# # tests for a workflow that have its own input and mapper
+#
+# #@pytest.mark.parametrize("plugin", Plugins)
+# @python35_only
+# @pytest.mark.xfail(reason="mapper in workflow still not impplemented")
+# def test_workflow_9(plugin="serial"):
+#     """using inputs for workflow and connect_workflow"""
+#     wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf9_{}".format(plugin))
+#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+#
+#     wf.add(na)
+#     wf.connect_workflow(na, "wf_a","a")
+#     wf.run(plugin=plugin)
+#
+#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+#     key_sort = list(expected[0][0].keys())
+#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+#     for i, res in enumerate(expected):
+#         assert wf.nodes[0].result["out"][i][0] == res[0]
+#         assert wf.nodes[0].result["out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index a1babb2023..5847f391ba 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1072,33 +1072,111 @@ class MapState(object):
     pass
 
 # dj ??: should I use EngineBase?
-class NewNodeCore(object):
-    def __init__(self, name, interface, state, inputs=None, state_inputs=None, base_dir=None, *args, **kwargs):
+class NewBase(object):
+    def __init__(self, name, inputs=None, mapper=None, base_dir=None, *args, **kwargs):
         # adding interface: i'm using Function Interface from aux that has input_map that can change the name of arguments
-        self.nodedir = base_dir
+        self._nodedir = base_dir
         self.name = name
-        #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
-        self.state = state
-        self._inputs = inputs
-        self._state_inputs = state_inputs
+
+        if inputs:
+            # adding name of the node to the input name
+            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                                for (key, val) in self._inputs.items())
+            self._state_inputs = self._inputs.copy()
+        else:
+            self._inputs = {}
+            self._state_inputs = {}
+        if mapper:
+            # adding name of the node to the input name within the mapper
+            mapper = aux.change_mapper(mapper, self.name)
+        self._mapper = mapper
+        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
+        self._state = state.State(mapper=self._mapper, node_name=self.name)
+
+
+    @property
+    def state(self):
+        return self._state
+
+    @property
+    def nodedir(self):
+        return self._nodedir
+
+
+    @property
+    def mapper(self):
+        return self._mapper
+
+    # @mapper.setter
+    # def mapper(self, mapper):
+    #     self._mapper = mapper
+    #     #updating state
+    #     self._state = state.State(mapper=self._mapper, node_name=self.name)
+    #
+    @property
+    def inputs(self):
+        return self._inputs
+
+    @property
+    def state_inputs(self):
+        return self._state_inputs
+
+
+
+
+
+class NewNode(NewBase):
+    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
+                 base_dir=None, *args, **kwargs):
+        # dj: should be changed for wf
+        super(NewNode, self).__init__(name=name, inputs=inputs, mapper=mapper,
+                                      base_dir=base_dir, *args, **kwargs)
         self._interface = interface
         self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
                                          for (key, value) in self._interface.input_map.items())
 
         self.needed_outputs = []
         self._out_nm = self._interface._output_nm
-        self._global_done = False
-        self._result = {}
 
+        self.sufficient = True
+        # dj TODO: have to decide where to put this
+        # self._global_done = False
+        # self._result = {}
 
-    @property
-    def interface(self):
-        return self._interface
 
+    def map(self, mapper, inputs=None):
+        if self._mapper:
+            raise Exception("mapper is already set")
+        else:
+            self._mapper = aux.change_mapper(mapper, self.name)
+
+        if inputs:
+            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                          for (key, val) in inputs.items())
+            self._inputs.update(inputs)
+            self._state_inputs.update(inputs)
+        if mapper:
+            # updating state if we have a new mapper
+            self._state = state.State(mapper=self._mapper, node_name=self.name)
 
-    @property
-    def inputs(self):
-        return self._inputs
+#    def map_orig(self, field, values=None):
+#        if isinstance(field, list):
+#            for field_
+#            if values is not None:
+#                if len(values != len(field)):
+#        elif isinstance(field, tuple):
+#            pass
+#        if values is None:
+#            values = getattr(self._inputs, field)
+#            if values is None:
+#                raise MappingError('Cannot map unassigned input field')
+#        self._mappers[field] = values
+
+    # TBD
+    def join(self, field):
+        pass
 
 
     def run_interface_el(self, i, ind):
@@ -1202,125 +1280,31 @@ def _reading_results(self):
 
 
 
-
-class NewNode(object):
-    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, singlenode=True, *args, **kwargs):
-        # dj: should be changed for wf
-        self.name = name
-        self._nodedir = base_dir
-        self._interface = interface
-        # dj: do I need a state_input and state_mapper??
-        # dj: reading the input from files should be added
-        if inputs:
-            # adding name of the node to the input name
-            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                                for (key, val) in self._inputs.items())
-            self._state_inputs = self._inputs.copy()
-        else:
-            self._inputs = {}
-            self._state_inputs = {}
-        if mapper:
-            # adding name of the node to the input name within the mapper
-            mapper = aux.change_mapper(mapper, self.name)
-        self._mapper = mapper
-        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
-        self._state = state.State(mapper=self._mapper, node_name=self.name)
-        if singlenode:
-            self.nodecore = NewNodeCore(name=self.name, base_dir=self.nodedir, interface=self._interface,
-            inputs=self._inputs, state_inputs=self._state_inputs, state=self._state)
-        self.sufficient = True
-
-    def map(self, mapper, inputs=None):
-        if self._mapper:
-            raise Exception("mapper is already set")
-        else:
-            self._mapper = aux.change_mapper(mapper, self.name)
-
-        if inputs:
-            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                          for (key, val) in inputs.items())
-            self._inputs.update(inputs)
-            self._state_inputs.update(inputs)
-        if mapper:
-            # updating state if we have a new mapper
-            self._state = state.State(mapper=self._mapper, node_name=self.name)
-
-#    def map_orig(self, field, values=None):
-#        if isinstance(field, list):
-#            for field_
-#            if values is not None:
-#                if len(values != len(field)):
-#        elif isinstance(field, tuple):
-#            pass
-#        if values is None:
-#            values = getattr(self._inputs, field)
-#            if values is None:
-#                raise MappingError('Cannot map unassigned input field')
-#        self._mappers[field] = values
-
-    # TBD
-    def join(self, field):
-        pass
-
-    @property
-    def state(self):
-        return self._state
-
-    @property
-    def nodedir(self):
-        return self._nodedir
-
-    @nodedir.setter
-    def nodedir(self, nodedir):
-        self._nodedir = nodedir
-        self.nodecore.nodedir = nodedir
-
-
-    @property
-    def mapper(self):
-        return self._mapper
-
-    @mapper.setter
-    def mapper(self, mapper):
-        self._mapper = mapper
-        #updating state
-        self._state = state.State(mapper=self._mapper, node_name=self.name)
-
-    @property
-    def inputs(self):
-        return self._inputs
-
-    @property
-    def state_inputs(self):
-        return self.nodecore._state_inputs
-
-    @property
-    def global_done(self):
-        return self.nodecore.global_done
-
-
-    @property
-    def needed_outputs(self):
-        return self.nodecore.needed_outputs
-
-    @property
-    def result(self):
-        return self.nodecore.result
-
-
+     #
+    # @property
+    # def global_done(self):
+    #     return self.nodecore.global_done
+    #
+    #
+    # @property
+    # def needed_outputs(self):
+    #     return self.nodecore.needed_outputs
+    #
+    # @property
+    # def result(self):
+    #     return self.nodecore.result
+    #
+    #
     def prepare_state_input(self):
         self._state.prepare_state_input(state_inputs=self.state_inputs)
-        # updating node
-        self.nodecore.state = self._state
-        self.nodecore._inputs = self._inputs
-        self.nodecore._state_inputs = self._state_inputs
-
+    #     # updating node
+    #     self.nodecore.state = self._state
+    #     self.nodecore._inputs = self._inputs
+    #     self.nodecore._state_inputs = self._state_inputs
+    #
     def run(self, plugin="serial"):
         self.prepare_state_input()
-        self.sub = sub.SubmitterNode(plugin, node=self.nodecore)
+        self.sub = sub.SubmitterNode(plugin, node=self)
         self.sub.run_node()
         self.sub.close()
 
@@ -1332,172 +1316,172 @@ def __init__(self, inputs=None, mapper=None, #join_by=None,
         super(NewWorkflow, self).__init__(inputs=inputs, mapper=mapper, interface=None,
                  base_dir=base_dir, singlenode=False, *args, **kwargs)
 
-        self.graph = nx.DiGraph()
-        self._nodes = []
-        self.connected_var = {}
-        if nodes:
-            self.add_nodes(nodes)
-        for nn in self._nodes:
-            self.connected_var[nn] = {}
-
-        # dj: I have nodedir and workingdir...
-        self.workingdir = os.path.join(os.getcwd(), workingdir)
-
-        if self.mapper:
-            #dj: TODO have to implement mapper for workflow. should I create as many workflows??
-            pass
-
-        # dj not sure what was the motivation, wf_klasses gives an empty list
-        #mro = self.__class__.mro()
-        #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
-        #items = {}
-        #for klass in wf_klasses:
-        #    items.update(klass.__dict__)
-        #for name, runnable in items.items():
-        #    if name in ('__module__', '__doc__'):
-        #        continue
-
-        #    self.add(name, value)
-
-
-
-    @property
-    def nodes(self):
-        return self._nodes
-
-    def add_nodes(self, nodes):
-        """adding nodes without defining connections"""
-        self.graph.add_nodes_from(nodes)
-        for nn in nodes:
-            self._nodes.append(nn)
-            #self._inputs.update(nn.inputs)
-            self.connected_var[nn] = {}
-            nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
-
-
-    def connect(self, from_node, from_socket, to_node, to_socket):
-        if from_node:
-            self.graph.add_edges_from([(from_node, to_node)])
-            if not to_node in self.nodes:
-                self.add_nodes(to_node)
-            self.connected_var[to_node][to_socket] = (from_node, from_socket)
-            # from_node.sending_output.append((from_socket, to_node, to_socket))
-            logger.debug('connecting {} and {}'.format(from_node, to_node))
-        else:
-            self.connect_workflow(to_node, from_socket, to_socket)
-
-
-    def connect_workflow(self, node, inp_wf, inp_nd):
-        if "{}-{}".format(self.name, inp_wf) in self.inputs:
-            node.state_inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
-            node.inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
-        else:
-            raise Exception("{} not in the workflow inputs".format(inp_wf))
-
-
-    def _preparing(self):
-        """preparing nodes which are connected: setting the final mapper and state_inputs"""
-        self.graph_sorted = list(nx.topological_sort(self.graph))
-        logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
-        for nn in self.graph_sorted:
-            nn.wfdir = self.workingdir
-            try:
-                for inp, (out_node, out_var) in self.connected_var[nn].items():
-                    nn.sufficient = False #it has some history (doesnt have to be in the loop)
-                    nn.state_inputs.update(out_node.state_inputs)
-                    nn.needed_outputs.append((out_node, out_var, inp))
-                    #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
-                    if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
-                        nn.mapper = out_node.mapper
-                        #nn._mapper = inp #not used
-                    elif not out_node.mapper: # we shouldn't change anything
-                        pass
-                    # when the mapper from previous node is used in the current node (it has to be the same syntax)
-                    elif nn.mapper and out_node.mapper in nn.mapper:  # state_mapper or _mapper?? TODO
-                        #dj: if I use the syntax with state_inp name than I don't have to change the mapper...
-                        #if type(nn._mapper) is tuple:
-                        #    nn._mapper = tuple([inp if x == out_node.state_mapper else x for x in list(nn._mapper)])
-                        # TODO: not sure if I'll have to implement more
-                        pass
-
-                    #TODO: implement inner mapper
-                    # TODO: if nn.mapper is a string and inp can be a letter that exists in nn.mapper
-                    #elif nn.mapper and inp in nn.mapper:
-                    #    raise Exception("{} can be in the mapper only together with {}, i.e. {})".format(inp, out[1],
-                    #                                                                                    [out[1], inp]))
-                    else:
-                        raise Exception("worflow._preparing: should I implement something more?")
-                pass
-            except(KeyError):
-                # tmp: we don't care about nn that are not in self.connected_var
-                pass
-
-            #nn.preparing_node() #dj: only one this needed?:
-            # do i need it at all?
-            nn.prepare_state_input()
-
-
-    def run(self, plugin="serial"):
-        self._preparing()
-        self.sub = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
-        self.sub.run_workflow()
-        self.sub.close()
-
-    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
-        # dj TODO: should I move this if checks to NewNode __init__?
-        if is_function(runnable):
-            if not output_nm:
-                output_nm = ["out"]
-            interface = aux.Function_Interface(function=runnable, output_nm=output_nm)
-            if not name:
-                raise Exception("you have to specify name for the node")
-            if not base_dir:
-                base_dir = name
-            node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
-        elif is_interface(runnable):
-            if not name:
-                raise Exception("you have to specify name for the node")
-            if not base_dir:
-                base_dir = name
-            node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
-        elif is_node(runnable):
-            node = runnable
-            #dj: dont have clonning right now
-            #node = runnable if runnable.name == name else runnable.clone(name=name)
-        else:
-            raise ValueError("Unknown workflow element: {!r}".format(runnable))
-        self.add_nodes([node])
-        # dj: i'm using name as a name of a workflow
-        #setattr(self, name, node)
-        #self._nodes[name] = node
-        self._last_added = node #name
-        #dj: so I can call map right away
-        return self
-
-
-    def map(self, mapper, node=None, inputs=None):
-        # if node is None:
-        #     if '.' in field:
-        #         node, field = field.rsplit('.', 1)
-        #     else:
-        #         node = self._last_added
-        #
-        # if '.' in node:
-        #     subwf, node = node.split('.', 1)
-        #     self._nodes[subwf].map(field, node, values)
-        #     return
-
-        if not node:
-            node = self._last_added
-
-        if node.mapper:
-            raise WorkflowError("Cannot assign two mappings to the same input")
-        node.map(mapper=mapper, inputs=inputs)
-
-
-    def join(self, field, node=None):
-        pass
-
+#         self.graph = nx.DiGraph()
+#         self._nodes = []
+#         self.connected_var = {}
+#         if nodes:
+#             self.add_nodes(nodes)
+#         for nn in self._nodes:
+#             self.connected_var[nn] = {}
+#
+#         # dj: I have nodedir and workingdir...
+#         self.workingdir = os.path.join(os.getcwd(), workingdir)
+#
+#         if self.mapper:
+#             #dj: TODO have to implement mapper for workflow. should I create as many workflows??
+#             pass
+#
+#         # dj not sure what was the motivation, wf_klasses gives an empty list
+#         #mro = self.__class__.mro()
+#         #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
+#         #items = {}
+#         #for klass in wf_klasses:
+#         #    items.update(klass.__dict__)
+#         #for name, runnable in items.items():
+#         #    if name in ('__module__', '__doc__'):
+#         #        continue
+#
+#         #    self.add(name, value)
+#
+#
+#
+#     @property
+#     def nodes(self):
+#         return self._nodes
+#
+#     def add_nodes(self, nodes):
+#         """adding nodes without defining connections"""
+#         self.graph.add_nodes_from(nodes)
+#         for nn in nodes:
+#             self._nodes.append(nn)
+#             #self._inputs.update(nn.inputs)
+#             self.connected_var[nn] = {}
+#             nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
+#
+#
+#     def connect(self, from_node, from_socket, to_node, to_socket):
+#         if from_node:
+#             self.graph.add_edges_from([(from_node, to_node)])
+#             if not to_node in self.nodes:
+#                 self.add_nodes(to_node)
+#             self.connected_var[to_node][to_socket] = (from_node, from_socket)
+#             # from_node.sending_output.append((from_socket, to_node, to_socket))
+#             logger.debug('connecting {} and {}'.format(from_node, to_node))
+#         else:
+#             self.connect_workflow(to_node, from_socket, to_socket)
+#
+#
+#     def connect_workflow(self, node, inp_wf, inp_nd):
+#         if "{}-{}".format(self.name, inp_wf) in self.inputs:
+#             node.state_inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
+#             node.inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
+#         else:
+#             raise Exception("{} not in the workflow inputs".format(inp_wf))
+#
+#
+#     def _preparing(self):
+#         """preparing nodes which are connected: setting the final mapper and state_inputs"""
+#         self.graph_sorted = list(nx.topological_sort(self.graph))
+#         logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
+#         for nn in self.graph_sorted:
+#             nn.wfdir = self.workingdir
+#             try:
+#                 for inp, (out_node, out_var) in self.connected_var[nn].items():
+#                     nn.sufficient = False #it has some history (doesnt have to be in the loop)
+#                     nn.state_inputs.update(out_node.state_inputs)
+#                     nn.needed_outputs.append((out_node, out_var, inp))
+#                     #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
+#                     if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
+#                         nn.mapper = out_node.mapper
+#                         #nn._mapper = inp #not used
+#                     elif not out_node.mapper: # we shouldn't change anything
+#                         pass
+#                     # when the mapper from previous node is used in the current node (it has to be the same syntax)
+#                     elif nn.mapper and out_node.mapper in nn.mapper:  # state_mapper or _mapper?? TODO
+#                         #dj: if I use the syntax with state_inp name than I don't have to change the mapper...
+#                         #if type(nn._mapper) is tuple:
+#                         #    nn._mapper = tuple([inp if x == out_node.state_mapper else x for x in list(nn._mapper)])
+#                         # TODO: not sure if I'll have to implement more
+#                         pass
+#
+#                     #TODO: implement inner mapper
+#                     # TODO: if nn.mapper is a string and inp can be a letter that exists in nn.mapper
+#                     #elif nn.mapper and inp in nn.mapper:
+#                     #    raise Exception("{} can be in the mapper only together with {}, i.e. {})".format(inp, out[1],
+#                     #                                                                                    [out[1], inp]))
+#                     else:
+#                         raise Exception("worflow._preparing: should I implement something more?")
+#                 pass
+#             except(KeyError):
+#                 # tmp: we don't care about nn that are not in self.connected_var
+#                 pass
+#
+#             #nn.preparing_node() #dj: only one this needed?:
+#             # do i need it at all?
+#             nn.prepare_state_input()
+#
+#
+#     def run(self, plugin="serial"):
+#         self._preparing()
+#         self.sub = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
+#         self.sub.run_workflow()
+#         self.sub.close()
+#
+#     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
+#         # dj TODO: should I move this if checks to NewNode __init__?
+#         if is_function(runnable):
+#             if not output_nm:
+#                 output_nm = ["out"]
+#             interface = aux.Function_Interface(function=runnable, output_nm=output_nm)
+#             if not name:
+#                 raise Exception("you have to specify name for the node")
+#             if not base_dir:
+#                 base_dir = name
+#             node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
+#         elif is_interface(runnable):
+#             if not name:
+#                 raise Exception("you have to specify name for the node")
+#             if not base_dir:
+#                 base_dir = name
+#             node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
+#         elif is_node(runnable):
+#             node = runnable
+#             #dj: dont have clonning right now
+#             #node = runnable if runnable.name == name else runnable.clone(name=name)
+#         else:
+#             raise ValueError("Unknown workflow element: {!r}".format(runnable))
+#         self.add_nodes([node])
+#         # dj: i'm using name as a name of a workflow
+#         #setattr(self, name, node)
+#         #self._nodes[name] = node
+#         self._last_added = node #name
+#         #dj: so I can call map right away
+#         return self
+#
+#
+#     def map(self, mapper, node=None, inputs=None):
+#         # if node is None:
+#         #     if '.' in field:
+#         #         node, field = field.rsplit('.', 1)
+#         #     else:
+#         #         node = self._last_added
+#         #
+#         # if '.' in node:
+#         #     subwf, node = node.split('.', 1)
+#         #     self._nodes[subwf].map(field, node, values)
+#         #     return
+#
+#         if not node:
+#             node = self._last_added
+#
+#         if node.mapper:
+#             raise WorkflowError("Cannot assign two mappings to the same input")
+#         node.map(mapper=mapper, inputs=inputs)
+#
+#
+#     def join(self, field, node=None):
+#         pass
+#
 
 def is_function(obj):
     return hasattr(obj, '__call__')

From d0bdf1dafebc8eabc3a4ab9e82f6cc39ef381183 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Thu, 26 Jul 2018 21:41:20 -0400
Subject: [PATCH 22/55] Revert "[wip] rearranging classes - showing the problem
 with cf/dask when passisng node=self to SumitterNode"

This reverts commit 15b4ba8552feda36f3ae718bc609d02802ddabef.
---
 nipype/pipeline/engine/submitter.py          |   6 +-
 nipype/pipeline/engine/tests/test_newnode.py | 908 +++++++++----------
 nipype/pipeline/engine/workflows.py          | 576 ++++++------
 3 files changed, 753 insertions(+), 737 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index f11a276a5a..0f5e4b3b14 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -68,7 +68,7 @@ def run_workflow(self):
         for (i_n, node) in enumerate(self.graph):
             # submitting all the nodes who are self sufficient (self.graph is already sorted)
             if node.sufficient:
-                self.submit_work(node)
+                self.submit_work(node.nodecore)
             # if its not, its been added to a line
             else:
                 break
@@ -103,8 +103,8 @@ def _nodes_check(self):
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
             print("NODE LINE", self.node_line)
-            if to_node.checking_input_el(ind):
-                self._submit_work_el(to_node, i, ind)
+            if to_node.nodecore.checking_input_el(ind):
+                self._submit_work_el(to_node.nodecore, i, ind)
                 _to_remove.append((to_node, i, ind))
             else:
                 pass
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 535322f769..e07bccbfe6 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -160,457 +160,457 @@ def test_node_8(plugin):
 
 # tests for workflows that set mapper to node that are later added to a workflow
 
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_1(plugin):
-#     """workflow with one node with a mapper"""
-#     wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#     na.map(mapper="a", inputs={"a": [3, 5]})
-#     wf.add_nodes([na])
-#     assert wf.nodes[0].mapper == "NA-a"
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_2(plugin):
-#     """workflow with two nodes, second node without mapper"""
-#     wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#     na.map(mapper="a", inputs={"a": [3, 5]})
-#
-#     interf_addvar = Function_Interface(fun_addvar, ["out"])
-#     nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, base_dir="nb")
-#
-#     wf.add_nodes([na, nb])
-#     wf.connect(na, "out", nb, "a")
-#
-#     assert wf.nodes[0].mapper == "NA-a"
-#     wf.run(plugin=plugin)
-#
-#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected_A[0][0].keys())
-#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_A):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-#     expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
-#     key_sort = list(expected_B[0][0].keys())
-#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_B):
-#         assert wf.nodes[1].result["out"][i][0] == res[0]
-#         assert wf.nodes[1].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_2a(plugin):
-#     """workflow with two nodes, second node with a scalar mapper"""
-#     wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#     na.map(mapper="a", inputs={"a": [3, 5]})
-#
-#     interf_addvar = Function_Interface(fun_addvar, ["out"])
-#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-#     nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
-#
-#     wf.add_nodes([na, nb])
-#     wf.connect(na, "out", nb, "a")
-#
-#     assert wf.nodes[0].mapper == "NA-a"
-#     assert wf.nodes[1].mapper == ("NA-a", "NB-b")
-#     wf.run(plugin=plugin)
-#
-#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected_A[0][0].keys())
-#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_A):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-#     key_sort = list(expected_B[0][0].keys())
-#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_B):
-#         assert wf.nodes[1].result["out"][i][0] == res[0]
-#         assert wf.nodes[1].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_2b(plugin):
-#     """workflow with two nodes, second node with a vector mapper"""
-#     wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#     na.map(mapper="a", inputs={"a": [3, 5]})
-#
-#     interf_addvar = Function_Interface(fun_addvar, ["out"])
-#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-#     nb.map(mapper=["NA-a", "b"], inputs={"b": [2, 1]})
-#
-#
-#     wf.add_nodes([na, nb])
-#     wf.connect(na, "out", nb, "a")
-#
-#     assert wf.nodes[0].mapper == "NA-a"
-#     assert wf.nodes[1].mapper == ["NA-a", "NB-b"]
-#     wf.run(plugin=plugin)
-#
-#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected_A[0][0].keys())
-#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_A):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-#     expected_B = [({"NA-a": 3, "NB-b": 1}, 6), ({"NA-a": 3, "NB-b": 2}, 7),
-#                   ({"NA-a": 5, "NB-b": 1}, 8), ({"NA-a": 5, "NB-b": 2}, 9)]
-#     key_sort = list(expected_B[0][0].keys())
-#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_B):
-#         assert wf.nodes[1].result["out"][i][0] == res[0]
-#         assert wf.nodes[1].result["out"][i][1] == res[1]
-#
-#
-# # using add method to add nodes
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_3(plugin):
-#     """using add(node) method"""
-#     wf = NewWorkflow(name="wf3", workingdir="test_wf3_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#     na.map(mapper="a", inputs={"a": [3, 5]})
-#
-#     wf.add(na)
-#
-#     assert wf.nodes[0].mapper == "NA-a"
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_3a(plugin):
-#     """using add(interface) method"""
-#     wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#
-#     wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
-#
-#     assert wf.nodes[0].mapper == "NA-a"
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_3b(plugin):
-#     """using add(interface) method"""
-#     wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
-#
-#     wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
-#
-#     assert wf.nodes[0].mapper == "NA-a"
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_4(plugin):
-#     """using add(node) method"""
-#     wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#     na.map(mapper="a", inputs={"a": [3, 5]})
-#     wf.add(na)
-#
-#     interf_addvar = Function_Interface(fun_addvar, ["out"])
-#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-#     nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
-#     wf.add(nb)
-#
-#     wf.connect(na, "out", nb, "a")
-#
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-#     key_sort = list(expected_B[0][0].keys())
-#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_B):
-#         assert wf.nodes[1].result["out"][i][0] == res[0]
-#         assert wf.nodes[1].result["out"][i][1] == res[1]
-#
-#
-# # using map after add method
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_5(plugin):
-#     """using a map method for one node"""
-#     wf = NewWorkflow(name="wf5", workingdir="test_wf5_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#
-#     wf.add(na)
-#     wf.map(mapper="a", inputs={"a": [3, 5]})
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_5a(plugin):
-#     """using a map method for one node (using add and map in one chain)"""
-#     wf = NewWorkflow(name="wf5a", workingdir="test_wf5a_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#
-#     wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_6(plugin):
-#     """using a map method for two nodes (using last added node as default)"""
-#     wf = NewWorkflow(name="wf6", workingdir="test_wf6_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#
-#     interf_addvar = Function_Interface(fun_addvar, ["out"])
-#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-#
-#     wf.add(na)
-#     wf.map(mapper="a", inputs={"a": [3, 5]})
-#     wf.add(nb)
-#     wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
-#     wf.connect(na, "out", nb, "a")
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-#     key_sort = list(expected_B[0][0].keys())
-#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_B):
-#         assert wf.nodes[1].result["out"][i][0] == res[0]
-#         assert wf.nodes[1].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_6a(plugin):
-#     """using a map method for two nodes (specifying the node)"""
-#     wf = NewWorkflow(name="wf6a", workingdir="test_wf6a_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#
-#     interf_addvar = Function_Interface(fun_addvar, ["out"])
-#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-#
-#     wf.add(na)
-#     wf.add(nb)
-#     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
-#     wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
-#     wf.connect(na, "out", nb, "a")
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#     expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
-#     key_sort = list(expected_B[0][0].keys())
-#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_B):
-#         assert wf.nodes[1].result["out"][i][0] == res[0]
-#         assert wf.nodes[1].result["out"][i][1] == res[1]
-#
-#
-# # tests for a workflow that have its own input
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_7(plugin):
-#     """using inputs for workflow and connect_workflow"""
-#     wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#
-#     wf.add(na)
-#     wf.connect_workflow(na, "wf_a","a")
-#     wf.map(mapper="a")
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_7a(plugin):
-#     """using inputs for workflow and connect(None...)"""
-#     wf = NewWorkflow(name="wf7a", inputs={"wf_a": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#
-#     wf.add(na)
-#     wf.connect(None, "wf_a", na, "a")
-#     wf.map(mapper="a")
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-# @pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# def test_workflow_8(plugin):
-#     """using inputs for workflow and connect_workflow for the second node"""
-#     wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#     na.map(mapper="a", inputs={"a": [3, 5]})
-#
-#     interf_addvar = Function_Interface(fun_addvar, ["out"])
-#     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-#
-#     wf.add_nodes([na, nb])
-#     wf.connect(na, "out", nb, "a")
-#     wf.connect_workflow(nb, "b", "b")
-#     assert wf.nodes[0].mapper == "NA-a"
-#     wf.run(plugin=plugin)
-#
-#     expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected_A[0][0].keys())
-#     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_A):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
-#
-#
-#     expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
-#     key_sort = list(expected_B[0][0].keys())
-#     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected_B):
-#         assert wf.nodes[1].result["out"][i][0] == res[0]
-#         assert wf.nodes[1].result["out"][i][1] == res[1]
-#
-#
-# # tests for a workflow that have its own input and mapper
-#
-# #@pytest.mark.parametrize("plugin", Plugins)
-# @python35_only
-# @pytest.mark.xfail(reason="mapper in workflow still not impplemented")
-# def test_workflow_9(plugin="serial"):
-#     """using inputs for workflow and connect_workflow"""
-#     wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf9_{}".format(plugin))
-#     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-#     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-#
-#     wf.add(na)
-#     wf.connect_workflow(na, "wf_a","a")
-#     wf.run(plugin=plugin)
-#
-#     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
-#     key_sort = list(expected[0][0].keys())
-#     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-#     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-#     for i, res in enumerate(expected):
-#         assert wf.nodes[0].result["out"][i][0] == res[0]
-#         assert wf.nodes[0].result["out"][i][1] == res[1]
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_1(plugin):
+    """workflow with one node with a mapper"""
+    wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add_nodes([na])
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_2(plugin):
+    """workflow with two nodes, second node without mapper"""
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, base_dir="nb")
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_2a(plugin):
+    """workflow with two nodes, second node with a scalar mapper"""
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[1].mapper == ("NA-a", "NB-b")
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_2b(plugin):
+    """workflow with two nodes, second node with a vector mapper"""
+    wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb.map(mapper=["NA-a", "b"], inputs={"b": [2, 1]})
+
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[1].mapper == ["NA-a", "NB-b"]
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+    expected_B = [({"NA-a": 3, "NB-b": 1}, 6), ({"NA-a": 3, "NB-b": 2}, 7),
+                  ({"NA-a": 5, "NB-b": 1}, 8), ({"NA-a": 5, "NB-b": 2}, 9)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# using add method to add nodes
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_3(plugin):
+    """using add(node) method"""
+    wf = NewWorkflow(name="wf3", workingdir="test_wf3_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    wf.add(na)
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_3a(plugin):
+    """using add(interface) method"""
+    wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+
+    wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_3b(plugin):
+    """using add(interface) method"""
+    wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
+
+    wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_4(plugin):
+    """using add(node) method"""
+    wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(na)
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    wf.add(nb)
+
+    wf.connect(na, "out", nb, "a")
+
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# using map after add method
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_5(plugin):
+    """using a map method for one node"""
+    wf = NewWorkflow(name="wf5", workingdir="test_wf5_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_5a(plugin):
+    """using a map method for one node (using add and map in one chain)"""
+    wf = NewWorkflow(name="wf5a", workingdir="test_wf5a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_6(plugin):
+    """using a map method for two nodes (using last added node as default)"""
+    wf = NewWorkflow(name="wf6", workingdir="test_wf6_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+
+    wf.add(na)
+    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(nb)
+    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    wf.connect(na, "out", nb, "a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_6a(plugin):
+    """using a map method for two nodes (specifying the node)"""
+    wf = NewWorkflow(name="wf6a", workingdir="test_wf6a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+
+    wf.add(na)
+    wf.add(nb)
+    wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
+    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
+    wf.connect(na, "out", nb, "a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# tests for a workflow that have its own input
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_7(plugin):
+    """using inputs for workflow and connect_workflow"""
+    wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.connect_workflow(na, "wf_a","a")
+    wf.map(mapper="a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_7a(plugin):
+    """using inputs for workflow and connect(None...)"""
+    wf = NewWorkflow(name="wf7a", inputs={"wf_a": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.connect(None, "wf_a", na, "a")
+    wf.map(mapper="a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_8(plugin):
+    """using inputs for workflow and connect_workflow for the second node"""
+    wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+
+    wf.add_nodes([na, nb])
+    wf.connect(na, "out", nb, "a")
+    wf.connect_workflow(nb, "b", "b")
+    assert wf.nodes[0].mapper == "NA-a"
+    wf.run(plugin=plugin)
+
+    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected_A[0][0].keys())
+    expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_A):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+# tests for a workflow that have its own input and mapper
+
+#@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+@pytest.mark.xfail(reason="mapper in workflow still not impplemented")
+def test_workflow_9(plugin="serial"):
+    """using inputs for workflow and connect_workflow"""
+    wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf9_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na)
+    wf.connect_workflow(na, "wf_a","a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 5847f391ba..a1babb2023 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1072,111 +1072,33 @@ class MapState(object):
     pass
 
 # dj ??: should I use EngineBase?
-class NewBase(object):
-    def __init__(self, name, inputs=None, mapper=None, base_dir=None, *args, **kwargs):
+class NewNodeCore(object):
+    def __init__(self, name, interface, state, inputs=None, state_inputs=None, base_dir=None, *args, **kwargs):
         # adding interface: i'm using Function Interface from aux that has input_map that can change the name of arguments
-        self._nodedir = base_dir
+        self.nodedir = base_dir
         self.name = name
-
-        if inputs:
-            # adding name of the node to the input name
-            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                                for (key, val) in self._inputs.items())
-            self._state_inputs = self._inputs.copy()
-        else:
-            self._inputs = {}
-            self._state_inputs = {}
-        if mapper:
-            # adding name of the node to the input name within the mapper
-            mapper = aux.change_mapper(mapper, self.name)
-        self._mapper = mapper
-        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
-        self._state = state.State(mapper=self._mapper, node_name=self.name)
-
-
-    @property
-    def state(self):
-        return self._state
-
-    @property
-    def nodedir(self):
-        return self._nodedir
-
-
-    @property
-    def mapper(self):
-        return self._mapper
-
-    # @mapper.setter
-    # def mapper(self, mapper):
-    #     self._mapper = mapper
-    #     #updating state
-    #     self._state = state.State(mapper=self._mapper, node_name=self.name)
-    #
-    @property
-    def inputs(self):
-        return self._inputs
-
-    @property
-    def state_inputs(self):
-        return self._state_inputs
-
-
-
-
-
-class NewNode(NewBase):
-    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, *args, **kwargs):
-        # dj: should be changed for wf
-        super(NewNode, self).__init__(name=name, inputs=inputs, mapper=mapper,
-                                      base_dir=base_dir, *args, **kwargs)
+        #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
+        self.state = state
+        self._inputs = inputs
+        self._state_inputs = state_inputs
         self._interface = interface
         self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
                                          for (key, value) in self._interface.input_map.items())
 
         self.needed_outputs = []
         self._out_nm = self._interface._output_nm
+        self._global_done = False
+        self._result = {}
 
-        self.sufficient = True
-        # dj TODO: have to decide where to put this
-        # self._global_done = False
-        # self._result = {}
-
-
-    def map(self, mapper, inputs=None):
-        if self._mapper:
-            raise Exception("mapper is already set")
-        else:
-            self._mapper = aux.change_mapper(mapper, self.name)
 
-        if inputs:
-            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                          for (key, val) in inputs.items())
-            self._inputs.update(inputs)
-            self._state_inputs.update(inputs)
-        if mapper:
-            # updating state if we have a new mapper
-            self._state = state.State(mapper=self._mapper, node_name=self.name)
+    @property
+    def interface(self):
+        return self._interface
 
-#    def map_orig(self, field, values=None):
-#        if isinstance(field, list):
-#            for field_
-#            if values is not None:
-#                if len(values != len(field)):
-#        elif isinstance(field, tuple):
-#            pass
-#        if values is None:
-#            values = getattr(self._inputs, field)
-#            if values is None:
-#                raise MappingError('Cannot map unassigned input field')
-#        self._mappers[field] = values
 
-    # TBD
-    def join(self, field):
-        pass
+    @property
+    def inputs(self):
+        return self._inputs
 
 
     def run_interface_el(self, i, ind):
@@ -1280,31 +1202,125 @@ def _reading_results(self):
 
 
 
-     #
-    # @property
-    # def global_done(self):
-    #     return self.nodecore.global_done
-    #
-    #
-    # @property
-    # def needed_outputs(self):
-    #     return self.nodecore.needed_outputs
-    #
-    # @property
-    # def result(self):
-    #     return self.nodecore.result
-    #
-    #
+
+class NewNode(object):
+    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
+                 base_dir=None, singlenode=True, *args, **kwargs):
+        # dj: should be changed for wf
+        self.name = name
+        self._nodedir = base_dir
+        self._interface = interface
+        # dj: do I need a state_input and state_mapper??
+        # dj: reading the input from files should be added
+        if inputs:
+            # adding name of the node to the input name
+            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                                for (key, val) in self._inputs.items())
+            self._state_inputs = self._inputs.copy()
+        else:
+            self._inputs = {}
+            self._state_inputs = {}
+        if mapper:
+            # adding name of the node to the input name within the mapper
+            mapper = aux.change_mapper(mapper, self.name)
+        self._mapper = mapper
+        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
+        self._state = state.State(mapper=self._mapper, node_name=self.name)
+        if singlenode:
+            self.nodecore = NewNodeCore(name=self.name, base_dir=self.nodedir, interface=self._interface,
+            inputs=self._inputs, state_inputs=self._state_inputs, state=self._state)
+        self.sufficient = True
+
+    def map(self, mapper, inputs=None):
+        if self._mapper:
+            raise Exception("mapper is already set")
+        else:
+            self._mapper = aux.change_mapper(mapper, self.name)
+
+        if inputs:
+            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                          for (key, val) in inputs.items())
+            self._inputs.update(inputs)
+            self._state_inputs.update(inputs)
+        if mapper:
+            # updating state if we have a new mapper
+            self._state = state.State(mapper=self._mapper, node_name=self.name)
+
+#    def map_orig(self, field, values=None):
+#        if isinstance(field, list):
+#            for field_
+#            if values is not None:
+#                if len(values != len(field)):
+#        elif isinstance(field, tuple):
+#            pass
+#        if values is None:
+#            values = getattr(self._inputs, field)
+#            if values is None:
+#                raise MappingError('Cannot map unassigned input field')
+#        self._mappers[field] = values
+
+    # TBD
+    def join(self, field):
+        pass
+
+    @property
+    def state(self):
+        return self._state
+
+    @property
+    def nodedir(self):
+        return self._nodedir
+
+    @nodedir.setter
+    def nodedir(self, nodedir):
+        self._nodedir = nodedir
+        self.nodecore.nodedir = nodedir
+
+
+    @property
+    def mapper(self):
+        return self._mapper
+
+    @mapper.setter
+    def mapper(self, mapper):
+        self._mapper = mapper
+        #updating state
+        self._state = state.State(mapper=self._mapper, node_name=self.name)
+
+    @property
+    def inputs(self):
+        return self._inputs
+
+    @property
+    def state_inputs(self):
+        return self.nodecore._state_inputs
+
+    @property
+    def global_done(self):
+        return self.nodecore.global_done
+
+
+    @property
+    def needed_outputs(self):
+        return self.nodecore.needed_outputs
+
+    @property
+    def result(self):
+        return self.nodecore.result
+
+
     def prepare_state_input(self):
         self._state.prepare_state_input(state_inputs=self.state_inputs)
-    #     # updating node
-    #     self.nodecore.state = self._state
-    #     self.nodecore._inputs = self._inputs
-    #     self.nodecore._state_inputs = self._state_inputs
-    #
+        # updating node
+        self.nodecore.state = self._state
+        self.nodecore._inputs = self._inputs
+        self.nodecore._state_inputs = self._state_inputs
+
     def run(self, plugin="serial"):
         self.prepare_state_input()
-        self.sub = sub.SubmitterNode(plugin, node=self)
+        self.sub = sub.SubmitterNode(plugin, node=self.nodecore)
         self.sub.run_node()
         self.sub.close()
 
@@ -1316,172 +1332,172 @@ def __init__(self, inputs=None, mapper=None, #join_by=None,
         super(NewWorkflow, self).__init__(inputs=inputs, mapper=mapper, interface=None,
                  base_dir=base_dir, singlenode=False, *args, **kwargs)
 
-#         self.graph = nx.DiGraph()
-#         self._nodes = []
-#         self.connected_var = {}
-#         if nodes:
-#             self.add_nodes(nodes)
-#         for nn in self._nodes:
-#             self.connected_var[nn] = {}
-#
-#         # dj: I have nodedir and workingdir...
-#         self.workingdir = os.path.join(os.getcwd(), workingdir)
-#
-#         if self.mapper:
-#             #dj: TODO have to implement mapper for workflow. should I create as many workflows??
-#             pass
-#
-#         # dj not sure what was the motivation, wf_klasses gives an empty list
-#         #mro = self.__class__.mro()
-#         #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
-#         #items = {}
-#         #for klass in wf_klasses:
-#         #    items.update(klass.__dict__)
-#         #for name, runnable in items.items():
-#         #    if name in ('__module__', '__doc__'):
-#         #        continue
-#
-#         #    self.add(name, value)
-#
-#
-#
-#     @property
-#     def nodes(self):
-#         return self._nodes
-#
-#     def add_nodes(self, nodes):
-#         """adding nodes without defining connections"""
-#         self.graph.add_nodes_from(nodes)
-#         for nn in nodes:
-#             self._nodes.append(nn)
-#             #self._inputs.update(nn.inputs)
-#             self.connected_var[nn] = {}
-#             nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
-#
-#
-#     def connect(self, from_node, from_socket, to_node, to_socket):
-#         if from_node:
-#             self.graph.add_edges_from([(from_node, to_node)])
-#             if not to_node in self.nodes:
-#                 self.add_nodes(to_node)
-#             self.connected_var[to_node][to_socket] = (from_node, from_socket)
-#             # from_node.sending_output.append((from_socket, to_node, to_socket))
-#             logger.debug('connecting {} and {}'.format(from_node, to_node))
-#         else:
-#             self.connect_workflow(to_node, from_socket, to_socket)
-#
-#
-#     def connect_workflow(self, node, inp_wf, inp_nd):
-#         if "{}-{}".format(self.name, inp_wf) in self.inputs:
-#             node.state_inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
-#             node.inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
-#         else:
-#             raise Exception("{} not in the workflow inputs".format(inp_wf))
-#
-#
-#     def _preparing(self):
-#         """preparing nodes which are connected: setting the final mapper and state_inputs"""
-#         self.graph_sorted = list(nx.topological_sort(self.graph))
-#         logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
-#         for nn in self.graph_sorted:
-#             nn.wfdir = self.workingdir
-#             try:
-#                 for inp, (out_node, out_var) in self.connected_var[nn].items():
-#                     nn.sufficient = False #it has some history (doesnt have to be in the loop)
-#                     nn.state_inputs.update(out_node.state_inputs)
-#                     nn.needed_outputs.append((out_node, out_var, inp))
-#                     #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
-#                     if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
-#                         nn.mapper = out_node.mapper
-#                         #nn._mapper = inp #not used
-#                     elif not out_node.mapper: # we shouldn't change anything
-#                         pass
-#                     # when the mapper from previous node is used in the current node (it has to be the same syntax)
-#                     elif nn.mapper and out_node.mapper in nn.mapper:  # state_mapper or _mapper?? TODO
-#                         #dj: if I use the syntax with state_inp name than I don't have to change the mapper...
-#                         #if type(nn._mapper) is tuple:
-#                         #    nn._mapper = tuple([inp if x == out_node.state_mapper else x for x in list(nn._mapper)])
-#                         # TODO: not sure if I'll have to implement more
-#                         pass
-#
-#                     #TODO: implement inner mapper
-#                     # TODO: if nn.mapper is a string and inp can be a letter that exists in nn.mapper
-#                     #elif nn.mapper and inp in nn.mapper:
-#                     #    raise Exception("{} can be in the mapper only together with {}, i.e. {})".format(inp, out[1],
-#                     #                                                                                    [out[1], inp]))
-#                     else:
-#                         raise Exception("worflow._preparing: should I implement something more?")
-#                 pass
-#             except(KeyError):
-#                 # tmp: we don't care about nn that are not in self.connected_var
-#                 pass
-#
-#             #nn.preparing_node() #dj: only one this needed?:
-#             # do i need it at all?
-#             nn.prepare_state_input()
-#
-#
-#     def run(self, plugin="serial"):
-#         self._preparing()
-#         self.sub = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
-#         self.sub.run_workflow()
-#         self.sub.close()
-#
-#     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
-#         # dj TODO: should I move this if checks to NewNode __init__?
-#         if is_function(runnable):
-#             if not output_nm:
-#                 output_nm = ["out"]
-#             interface = aux.Function_Interface(function=runnable, output_nm=output_nm)
-#             if not name:
-#                 raise Exception("you have to specify name for the node")
-#             if not base_dir:
-#                 base_dir = name
-#             node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
-#         elif is_interface(runnable):
-#             if not name:
-#                 raise Exception("you have to specify name for the node")
-#             if not base_dir:
-#                 base_dir = name
-#             node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
-#         elif is_node(runnable):
-#             node = runnable
-#             #dj: dont have clonning right now
-#             #node = runnable if runnable.name == name else runnable.clone(name=name)
-#         else:
-#             raise ValueError("Unknown workflow element: {!r}".format(runnable))
-#         self.add_nodes([node])
-#         # dj: i'm using name as a name of a workflow
-#         #setattr(self, name, node)
-#         #self._nodes[name] = node
-#         self._last_added = node #name
-#         #dj: so I can call map right away
-#         return self
-#
-#
-#     def map(self, mapper, node=None, inputs=None):
-#         # if node is None:
-#         #     if '.' in field:
-#         #         node, field = field.rsplit('.', 1)
-#         #     else:
-#         #         node = self._last_added
-#         #
-#         # if '.' in node:
-#         #     subwf, node = node.split('.', 1)
-#         #     self._nodes[subwf].map(field, node, values)
-#         #     return
-#
-#         if not node:
-#             node = self._last_added
-#
-#         if node.mapper:
-#             raise WorkflowError("Cannot assign two mappings to the same input")
-#         node.map(mapper=mapper, inputs=inputs)
-#
-#
-#     def join(self, field, node=None):
-#         pass
-#
+        self.graph = nx.DiGraph()
+        self._nodes = []
+        self.connected_var = {}
+        if nodes:
+            self.add_nodes(nodes)
+        for nn in self._nodes:
+            self.connected_var[nn] = {}
+
+        # dj: I have nodedir and workingdir...
+        self.workingdir = os.path.join(os.getcwd(), workingdir)
+
+        if self.mapper:
+            #dj: TODO have to implement mapper for workflow. should I create as many workflows??
+            pass
+
+        # dj not sure what was the motivation, wf_klasses gives an empty list
+        #mro = self.__class__.mro()
+        #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
+        #items = {}
+        #for klass in wf_klasses:
+        #    items.update(klass.__dict__)
+        #for name, runnable in items.items():
+        #    if name in ('__module__', '__doc__'):
+        #        continue
+
+        #    self.add(name, value)
+
+
+
+    @property
+    def nodes(self):
+        return self._nodes
+
+    def add_nodes(self, nodes):
+        """adding nodes without defining connections"""
+        self.graph.add_nodes_from(nodes)
+        for nn in nodes:
+            self._nodes.append(nn)
+            #self._inputs.update(nn.inputs)
+            self.connected_var[nn] = {}
+            nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
+
+
+    def connect(self, from_node, from_socket, to_node, to_socket):
+        if from_node:
+            self.graph.add_edges_from([(from_node, to_node)])
+            if not to_node in self.nodes:
+                self.add_nodes(to_node)
+            self.connected_var[to_node][to_socket] = (from_node, from_socket)
+            # from_node.sending_output.append((from_socket, to_node, to_socket))
+            logger.debug('connecting {} and {}'.format(from_node, to_node))
+        else:
+            self.connect_workflow(to_node, from_socket, to_socket)
+
+
+    def connect_workflow(self, node, inp_wf, inp_nd):
+        if "{}-{}".format(self.name, inp_wf) in self.inputs:
+            node.state_inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
+            node.inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
+        else:
+            raise Exception("{} not in the workflow inputs".format(inp_wf))
+
+
+    def _preparing(self):
+        """preparing nodes which are connected: setting the final mapper and state_inputs"""
+        self.graph_sorted = list(nx.topological_sort(self.graph))
+        logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
+        for nn in self.graph_sorted:
+            nn.wfdir = self.workingdir
+            try:
+                for inp, (out_node, out_var) in self.connected_var[nn].items():
+                    nn.sufficient = False #it has some history (doesnt have to be in the loop)
+                    nn.state_inputs.update(out_node.state_inputs)
+                    nn.needed_outputs.append((out_node, out_var, inp))
+                    #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
+                    if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
+                        nn.mapper = out_node.mapper
+                        #nn._mapper = inp #not used
+                    elif not out_node.mapper: # we shouldn't change anything
+                        pass
+                    # when the mapper from previous node is used in the current node (it has to be the same syntax)
+                    elif nn.mapper and out_node.mapper in nn.mapper:  # state_mapper or _mapper?? TODO
+                        #dj: if I use the syntax with state_inp name than I don't have to change the mapper...
+                        #if type(nn._mapper) is tuple:
+                        #    nn._mapper = tuple([inp if x == out_node.state_mapper else x for x in list(nn._mapper)])
+                        # TODO: not sure if I'll have to implement more
+                        pass
+
+                    #TODO: implement inner mapper
+                    # TODO: if nn.mapper is a string and inp can be a letter that exists in nn.mapper
+                    #elif nn.mapper and inp in nn.mapper:
+                    #    raise Exception("{} can be in the mapper only together with {}, i.e. {})".format(inp, out[1],
+                    #                                                                                    [out[1], inp]))
+                    else:
+                        raise Exception("worflow._preparing: should I implement something more?")
+                pass
+            except(KeyError):
+                # tmp: we don't care about nn that are not in self.connected_var
+                pass
+
+            #nn.preparing_node() #dj: only one this needed?:
+            # do i need it at all?
+            nn.prepare_state_input()
+
+
+    def run(self, plugin="serial"):
+        self._preparing()
+        self.sub = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
+        self.sub.run_workflow()
+        self.sub.close()
+
+    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
+        # dj TODO: should I move this if checks to NewNode __init__?
+        if is_function(runnable):
+            if not output_nm:
+                output_nm = ["out"]
+            interface = aux.Function_Interface(function=runnable, output_nm=output_nm)
+            if not name:
+                raise Exception("you have to specify name for the node")
+            if not base_dir:
+                base_dir = name
+            node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
+        elif is_interface(runnable):
+            if not name:
+                raise Exception("you have to specify name for the node")
+            if not base_dir:
+                base_dir = name
+            node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
+        elif is_node(runnable):
+            node = runnable
+            #dj: dont have clonning right now
+            #node = runnable if runnable.name == name else runnable.clone(name=name)
+        else:
+            raise ValueError("Unknown workflow element: {!r}".format(runnable))
+        self.add_nodes([node])
+        # dj: i'm using name as a name of a workflow
+        #setattr(self, name, node)
+        #self._nodes[name] = node
+        self._last_added = node #name
+        #dj: so I can call map right away
+        return self
+
+
+    def map(self, mapper, node=None, inputs=None):
+        # if node is None:
+        #     if '.' in field:
+        #         node, field = field.rsplit('.', 1)
+        #     else:
+        #         node = self._last_added
+        #
+        # if '.' in node:
+        #     subwf, node = node.split('.', 1)
+        #     self._nodes[subwf].map(field, node, values)
+        #     return
+
+        if not node:
+            node = self._last_added
+
+        if node.mapper:
+            raise WorkflowError("Cannot assign two mappings to the same input")
+        node.map(mapper=mapper, inputs=inputs)
+
+
+    def join(self, field, node=None):
+        pass
+
 
 def is_function(obj):
     return hasattr(obj, '__call__')

From 522f64c0cede7313ba64a8fc6c152f6cdc972d94 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 30 Jul 2018 19:53:20 -0400
Subject: [PATCH 23/55] rearranging classes, NewNode and NewWorkflow inherit
 from NewBase; fixing problem with run method in Node when running with mp, cf
 or dask (submitter shouldnt be a class attr)

---
 nipype/pipeline/engine/submitter.py          |   7 +-
 nipype/pipeline/engine/tests/test_newnode.py |  16 +-
 nipype/pipeline/engine/workflows.py          | 269 +++++++++----------
 3 files changed, 145 insertions(+), 147 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 0f5e4b3b14..2141e20c65 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -68,7 +68,7 @@ def run_workflow(self):
         for (i_n, node) in enumerate(self.graph):
             # submitting all the nodes who are self sufficient (self.graph is already sorted)
             if node.sufficient:
-                self.submit_work(node.nodecore)
+                self.submit_work(node)
             # if its not, its been added to a line
             else:
                 break
@@ -102,9 +102,8 @@ def run_workflow(self):
     def _nodes_check(self):
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
-            print("NODE LINE", self.node_line)
-            if to_node.nodecore.checking_input_el(ind):
-                self._submit_work_el(to_node.nodecore, i, ind)
+            if to_node.checking_input_el(ind):
+                self._submit_work_el(to_node, i, ind)
                 _to_remove.append((to_node, i, ind))
             else:
                 pass
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index e07bccbfe6..a52fbf53e8 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1,7 +1,7 @@
 from .. import NewNode, NewWorkflow
 from ..auxiliary import Function_Interface
 
-import sys, time
+import sys, time, os
 import numpy as np
 import pytest, pdb
 
@@ -160,6 +160,18 @@ def test_node_8(plugin):
 
 # tests for workflows that set mapper to node that are later added to a workflow
 
+@python35_only
+def test_workflow_0(plugin="serial"):
+    """workflow with one node with a mapper"""
+    wf = NewWorkflow(name="wf0", workingdir="test_wf0_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add_nodes([na])
+    assert wf.nodes[0].mapper == "NA-a"
+    assert (wf.nodes[0].inputs['NA-a'] == np.array([3, 5])).all()
+    assert len(wf.graph.nodes) == 1
+
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_1(plugin):
@@ -169,7 +181,7 @@ def test_workflow_1(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add_nodes([na])
-    assert wf.nodes[0].mapper == "NA-a"
+
     wf.run(plugin=plugin)
 
     expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index a1babb2023..66b76528b1 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1072,23 +1072,84 @@ class MapState(object):
     pass
 
 # dj ??: should I use EngineBase?
-class NewNodeCore(object):
-    def __init__(self, name, interface, state, inputs=None, state_inputs=None, base_dir=None, *args, **kwargs):
-        # adding interface: i'm using Function Interface from aux that has input_map that can change the name of arguments
-        self.nodedir = base_dir
+class NewBase(object):
+    def __init__(self, name, mapper=None, inputs=None, *args, **kwargs):
         self.name = name
         #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
-        self.state = state
-        self._inputs = inputs
-        self._state_inputs = state_inputs
+        if inputs:
+            # adding name of the node to the input name
+            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                                for (key, val) in self._inputs.items())
+            self._state_inputs = self._inputs.copy()
+        else:
+            self._inputs = {}
+            self._state_inputs = {}
+        if mapper:
+            # adding name of the node to the input name within the mapper
+            mapper = aux.change_mapper(mapper, self.name)
+        self._mapper = mapper
+        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
+        self._state = state.State(mapper=self._mapper, node_name=self.name)
+
+        self._result = {}
+
+
+    # TBD
+    def join(self, field):
+        pass
+
+    @property
+    def state(self):
+        return self._state
+
+    @property
+    def mapper(self):
+        return self._mapper
+
+    @mapper.setter
+    def mapper(self, mapper):
+        self._mapper = mapper
+        # updating state
+        self._state = state.State(mapper=self._mapper, node_name=self.name)
+
+    @property
+    def inputs(self):
+        return self._inputs
+
+    @property
+    def state_inputs(self):
+        return self._state_inputs
+
+
+
+
+class NewNode(NewBase):
+    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
+                 base_dir=None, *args, **kwargs):
+        super(NewNode, self).__init__(name=name, mapper=mapper, inputs=inputs,
+                                      *args, **kwargs)
+
+        self._nodedir = base_dir
         self._interface = interface
         self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
                                          for (key, value) in self._interface.input_map.items())
 
-        self.needed_outputs = []
+        self._needed_outputs = []
         self._out_nm = self._interface._output_nm
         self._global_done = False
-        self._result = {}
+
+        self.sufficient = True
+
+
+    @property
+    def nodedir(self):
+        return self._nodedir
+
+
+    @nodedir.setter
+    def nodedir(self, nodedir):
+        self._nodedir = nodedir
 
 
     @property
@@ -1096,9 +1157,49 @@ def interface(self):
         return self._interface
 
 
+
+    def map(self, mapper, inputs=None):
+        if self._mapper:
+            raise Exception("mapper is already set")
+        else:
+            self._mapper = aux.change_mapper(mapper, self.name)
+
+        if inputs:
+            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                          for (key, val) in inputs.items())
+            self._inputs.update(inputs)
+            self._state_inputs.update(inputs)
+        if mapper:
+            # updating state if we have a new mapper
+            self._state = state.State(mapper=self._mapper, node_name=self.name)
+
+#    def map_orig(self, field, values=None):
+#        if isinstance(field, list):
+#            for field_
+#            if values is not None:
+#                if len(values != len(field)):
+#        elif isinstance(field, tuple):
+#            pass
+#        if values is None:
+#            values = getattr(self._inputs, field)
+#            if values is None:
+#                raise MappingError('Cannot map unassigned input field')
+#        self._mappers[field] = values
+
+
     @property
-    def inputs(self):
-        return self._inputs
+    def global_done(self):
+        return self._global_done
+
+
+    @property
+    def needed_outputs(self):
+        return self._needed_outputs
+
+    @property
+    def result(self):
+        return self.result
 
 
     def run_interface_el(self, i, ind):
@@ -1201,136 +1302,22 @@ def _reading_results(self):
                     self._result[key_out].append(({}, eval(fout.readline())))
 
 
-
-
-class NewNode(object):
-    def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, singlenode=True, *args, **kwargs):
-        # dj: should be changed for wf
-        self.name = name
-        self._nodedir = base_dir
-        self._interface = interface
-        # dj: do I need a state_input and state_mapper??
-        # dj: reading the input from files should be added
-        if inputs:
-            # adding name of the node to the input name
-            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                                for (key, val) in self._inputs.items())
-            self._state_inputs = self._inputs.copy()
-        else:
-            self._inputs = {}
-            self._state_inputs = {}
-        if mapper:
-            # adding name of the node to the input name within the mapper
-            mapper = aux.change_mapper(mapper, self.name)
-        self._mapper = mapper
-        # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
-        self._state = state.State(mapper=self._mapper, node_name=self.name)
-        if singlenode:
-            self.nodecore = NewNodeCore(name=self.name, base_dir=self.nodedir, interface=self._interface,
-            inputs=self._inputs, state_inputs=self._state_inputs, state=self._state)
-        self.sufficient = True
-
-    def map(self, mapper, inputs=None):
-        if self._mapper:
-            raise Exception("mapper is already set")
-        else:
-            self._mapper = aux.change_mapper(mapper, self.name)
-
-        if inputs:
-            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
-            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                          for (key, val) in inputs.items())
-            self._inputs.update(inputs)
-            self._state_inputs.update(inputs)
-        if mapper:
-            # updating state if we have a new mapper
-            self._state = state.State(mapper=self._mapper, node_name=self.name)
-
-#    def map_orig(self, field, values=None):
-#        if isinstance(field, list):
-#            for field_
-#            if values is not None:
-#                if len(values != len(field)):
-#        elif isinstance(field, tuple):
-#            pass
-#        if values is None:
-#            values = getattr(self._inputs, field)
-#            if values is None:
-#                raise MappingError('Cannot map unassigned input field')
-#        self._mappers[field] = values
-
-    # TBD
-    def join(self, field):
-        pass
-
-    @property
-    def state(self):
-        return self._state
-
-    @property
-    def nodedir(self):
-        return self._nodedir
-
-    @nodedir.setter
-    def nodedir(self, nodedir):
-        self._nodedir = nodedir
-        self.nodecore.nodedir = nodedir
-
-
-    @property
-    def mapper(self):
-        return self._mapper
-
-    @mapper.setter
-    def mapper(self, mapper):
-        self._mapper = mapper
-        #updating state
-        self._state = state.State(mapper=self._mapper, node_name=self.name)
-
-    @property
-    def inputs(self):
-        return self._inputs
-
-    @property
-    def state_inputs(self):
-        return self.nodecore._state_inputs
-
-    @property
-    def global_done(self):
-        return self.nodecore.global_done
-
-
-    @property
-    def needed_outputs(self):
-        return self.nodecore.needed_outputs
-
-    @property
-    def result(self):
-        return self.nodecore.result
-
-
     def prepare_state_input(self):
         self._state.prepare_state_input(state_inputs=self.state_inputs)
-        # updating node
-        self.nodecore.state = self._state
-        self.nodecore._inputs = self._inputs
-        self.nodecore._state_inputs = self._state_inputs
+
 
     def run(self, plugin="serial"):
         self.prepare_state_input()
-        self.sub = sub.SubmitterNode(plugin, node=self.nodecore)
-        self.sub.run_node()
-        self.sub.close()
+        submitter = sub.SubmitterNode(plugin, node=self)
+        submitter.run_node()
+        submitter.close()
 
 
-class NewWorkflow(NewNode):
-    def __init__(self, inputs=None, mapper=None, #join_by=None,
-                 base_dir=None, nodes=None, workingdir=None, *args, **kwargs):
-        # dj: workflow can't have interface...
-        super(NewWorkflow, self).__init__(inputs=inputs, mapper=mapper, interface=None,
-                 base_dir=base_dir, singlenode=False, *args, **kwargs)
+class NewWorkflow(NewBase):
+    def __init__(self, name, inputs=None, mapper=None, #join_by=None,
+                 nodes=None, workingdir=None, *args, **kwargs):
+        super(NewWorkflow, self).__init__(name=name, mapper=mapper, inputs=inputs,
+                                          *args, **kwargs)
 
         self.graph = nx.DiGraph()
         self._nodes = []
@@ -1340,12 +1327,12 @@ def __init__(self, inputs=None, mapper=None, #join_by=None,
         for nn in self._nodes:
             self.connected_var[nn] = {}
 
-        # dj: I have nodedir and workingdir...
+        # dj: not sure if this should be different than base_dir
         self.workingdir = os.path.join(os.getcwd(), workingdir)
 
-        if self.mapper:
-            #dj: TODO have to implement mapper for workflow. should I create as many workflows??
-            pass
+        #if self.mapper:
+        #    #dj: TODO have to implement mapper for workflow. should I create as many workflows??
+        #    pass
 
         # dj not sure what was the motivation, wf_klasses gives an empty list
         #mro = self.__class__.mro()
@@ -1439,9 +1426,9 @@ def _preparing(self):
 
     def run(self, plugin="serial"):
         self._preparing()
-        self.sub = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
-        self.sub.run_workflow()
-        self.sub.close()
+        submitter = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
+        submitter.run_workflow()
+        submitter.close()
 
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
         # dj TODO: should I move this if checks to NewNode __init__?

From 2e02f694646fa31abcfe56b919a25069d7352832 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 30 Jul 2018 23:48:49 -0400
Subject: [PATCH 24/55] allowing for kwarg arg in add method instead of connect
 method

---
 nipype/pipeline/engine/tests/test_newnode.py | 90 ++++++++++++++++++++
 nipype/pipeline/engine/workflows.py          | 14 ++-
 2 files changed, 102 insertions(+), 2 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index a52fbf53e8..0a58eeb8e1 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -409,6 +409,41 @@ def test_workflow_4(plugin):
         assert wf.nodes[1].result["out"][i][1] == res[1]
 
 
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_4a(plugin):
+    """using add(node) method with kwarg arg instead of connect """
+    wf = NewWorkflow(name="wf4a", workingdir="test_wf4a_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(na)
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    wf.add(nb, a="NA-out")
+
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+
 # using map after add method
 
 @pytest.mark.parametrize("plugin", Plugins)
@@ -522,6 +557,40 @@ def test_workflow_6a(plugin):
         assert wf.nodes[1].result["out"][i][1] == res[1]
 
 
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_6b(plugin):
+    """using a map method for two nodes (specifying the node), usng kwarg arg instead of connect"""
+    wf = NewWorkflow(name="wf6b", workingdir="test_wf6b_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+
+    wf.add(na)
+    wf.add(nb, a="NA-out")
+    wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
+    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
 # tests for a workflow that have its own input
 
 @pytest.mark.parametrize("plugin", Plugins)
@@ -568,6 +637,27 @@ def test_workflow_7a(plugin):
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
 
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_7b(plugin):
+    """using inputs for workflow and kwarg arg in add (instead of connect)"""
+    wf = NewWorkflow(name="wf7b", inputs={"wf_a": [3, 5]}, workingdir="test_wf7b_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    wf.add(na, a="wf_a")
+    wf.map(mapper="a")
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_8(plugin):
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 66b76528b1..c7e549dc52 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1326,6 +1326,7 @@ def __init__(self, name, inputs=None, mapper=None, #join_by=None,
             self.add_nodes(nodes)
         for nn in self._nodes:
             self.connected_var[nn] = {}
+        self._node_names = {}
 
         # dj: not sure if this should be different than base_dir
         self.workingdir = os.path.join(os.getcwd(), workingdir)
@@ -1360,6 +1361,7 @@ def add_nodes(self, nodes):
             #self._inputs.update(nn.inputs)
             self.connected_var[nn] = {}
             nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
+            self._node_names[nn.name] = nn
 
 
     def connect(self, from_node, from_socket, to_node, to_socket):
@@ -1430,7 +1432,7 @@ def run(self, plugin="serial"):
         submitter.run_workflow()
         submitter.close()
 
-    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None):
+    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None, **kwargs):
         # dj TODO: should I move this if checks to NewNode __init__?
         if is_function(runnable):
             if not output_nm:
@@ -1458,7 +1460,15 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
         #setattr(self, name, node)
         #self._nodes[name] = node
         self._last_added = node #name
-        #dj: so I can call map right away
+
+        # connecting inputs to other nodes outputs
+        for (inp, source) in kwargs.items():
+            try:
+                from_node, from_socket = source.split("-")
+                self.connect(self._node_names[from_node], from_socket, node, inp)
+            except(ValueError):
+                self.connect(None, source, node, inp)
+
         return self
 
 

From 730f3710807ecb761a85064a4708436e9d20db7d Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 31 Jul 2018 14:04:23 -0400
Subject: [PATCH 25/55] allowing for using mapper from previous nodes, e.g.
 [_NA, b], addding wf_mappers to NewBase/State class and _node_names to
 NewWorkflow to keep the info about mappers from all nodes

---
 nipype/pipeline/engine/auxiliary.py           |  45 +++++--
 nipype/pipeline/engine/state.py               |   4 +-
 .../pipeline/engine/tests/test_auxiliary.py   |  13 +-
 nipype/pipeline/engine/tests/test_newnode.py  | 123 +++++++++++++++++-
 nipype/pipeline/engine/workflows.py           |  40 +++---
 5 files changed, 187 insertions(+), 38 deletions(-)

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index 16e15f9906..bcb2302165 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -4,22 +4,49 @@
 logger = logging.getLogger('nipype.workflow')
 
 
+# dj: might create a new class or move to State
+
 # Function to change user provided mapper to "reverse polish notation" used in State
-def mapper2rpn(mapper):
+def mapper2rpn(mapper, wf_mappers=None):
     """ Functions that translate mapper to "reverse polish notation."""
     global output_mapper
     output_mapper = []
-    _ordering(mapper, i=0)
+    _ordering(mapper, i=0, wf_mappers=wf_mappers)
     return output_mapper
 
 
-def _ordering(el, i, current_sign=None):
+def _ordering(el, i, current_sign=None, wf_mappers=None):
     """ Used in the mapper2rpn to get a proper order of fields and signs. """
     global output_mapper
     if type(el) is tuple:
-        _iterate_list(el, ".")
+        # checking if the mapper dont contain mapper from previous nodes, i.e. has str "_NA", etc.
+        if type(el[0]) is str and el[0].startswith("_"):
+            node_nm = el[0][1:]
+            if node_nm not in wf_mappers:
+                raise Exception("can't ask for mapper from {}".format(node_nm))
+            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            el = (mapper_mod, el[1])
+        if type(el[1]) is str and el[1].startswith("_"):
+            node_nm = el[1][1:]
+            if node_nm not in wf_mappers:
+                raise Exception("can't ask for mapper from {}".format(node_nm))
+            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            el = (el[0], mapper_mod)
+        _iterate_list(el, ".", wf_mappers)
     elif type(el) is list:
-        _iterate_list(el, "*")
+        if type(el[0]) is str and el[0].startswith("_"):
+            node_nm = el[0][1:]
+            if node_nm not in wf_mappers:
+                raise Exception("can't ask for mapper from {}".format(node_nm))
+            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            el[0] = mapper_mod
+        if type(el[1]) is str and el[1].startswith("_"):
+            node_nm = el[1][1:]
+            if node_nm not in wf_mappers:
+                raise Exception("can't ask for mapper from {}".format(node_nm))
+            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            el[1] = mapper_mod
+        _iterate_list(el, "*", wf_mappers)
     elif type(el) is str:
         output_mapper.append(el)
     else:
@@ -29,10 +56,10 @@ def _ordering(el, i, current_sign=None):
         output_mapper.append(current_sign)
 
 
-def _iterate_list(element, sign):
+def _iterate_list(element, sign, wf_mappers):
     """ Used in the mapper2rpn to get recursion. """
     for i, el in enumerate(element):
-        _ordering(el, i, current_sign=sign)
+        _ordering(el, i, current_sign=sign, wf_mappers=wf_mappers)
 
 
 # functions used in State to know which element should be used for a specific axis
@@ -137,7 +164,7 @@ def converting_axis2input(state_inputs, axis_for_input, ndim):
 def change_mapper(mapper, name):
     """changing names of mapper: adding names of the node"""
     if isinstance(mapper, str):
-        if "-" in mapper:
+        if "-" in mapper or mapper.startswith("_"):
             return mapper
         else:
             return "{}-{}".format(name, mapper)
@@ -151,7 +178,7 @@ def change_mapper(mapper, name):
 def _add_name(mlist, name):
     for i, elem in enumerate(mlist):
         if isinstance(elem, str):
-            if "-" in elem:
+            if "-" in elem or elem.startswith("_"):
                 pass
             else:
                 mlist[i] = "{}-{}".format(name, mlist[i])
diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
index 7d6fc688dd..342a846b09 100644
--- a/nipype/pipeline/engine/state.py
+++ b/nipype/pipeline/engine/state.py
@@ -4,13 +4,13 @@
 from . import auxiliary as aux
 
 class State(object):
-    def __init__(self, node_name, mapper=None):
+    def __init__(self, node_name, mapper=None, wf_mappers=None):
         self._mapper = mapper
         self.node_name = node_name
         if self._mapper:
             # changing mapper (as in rpn), so I can read from left to right
             # e.g. if mapper=('d', ['e', 'r']), _mapper_rpn=['d', 'e', 'r', '*', '.']
-            self._mapper_rpn = aux.mapper2rpn(self._mapper)
+            self._mapper_rpn = aux.mapper2rpn(self._mapper, wf_mappers=wf_mappers)
             self._input_names_mapper = [i for i in self._mapper_rpn if i not in ["*", "."]]
         else:
             self._mapper_rpn = []
diff --git a/nipype/pipeline/engine/tests/test_auxiliary.py b/nipype/pipeline/engine/tests/test_auxiliary.py
index d5d3ffb536..049914d475 100644
--- a/nipype/pipeline/engine/tests/test_auxiliary.py
+++ b/nipype/pipeline/engine/tests/test_auxiliary.py
@@ -16,6 +16,17 @@ def test_mapper2rpn(mapper, rpn):
     assert aux.mapper2rpn(mapper) == rpn
 
 
+@pytest.mark.parametrize("mapper, wf_mappers, rpn",
+                         [
+                            (["a", "_NA"],        {"NA": ("b", "c")}, ["a", "NA-b", "NA-c", ".", "*"]),
+                            (["_NA", "c"],        {"NA": ("a", "b")}, ["NA-a", "NA-b", ".", "c", "*"]),
+                            (["a", ("b", "_NA")], {"NA": ["c", "d"]}, ["a", "b", "NA-c", "NA-d", "*", ".", "*"])
+                        ])
+
+def test_mapper2rpn_wf_mapper(mapper, wf_mappers, rpn):
+    assert aux.mapper2rpn(mapper, wf_mappers=wf_mappers) == rpn
+
+
 @pytest.mark.parametrize("mapper, mapper_changed",
                         [
                             ("a",               "Node-a"),
@@ -70,4 +81,4 @@ def test_mapping_axis_error():
                               [["c"], ["a", "b"], ["a", "b"]])
                          ])
 def test_converting_axis2input(inputs, axis_inputs, ndim, expected):
-    aux.converting_axis2input(inputs, axis_inputs, ndim)[0] == expected
\ No newline at end of file
+    aux.converting_axis2input(inputs, axis_inputs, ndim)[0] == expected
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 0a58eeb8e1..51cebf0869 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -694,12 +694,133 @@ def test_workflow_8(plugin):
         assert wf.nodes[1].result["out"][i][1] == res[1]
 
 
+# testing if _NA in mapper works, using interfaces in add
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_9(plugin):
+    """using add(interface) method and mapper from previous nodes"""
+    wf = NewWorkflow(name="wf9", workingdir="test_wf9_{}".format(plugin))
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    wf.add(name="NA", runnable=interf_addtwo, base_dir="na").map(mapper="a", inputs={"a": [3, 5]})
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA-out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
+
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_10(plugin):
+    """using add(interface) method and scalar mapper from previous nodes"""
+    wf = NewWorkflow(name="wf10", workingdir="test_wf10_{}".format(plugin))
+    interf_addvar1 = Function_Interface(fun_addvar, ["out"])
+    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
+    interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA-out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3, "NA-b": 0}, 3), ({"NA-a": 5, "NA-b": 10}, 15)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NA-b": 0, "NB-b": 2}, 5), ({"NA-a": 5, "NA-b": 10, "NB-b": 1}, 16)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_10a(plugin):
+    """using add(interface) method and vector mapper from previous nodes"""
+    wf = NewWorkflow(name="wf10a", workingdir="test_wf10a_{}".format(plugin))
+    interf_addvar1 = Function_Interface(fun_addvar, ["out"])
+    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
+    interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA-out").map(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3, "NA-b": 0}, 3), ({"NA-a": 3, "NA-b": 10}, 13),
+                ({"NA-a": 5, "NA-b": 0}, 5), ({"NA-a": 5, "NA-b": 10}, 15)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+    expected_B = [({"NA-a": 3, "NA-b": 0, "NB-b": 2}, 5), ({"NA-a": 3, "NA-b": 10, "NB-b": 1}, 14),
+                  ({"NA-a":5, "NA-b": 0, "NB-b": 0}, 5), ({"NA-a": 5, "NA-b": 10, "NB-b": 0}, 15)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.nodes[1].result["out"][i][0] == res[0]
+        assert wf.nodes[1].result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_11(plugin):
+    """using add(interface) method and vector mapper from previous two nodes"""
+    wf = NewWorkflow(name="wf11", workingdir="test_wf11_{}".format(plugin))
+    interf_addvar1 = Function_Interface(fun_addvar, ["out"])
+    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    wf.add(name="NB", runnable=interf_addtwo, base_dir="nb").map(mapper="a", inputs={"a": [2, 1]})
+    interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA-out", b="NB-out").map(mapper=["_NA", "_NB"])
+    wf.run(plugin=plugin)
+
+    expected = [({"NA-a": 3, "NA-b": 0}, 3), ({"NA-a": 5, "NA-b": 10}, 15)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.nodes[0].result["out"][i][0] == res[0]
+        assert wf.nodes[0].result["out"][i][1] == res[1]
+
+
+    expected_C = [({"NA-a": 3, "NA-b": 0, "NB-a": 1}, 6), ({"NA-a": 3, "NA-b": 0, "NB-a": 2}, 7),
+                  ({"NA-a": 5, "NA-b": 10, "NB-a": 1}, 18), ({"NA-a": 5, "NA-b": 10, "NB-a": 2}, 19)]
+    key_sort = list(expected_C[0][0].keys())
+    expected_C.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.nodes[2].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_C):
+        assert wf.nodes[2].result["out"][i][0] == res[0]
+        assert wf.nodes[2].result["out"][i][1] == res[1]
+
+
 # tests for a workflow that have its own input and mapper
 
 #@pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 @pytest.mark.xfail(reason="mapper in workflow still not impplemented")
-def test_workflow_9(plugin="serial"):
+def test_workflow_12(plugin="serial"):
     """using inputs for workflow and connect_workflow"""
     wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf9_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index c7e549dc52..e515ab2a1e 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1073,7 +1073,7 @@ class MapState(object):
 
 # dj ??: should I use EngineBase?
 class NewBase(object):
-    def __init__(self, name, mapper=None, inputs=None, *args, **kwargs):
+    def __init__(self, name, mapper=None, inputs=None, wf_mappers=None, *args, **kwargs):
         self.name = name
         #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
         if inputs:
@@ -1089,8 +1089,9 @@ def __init__(self, name, mapper=None, inputs=None, *args, **kwargs):
             # adding name of the node to the input name within the mapper
             mapper = aux.change_mapper(mapper, self.name)
         self._mapper = mapper
+        self._wf_mappers = wf_mappers
         # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
-        self._state = state.State(mapper=self._mapper, node_name=self.name)
+        self._state = state.State(mapper=self._mapper, node_name=self.name, wf_mappers=self._wf_mappers)
 
         self._result = {}
 
@@ -1111,7 +1112,7 @@ def mapper(self):
     def mapper(self, mapper):
         self._mapper = mapper
         # updating state
-        self._state = state.State(mapper=self._mapper, node_name=self.name)
+        self._state = state.State(mapper=self._mapper, node_name=self.name, wf_mappers=self._wf_mappers)
 
     @property
     def inputs(self):
@@ -1126,9 +1127,9 @@ def state_inputs(self):
 
 class NewNode(NewBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, *args, **kwargs):
+                 base_dir=None, wf_mappers=None, *args, **kwargs):
         super(NewNode, self).__init__(name=name, mapper=mapper, inputs=inputs,
-                                      *args, **kwargs)
+                                      wf_mappers=wf_mappers, *args, **kwargs)
 
         self._nodedir = base_dir
         self._interface = interface
@@ -1172,7 +1173,7 @@ def map(self, mapper, inputs=None):
             self._state_inputs.update(inputs)
         if mapper:
             # updating state if we have a new mapper
-            self._state = state.State(mapper=self._mapper, node_name=self.name)
+            self._state = state.State(mapper=self._mapper, node_name=self.name, wf_mappers=self._wf_mappers)
 
 #    def map_orig(self, field, values=None):
 #        if isinstance(field, list):
@@ -1209,7 +1210,6 @@ def run_interface_el(self, i, ind):
         logger.debug("Run interface el, name={}, inputs_dict={}, state_dict={}".format(
                                                             self.name, inputs_dict, state_dict))
         res = self._interface.run(inputs_dict)
-        #pdb.set_trace()
         output = self._interface.output
         logger.debug("Run interface el, output={}".format(output))
         dir_nm_el = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())])
@@ -1327,6 +1327,7 @@ def __init__(self, name, inputs=None, mapper=None, #join_by=None,
         for nn in self._nodes:
             self.connected_var[nn] = {}
         self._node_names = {}
+        self._node_mappers = {}
 
         # dj: not sure if this should be different than base_dir
         self.workingdir = os.path.join(os.getcwd(), workingdir)
@@ -1362,6 +1363,7 @@ def add_nodes(self, nodes):
             self.connected_var[nn] = {}
             nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
             self._node_names[nn.name] = nn
+            self._node_mappers[nn.name] = nn.mapper
 
 
     def connect(self, from_node, from_socket, to_node, to_socket):
@@ -1399,24 +1401,9 @@ def _preparing(self):
                     if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
                         nn.mapper = out_node.mapper
                         #nn._mapper = inp #not used
-                    elif not out_node.mapper: # we shouldn't change anything
-                        pass
-                    # when the mapper from previous node is used in the current node (it has to be the same syntax)
-                    elif nn.mapper and out_node.mapper in nn.mapper:  # state_mapper or _mapper?? TODO
-                        #dj: if I use the syntax with state_inp name than I don't have to change the mapper...
-                        #if type(nn._mapper) is tuple:
-                        #    nn._mapper = tuple([inp if x == out_node.state_mapper else x for x in list(nn._mapper)])
-                        # TODO: not sure if I'll have to implement more
+                    else:
                         pass
-
                     #TODO: implement inner mapper
-                    # TODO: if nn.mapper is a string and inp can be a letter that exists in nn.mapper
-                    #elif nn.mapper and inp in nn.mapper:
-                    #    raise Exception("{} can be in the mapper only together with {}, i.e. {})".format(inp, out[1],
-                    #                                                                                    [out[1], inp]))
-                    else:
-                        raise Exception("worflow._preparing: should I implement something more?")
-                pass
             except(KeyError):
                 # tmp: we don't care about nn that are not in self.connected_var
                 pass
@@ -1442,13 +1429,15 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
                 raise Exception("you have to specify name for the node")
             if not base_dir:
                 base_dir = name
-            node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
+            node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
+                           wf_mappers=self._node_mappers)
         elif is_interface(runnable):
             if not name:
                 raise Exception("you have to specify name for the node")
             if not base_dir:
                 base_dir = name
-            node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper)
+            node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
+                           wf_mappers=self._node_mappers)
         elif is_node(runnable):
             node = runnable
             #dj: dont have clonning right now
@@ -1490,6 +1479,7 @@ def map(self, mapper, node=None, inputs=None):
         if node.mapper:
             raise WorkflowError("Cannot assign two mappings to the same input")
         node.map(mapper=mapper, inputs=inputs)
+        self._node_mappers[node.name] = node.mapper
 
 
     def join(self, field, node=None):

From 1732b8815340a2eb0f38f46993600291d711a500 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 15 Aug 2018 17:02:20 -0400
Subject: [PATCH 26/55] removing globals  mapper2rpn

---
 nipype/pipeline/engine/auxiliary.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index bcb2302165..87e95e4633 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -9,15 +9,13 @@
 # Function to change user provided mapper to "reverse polish notation" used in State
 def mapper2rpn(mapper, wf_mappers=None):
     """ Functions that translate mapper to "reverse polish notation."""
-    global output_mapper
     output_mapper = []
-    _ordering(mapper, i=0, wf_mappers=wf_mappers)
+    _ordering(mapper, i=0, output_mapper=output_mapper, wf_mappers=wf_mappers)
     return output_mapper
 
 
-def _ordering(el, i, current_sign=None, wf_mappers=None):
+def _ordering(el, i, output_mapper, current_sign=None, wf_mappers=None):
     """ Used in the mapper2rpn to get a proper order of fields and signs. """
-    global output_mapper
     if type(el) is tuple:
         # checking if the mapper dont contain mapper from previous nodes, i.e. has str "_NA", etc.
         if type(el[0]) is str and el[0].startswith("_"):
@@ -32,7 +30,7 @@ def _ordering(el, i, current_sign=None, wf_mappers=None):
                 raise Exception("can't ask for mapper from {}".format(node_nm))
             mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
             el = (el[0], mapper_mod)
-        _iterate_list(el, ".", wf_mappers)
+        _iterate_list(el, ".", wf_mappers, output_mapper=output_mapper)
     elif type(el) is list:
         if type(el[0]) is str and el[0].startswith("_"):
             node_nm = el[0][1:]
@@ -46,7 +44,7 @@ def _ordering(el, i, current_sign=None, wf_mappers=None):
                 raise Exception("can't ask for mapper from {}".format(node_nm))
             mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
             el[1] = mapper_mod
-        _iterate_list(el, "*", wf_mappers)
+        _iterate_list(el, "*", wf_mappers, output_mapper=output_mapper)
     elif type(el) is str:
         output_mapper.append(el)
     else:
@@ -56,10 +54,10 @@ def _ordering(el, i, current_sign=None, wf_mappers=None):
         output_mapper.append(current_sign)
 
 
-def _iterate_list(element, sign, wf_mappers):
+def _iterate_list(element, sign, wf_mappers, output_mapper):
     """ Used in the mapper2rpn to get recursion. """
     for i, el in enumerate(element):
-        _ordering(el, i, current_sign=sign, wf_mappers=wf_mappers)
+        _ordering(el, i, current_sign=sign, wf_mappers=wf_mappers, output_mapper=output_mapper)
 
 
 # functions used in State to know which element should be used for a specific axis

From f1d603ee377c05bafde09b6e6adc522b843345cd Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 15 Aug 2018 19:11:13 -0400
Subject: [PATCH 27/55] adding index_generator to the state class

---
 nipype/pipeline/engine/state.py              | 10 ++++------
 nipype/pipeline/engine/submitter.py          |  6 ++----
 nipype/pipeline/engine/tests/test_newnode.py |  2 +-
 nipype/pipeline/engine/workflows.py          |  4 ++--
 4 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
index 342a846b09..38e3e687f5 100644
--- a/nipype/pipeline/engine/state.py
+++ b/nipype/pipeline/engine/state.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import itertools
 import pdb
 
 from . import auxiliary as aux
@@ -40,7 +41,8 @@ def prepare_state_input(self, state_inputs):
 
         # list of all possible indexes in each dim, will be use to iterate
         # e.g. [[0, 1], [0, 1, 2]]
-        self._all_elements = [range(i) for i in self._shape]
+        self.all_elements = [range(i) for i in self._shape]
+        self.index_generator = itertools.product(*self.all_elements)
 
 
     def __getitem__(self, key):
@@ -48,10 +50,6 @@ def __getitem__(self, key):
             key = (key,)
         return self.state_values(key)
 
-    @property
-    def all_elements(self):
-        return self._all_elements
-
     # not used?
     #@property
     #def mapper(self):
@@ -89,4 +87,4 @@ def state_values(self, ind):
 
         # in py3.7 we can skip OrderedDict
         # returning a named tuple?
-        return OrderedDict(sorted(state_dict.items(), key=lambda t: t[0]))
+        return OrderedDict(sorted(state_dict.items(), key=lambda t: t[0]))
\ No newline at end of file
diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 2141e20c65..06edc03978 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -6,8 +6,6 @@
 standard_library.install_aliases()
 
 import os, pdb, time, glob
-import itertools, collections
-import queue
 
 from .workers import MpWorker, SerialWorker, DaskWorker, ConcurrentFuturesWorker
 
@@ -31,7 +29,7 @@ def __init__(self, plugin):
 
 
     def submit_work(self, node):
-        for (i, ind) in enumerate(itertools.product(*node.state.all_elements)):
+        for (i, ind) in enumerate(node.state.index_generator):
             self._submit_work_el(node, i, ind)
 
     def _submit_work_el(self, node, i, ind):
@@ -82,7 +80,7 @@ def run_workflow(self):
         # iterating over all elements
         # (i think ordered list work well here, since it's more efficient to check within a specific order)
         for nn in list(self.graph)[i_n:]:
-            for (i, ind) in enumerate(itertools.product(*nn.state.all_elements)):
+            for (i, ind) in enumerate(nn.state.index_generator):
                 self.node_line.append((nn, i, ind))
 
         # this parts submits nodes that are waiting to be run
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 51cebf0869..1b53efc10f 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -8,7 +8,7 @@
 python35_only = pytest.mark.skipif(sys.version_info < (3, 5),
                                    reason="requires Python>3.4")
 
-Plugins = ["mp", "serial", "cf", "dask"]
+Plugins = ["serial"]#, "mp", ""cf", "dask"]
 
 def fun_addtwo(a):
     time.sleep(3)
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index e515ab2a1e..4f18ed5d4d 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -20,7 +20,7 @@
 
 import numpy as np
 import networkx as nx
-import itertools, collections
+import collections
 
 from ... import config, logging
 from ...exceptions import NodeError, WorkflowError, MappingError, JoinError
@@ -1262,7 +1262,7 @@ def global_done(self):
     # dj: version without join
     def _check_all_results(self):
         # checking if all files that should be created are present
-        for ind in itertools.product(*self.state._all_elements):
+        for ind in self.state.index_generator:
             state_dict = self.state.state_values(ind)
             dir_nm_el = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())])
             for key_out in self._out_nm:

From 25c501c72277f569e50433f4c392e4a3a7172995 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 29 Aug 2018 05:27:27 -0400
Subject: [PATCH 28/55] changing string representation of directories names and
 inputs names

---
 nipype/pipeline/engine/auxiliary.py           |   8 +-
 nipype/pipeline/engine/submitter.py           |   4 +-
 .../pipeline/engine/tests/test_auxiliary.py   |  12 +-
 nipype/pipeline/engine/tests/test_newnode.py  | 193 +++++++++---------
 nipype/pipeline/engine/workflows.py           |  22 +-
 5 files changed, 119 insertions(+), 120 deletions(-)

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index 87e95e4633..5b5275a20b 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -162,10 +162,10 @@ def converting_axis2input(state_inputs, axis_for_input, ndim):
 def change_mapper(mapper, name):
     """changing names of mapper: adding names of the node"""
     if isinstance(mapper, str):
-        if "-" in mapper or mapper.startswith("_"):
+        if "." in mapper or mapper.startswith("_"):
             return mapper
         else:
-            return "{}-{}".format(name, mapper)
+            return "{}.{}".format(name, mapper)
     elif isinstance(mapper, list):
         return _add_name(mapper, name)
     elif isinstance(mapper, tuple):
@@ -176,10 +176,10 @@ def change_mapper(mapper, name):
 def _add_name(mlist, name):
     for i, elem in enumerate(mlist):
         if isinstance(elem, str):
-            if "-" in elem or elem.startswith("_"):
+            if "." in elem or elem.startswith("_"):
                 pass
             else:
-                mlist[i] = "{}-{}".format(name, mlist[i])
+                mlist[i] = "{}.{}".format(name, mlist[i])
         elif isinstance(elem, list):
             mlist[i] = _add_name(elem, name)
         elif isinstance(elem, tuple):
diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 06edc03978..ea10085aa1 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -5,7 +5,7 @@
 from future import standard_library
 standard_library.install_aliases()
 
-import os, pdb, time, glob
+import os, pdb, time
 
 from .workers import MpWorker, SerialWorker, DaskWorker, ConcurrentFuturesWorker
 
@@ -96,7 +96,7 @@ def run_workflow(self):
             time.sleep(3)
 
 
-    # for now without callback, so checking all nodes(with ind) in some order
+    # for now without callback, so checking all nodes (with ind) in some order
     def _nodes_check(self):
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
diff --git a/nipype/pipeline/engine/tests/test_auxiliary.py b/nipype/pipeline/engine/tests/test_auxiliary.py
index 049914d475..1002eecfb5 100644
--- a/nipype/pipeline/engine/tests/test_auxiliary.py
+++ b/nipype/pipeline/engine/tests/test_auxiliary.py
@@ -18,9 +18,9 @@ def test_mapper2rpn(mapper, rpn):
 
 @pytest.mark.parametrize("mapper, wf_mappers, rpn",
                          [
-                            (["a", "_NA"],        {"NA": ("b", "c")}, ["a", "NA-b", "NA-c", ".", "*"]),
-                            (["_NA", "c"],        {"NA": ("a", "b")}, ["NA-a", "NA-b", ".", "c", "*"]),
-                            (["a", ("b", "_NA")], {"NA": ["c", "d"]}, ["a", "b", "NA-c", "NA-d", "*", ".", "*"])
+                            (["a", "_NA"],        {"NA": ("b", "c")}, ["a", "NA.b", "NA.c", ".", "*"]),
+                            (["_NA", "c"],        {"NA": ("a", "b")}, ["NA.a", "NA.b", ".", "c", "*"]),
+                            (["a", ("b", "_NA")], {"NA": ["c", "d"]}, ["a", "b", "NA.c", "NA.d", "*", ".", "*"])
                         ])
 
 def test_mapper2rpn_wf_mapper(mapper, wf_mappers, rpn):
@@ -29,9 +29,9 @@ def test_mapper2rpn_wf_mapper(mapper, wf_mappers, rpn):
 
 @pytest.mark.parametrize("mapper, mapper_changed",
                         [
-                            ("a",               "Node-a"),
-                            (["a", ("b", "c")], ["Node-a", ("Node-b", "Node-c")]),
-                            (("a", ["b", "c"]), ("Node-a", ["Node-b", "Node-c"]))
+                            ("a",               "Node.a"),
+                            (["a", ("b", "c")], ["Node.a", ("Node.b", "Node.c")]),
+                            (("a", ["b", "c"]), ("Node.a", ["Node.b", "Node.c"]))
                         ])
 def test_change_mapper(mapper, mapper_changed):
     assert aux.change_mapper(mapper, "Node") == mapper_changed
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 1b53efc10f..2c8ee9b4e1 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -34,7 +34,7 @@ def test_node_2():
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": 3})
     assert nn.mapper is None
-    assert nn.inputs == {"NA-a": 3}
+    assert nn.inputs == {"NA.a": 3}
     assert nn.state._mapper is None
 
 
@@ -42,14 +42,14 @@ def test_node_3():
     """Node with interface, inputs and mapper"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]}, mapper="a")
-    assert nn.mapper == "NA-a"
-    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
+    assert nn.mapper == "NA.a"
+    assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
     
-    assert nn.state._mapper == "NA-a"
+    assert nn.state._mapper == "NA.a"
 
     nn.prepare_state_input()
-    assert nn.state.state_values([0]) == {"NA-a": 3}
-    assert nn.state.state_values([1]) == {"NA-a": 5}
+    assert nn.state.state_values([0]) == {"NA.a": 3}
+    assert nn.state.state_values([1]) == {"NA.a": 5}
 
 
 def test_node_4():
@@ -57,13 +57,13 @@ def test_node_4():
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]})
     nn.map(mapper="a")
-    assert nn.mapper == "NA-a"
-    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
+    assert nn.mapper == "NA.a"
+    assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
 
     nn.prepare_state_input()
-    assert nn.state._mapper == "NA-a"
-    assert nn.state.state_values([0]) == {"NA-a": 3}
-    assert nn.state.state_values([1]) == {"NA-a": 5}
+    assert nn.state._mapper == "NA.a"
+    assert nn.state.state_values([0]) == {"NA.a": 3}
+    assert nn.state.state_values([1]) == {"NA.a": 5}
 
 
 def test_node_5():
@@ -71,13 +71,13 @@ def test_node_5():
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo)
     nn.map(mapper="a", inputs={"a": [3, 5]})
-    assert nn.mapper == "NA-a"
-    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
+    assert nn.mapper == "NA.a"
+    assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
 
-    assert nn.state._mapper == "NA-a"
+    assert nn.state._mapper == "NA.a"
     nn.prepare_state_input()
-    assert nn.state.state_values([0]) == {"NA-a": 3}
-    assert nn.state.state_values([1]) == {"NA-a": 5}
+    assert nn.state.state_values([0]) == {"NA.a": 3}
+    assert nn.state.state_values([1]) == {"NA.a": 5}
 
 
 @pytest.mark.parametrize("plugin", Plugins)
@@ -85,17 +85,17 @@ def test_node_5():
 def test_node_6(plugin):
     """Node with interface, inputs and the simplest mapper, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="NA", interface=interf_addtwo, base_dir="test6_{}".format(plugin))
+    nn = NewNode(name="NA", interface=interf_addtwo, base_dir="test_nd6_{}".format(plugin))
     nn.map(mapper="a", inputs={"a": [3, 5]})
 
-    assert nn.mapper == "NA-a"
-    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
+    assert nn.mapper == "NA.a"
+    assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
 
     # testing if the node runs properly
     nn.run(plugin=plugin)
 
     # checking teh results
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     # to be sure that there is the same order (not sure if node itself should keep the order)
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -110,18 +110,18 @@ def test_node_6(plugin):
 def test_node_7(plugin):
     """Node with interface, inputs and scalar mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test7_{}".format(plugin))
+    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test_nd7_{}".format(plugin))
     nn.map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [2, 1]})
 
-    assert nn.mapper == ("NA-a", "NA-b")
-    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
-    assert (nn.inputs["NA-b"] == np.array([2, 1])).all()
+    assert nn.mapper == ("NA.a", "NA.b")
+    assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
+    assert (nn.inputs["NA.b"] == np.array([2, 1])).all()
 
     # testing if the node runs properly
     nn.run(plugin=plugin)
 
     # checking teh results
-    expected = [({"NA-a": 3, "NA-b": 2}, 5), ({"NA-a": 5, "NA-b": 1}, 6)]
+    expected = [({"NA.a": 3, "NA.b": 2}, 5), ({"NA.a": 5, "NA.b": 1}, 6)]
     # to be sure that there is the same order (not sure if node itself should keep the order)
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -136,19 +136,19 @@ def test_node_7(plugin):
 def test_node_8(plugin):
     """Node with interface, inputs and vector mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test8_{}".format(plugin))
+    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test_nd8_{}".format(plugin))
     nn.map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [2, 1]})
 
-    assert nn.mapper == ["NA-a", "NA-b"]
-    assert (nn.inputs["NA-a"] == np.array([3, 5])).all()
-    assert (nn.inputs["NA-b"] == np.array([2, 1])).all()
+    assert nn.mapper == ["NA.a", "NA.b"]
+    assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
+    assert (nn.inputs["NA.b"] == np.array([2, 1])).all()
 
     # testing if the node runs properly
     nn.run(plugin=plugin)
 
     # checking teh results
-    expected = [({"NA-a": 3, "NA-b": 1}, 4), ({"NA-a": 3, "NA-b": 2}, 5),
-                ({"NA-a": 5, "NA-b": 1}, 6), ({"NA-a": 5, "NA-b": 2}, 7)]
+    expected = [({"NA.a": 3, "NA.b": 1}, 4), ({"NA.a": 3, "NA.b": 2}, 5),
+                ({"NA.a": 5, "NA.b": 1}, 6), ({"NA.a": 5, "NA.b": 2}, 7)]
     # to be sure that there is the same order (not sure if node itself should keep the order)
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -168,8 +168,8 @@ def test_workflow_0(plugin="serial"):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add_nodes([na])
-    assert wf.nodes[0].mapper == "NA-a"
-    assert (wf.nodes[0].inputs['NA-a'] == np.array([3, 5])).all()
+    assert wf.nodes[0].mapper == "NA.a"
+    assert (wf.nodes[0].inputs['NA.a'] == np.array([3, 5])).all()
     assert len(wf.graph.nodes) == 1
 
 @pytest.mark.parametrize("plugin", Plugins)
@@ -184,7 +184,7 @@ def test_workflow_1(plugin):
 
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -208,10 +208,10 @@ def test_workflow_2(plugin):
     wf.add_nodes([na, nb])
     wf.connect(na, "out", nb, "a")
 
-    assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[0].mapper == "NA.a"
     wf.run(plugin=plugin)
 
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -220,7 +220,7 @@ def test_workflow_2(plugin):
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
 
-    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+    expected_B = [({"NA.a": 3, "NB.b": 10}, 15), ({"NA.a": 5, "NB.b": 10}, 17)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -240,16 +240,16 @@ def test_workflow_2a(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
 
     wf.add_nodes([na, nb])
     wf.connect(na, "out", nb, "a")
 
-    assert wf.nodes[0].mapper == "NA-a"
-    assert wf.nodes[1].mapper == ("NA-a", "NB-b")
+    assert wf.nodes[0].mapper == "NA.a"
+    assert wf.nodes[1].mapper == ("NA.a", "NB.b")
     wf.run(plugin=plugin)
 
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -257,7 +257,7 @@ def test_workflow_2a(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -277,17 +277,17 @@ def test_workflow_2b(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-    nb.map(mapper=["NA-a", "b"], inputs={"b": [2, 1]})
+    nb.map(mapper=["NA.a", "b"], inputs={"b": [2, 1]})
 
 
     wf.add_nodes([na, nb])
     wf.connect(na, "out", nb, "a")
 
-    assert wf.nodes[0].mapper == "NA-a"
-    assert wf.nodes[1].mapper == ["NA-a", "NB-b"]
+    assert wf.nodes[0].mapper == "NA.a"
+    assert wf.nodes[1].mapper == ["NA.a", "NB.b"]
     wf.run(plugin=plugin)
 
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -296,8 +296,8 @@ def test_workflow_2b(plugin):
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
 
-    expected_B = [({"NA-a": 3, "NB-b": 1}, 6), ({"NA-a": 3, "NB-b": 2}, 7),
-                  ({"NA-a": 5, "NB-b": 1}, 8), ({"NA-a": 5, "NB-b": 2}, 9)]
+    expected_B = [({"NA.a": 3, "NB.b": 1}, 6), ({"NA.a": 3, "NB.b": 2}, 7),
+                  ({"NA.a": 5, "NB.b": 1}, 8), ({"NA.a": 5, "NB.b": 2}, 9)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -319,10 +319,10 @@ def test_workflow_3(plugin):
 
     wf.add(na)
 
-    assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[0].mapper == "NA.a"
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -340,10 +340,10 @@ def test_workflow_3a(plugin):
 
     wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
-    assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[0].mapper == "NA.a"
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -360,10 +360,10 @@ def test_workflow_3b(plugin):
 
     wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
-    assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[0].mapper == "NA.a"
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -385,14 +385,14 @@ def test_workflow_4(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.add(nb)
 
     wf.connect(na, "out", nb, "a")
 
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -400,7 +400,7 @@ def test_workflow_4(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -421,12 +421,12 @@ def test_workflow_4a(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-    nb.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
-    wf.add(nb, a="NA-out")
+    nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
+    wf.add(nb, a="NA.out")
 
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -434,7 +434,7 @@ def test_workflow_4a(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -458,7 +458,7 @@ def test_workflow_5(plugin):
     wf.map(mapper="a", inputs={"a": [3, 5]})
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -478,7 +478,7 @@ def test_workflow_5a(plugin):
     wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -501,11 +501,11 @@ def test_workflow_6(plugin):
     wf.add(na)
     wf.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(nb)
-    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]})
+    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect(na, "out", nb, "a")
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -513,7 +513,7 @@ def test_workflow_6(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -536,11 +536,11 @@ def test_workflow_6a(plugin):
     wf.add(na)
     wf.add(nb)
     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
-    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
+    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
     wf.connect(na, "out", nb, "a")
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -548,7 +548,7 @@ def test_workflow_6a(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -569,12 +569,12 @@ def test_workflow_6b(plugin):
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
 
     wf.add(na)
-    wf.add(nb, a="NA-out")
+    wf.add(nb, a="NA.out")
     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
-    wf.map(mapper=("NA-a", "b"), inputs={"b": [2, 1]}, node=nb)
+    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -582,7 +582,7 @@ def test_workflow_6b(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -606,7 +606,7 @@ def test_workflow_7(plugin):
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -628,7 +628,7 @@ def test_workflow_7a(plugin):
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -649,7 +649,7 @@ def test_workflow_7b(plugin):
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -673,10 +673,10 @@ def test_workflow_8(plugin):
     wf.add_nodes([na, nb])
     wf.connect(na, "out", nb, "a")
     wf.connect_workflow(nb, "b", "b")
-    assert wf.nodes[0].mapper == "NA-a"
+    assert wf.nodes[0].mapper == "NA.a"
     wf.run(plugin=plugin)
 
-    expected_A = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
     expected_A.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -685,7 +685,7 @@ def test_workflow_8(plugin):
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
 
-    expected_B = [({"NA-a": 3, "NB-b": 10}, 15), ({"NA-a": 5, "NB-b": 10}, 17)]
+    expected_B = [({"NA.a": 3, "NB.b": 10}, 15), ({"NA.a": 5, "NB.b": 10}, 17)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -704,11 +704,10 @@ def test_workflow_9(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     wf.add(name="NA", runnable=interf_addtwo, base_dir="na").map(mapper="a", inputs={"a": [3, 5]})
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA-out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
-
+    wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -716,7 +715,7 @@ def test_workflow_9(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NB-b": 2}, 7), ({"NA-a": 5, "NB-b": 1}, 8)]
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -733,10 +732,10 @@ def test_workflow_10(plugin):
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
     wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA-out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
+    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3, "NA-b": 0}, 3), ({"NA-a": 5, "NA-b": 10}, 15)]
+    expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 5, "NA.b": 10}, 15)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -744,7 +743,7 @@ def test_workflow_10(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NA-b": 0, "NB-b": 2}, 5), ({"NA-a": 5, "NA-b": 10, "NB-b": 1}, 16)]
+    expected_B = [({"NA.a": 3, "NA.b": 0, "NB.b": 2}, 5), ({"NA.a": 5, "NA.b": 10, "NB.b": 1}, 16)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -761,11 +760,11 @@ def test_workflow_10a(plugin):
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
     wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA-out").map(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
+    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3, "NA-b": 0}, 3), ({"NA-a": 3, "NA-b": 10}, 13),
-                ({"NA-a": 5, "NA-b": 0}, 5), ({"NA-a": 5, "NA-b": 10}, 15)]
+    expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 3, "NA.b": 10}, 13),
+                ({"NA.a": 5, "NA.b": 0}, 5), ({"NA.a": 5, "NA.b": 10}, 15)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -773,8 +772,8 @@ def test_workflow_10a(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-    expected_B = [({"NA-a": 3, "NA-b": 0, "NB-b": 2}, 5), ({"NA-a": 3, "NA-b": 10, "NB-b": 1}, 14),
-                  ({"NA-a":5, "NA-b": 0, "NB-b": 0}, 5), ({"NA-a": 5, "NA-b": 10, "NB-b": 0}, 15)]
+    expected_B = [({"NA.a": 3, "NA.b": 0, "NB.b": 2}, 5), ({"NA.a": 3, "NA.b": 10, "NB.b": 1}, 14),
+                  ({"NA.a": 5, "NA.b": 0, "NB.b": 0}, 5), ({"NA.a": 5, "NA.b": 10, "NB.b": 0}, 15)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[1].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -793,10 +792,10 @@ def test_workflow_11(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     wf.add(name="NB", runnable=interf_addtwo, base_dir="nb").map(mapper="a", inputs={"a": [2, 1]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA-out", b="NB-out").map(mapper=["_NA", "_NB"])
+    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map(mapper=["_NA", "_NB"])
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3, "NA-b": 0}, 3), ({"NA-a": 5, "NA-b": 10}, 15)]
+    expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 5, "NA.b": 10}, 15)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -805,8 +804,8 @@ def test_workflow_11(plugin):
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
 
-    expected_C = [({"NA-a": 3, "NA-b": 0, "NB-a": 1}, 6), ({"NA-a": 3, "NA-b": 0, "NB-a": 2}, 7),
-                  ({"NA-a": 5, "NA-b": 10, "NB-a": 1}, 18), ({"NA-a": 5, "NA-b": 10, "NB-a": 2}, 19)]
+    expected_C = [({"NA.a": 3, "NA.b": 0, "NB.a": 1}, 6),   ({"NA.a": 3, "NA.b": 0, "NB.a": 2}, 7),
+                  ({"NA.a": 5, "NA.b": 10, "NB.a": 1}, 18), ({"NA.a": 5, "NA.b": 10, "NB.a": 2}, 19)]
     key_sort = list(expected_C[0][0].keys())
     expected_C.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[2].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -822,7 +821,7 @@ def test_workflow_11(plugin):
 @pytest.mark.xfail(reason="mapper in workflow still not impplemented")
 def test_workflow_12(plugin="serial"):
     """using inputs for workflow and connect_workflow"""
-    wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf9_{}".format(plugin))
+    wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf12_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
@@ -830,7 +829,7 @@ def test_workflow_12(plugin="serial"):
     wf.connect_workflow(na, "wf_a","a")
     wf.run(plugin=plugin)
 
-    expected = [({"NA-a": 3}, 5), ({"NA-a": 5}, 7)]
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 4f18ed5d4d..e82feafa44 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1078,7 +1078,7 @@ def __init__(self, name, mapper=None, inputs=None, wf_mappers=None, *args, **kwa
         #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
         if inputs:
             # adding name of the node to the input name
-            self._inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            self._inputs = dict(("{}.{}".format(self.name, key), value) for (key, value) in inputs.items())
             self._inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
                                 for (key, val) in self._inputs.items())
             self._state_inputs = self._inputs.copy()
@@ -1133,7 +1133,7 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
 
         self._nodedir = base_dir
         self._interface = interface
-        self._interface.input_map = dict((key, "{}-{}".format(self.name, value))
+        self._interface.input_map = dict((key, "{}.{}".format(self.name, value))
                                          for (key, value) in self._interface.input_map.items())
 
         self._needed_outputs = []
@@ -1166,7 +1166,7 @@ def map(self, mapper, inputs=None):
             self._mapper = aux.change_mapper(mapper, self.name)
 
         if inputs:
-            inputs = dict(("{}-{}".format(self.name, key), value) for (key, value) in inputs.items())
+            inputs = dict(("{}.{}".format(self.name, key), value) for (key, value) in inputs.items())
             inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
                           for (key, val) in inputs.items())
             self._inputs.update(inputs)
@@ -1212,7 +1212,7 @@ def run_interface_el(self, i, ind):
         res = self._interface.run(inputs_dict)
         output = self._interface.output
         logger.debug("Run interface el, output={}".format(output))
-        dir_nm_el = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())])
+        dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
         # TODO when join
         #if self._joinByKey:
         #    dir_join = "join_" + "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items()) if i not in self._joinByKey])
@@ -1233,11 +1233,11 @@ def _collecting_input_el(self, ind):
         inputs_dict = {k: state_dict[k] for k in self._inputs.keys()}
         # reading extra inputs that come from previous nodes
         for (from_node, from_socket, to_socket) in self.needed_outputs:
-            dir_nm_el_from = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())
+            dir_nm_el_from = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())
                                        if i in list(from_node._state_inputs.keys())])
             file_from = os.path.join(from_node.nodedir, dir_nm_el_from, from_socket+".txt")
             with open(file_from) as f:
-                inputs_dict["{}-{}".format(self.name, to_socket)] = eval(f.readline())
+                inputs_dict["{}.{}".format(self.name, to_socket)] = eval(f.readline())
         return state_dict, inputs_dict
 
 
@@ -1290,7 +1290,7 @@ def _reading_results(self):
                 files = [name for name in glob.glob("{}/*/{}.txt".format(self.nodedir, key_out))]
                 for file in files:
                     st_el = file.split(os.sep)[-2].split("_")
-                    st_dict = collections.OrderedDict([(el.split(".")[0], eval(el.split(".")[1]))
+                    st_dict = collections.OrderedDict([(el.split(":")[0], eval(el.split(":")[1]))
                                                             for el in st_el])
                     with open(file) as fout:
                         logger.debug('Reading Results: file={}, st_dict={}'.format(file, st_dict))
@@ -1379,9 +1379,9 @@ def connect(self, from_node, from_socket, to_node, to_socket):
 
 
     def connect_workflow(self, node, inp_wf, inp_nd):
-        if "{}-{}".format(self.name, inp_wf) in self.inputs:
-            node.state_inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
-            node.inputs.update({"{}-{}".format(node.name, inp_nd): self.inputs["{}-{}".format(self.name, inp_wf)]})
+        if "{}.{}".format(self.name, inp_wf) in self.inputs:
+            node.state_inputs.update({"{}.{}".format(node.name, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
+            node.inputs.update({"{}.{}".format(node.name, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
         else:
             raise Exception("{} not in the workflow inputs".format(inp_wf))
 
@@ -1453,7 +1453,7 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
         # connecting inputs to other nodes outputs
         for (inp, source) in kwargs.items():
             try:
-                from_node, from_socket = source.split("-")
+                from_node, from_socket = source.split(".")
                 self.connect(self._node_names[from_node], from_socket, node, inp)
             except(ValueError):
                 self.connect(None, source, node, inp)

From 1e4067ceb3e38afbef8fcc49e2575fc69c35dd55 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 29 Aug 2018 08:50:12 -0400
Subject: [PATCH 29/55] adding comments

---
 nipype/pipeline/engine/tests/test_newnode.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 2c8ee9b4e1..02e3ee52e1 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -162,7 +162,7 @@ def test_node_8(plugin):
 
 @python35_only
 def test_workflow_0(plugin="serial"):
-    """workflow with one node with a mapper"""
+    """workflow (without run) with one node with a mapper"""
     wf = NewWorkflow(name="wf0", workingdir="test_wf0_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
@@ -279,7 +279,6 @@ def test_workflow_2b(plugin):
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
     nb.map(mapper=["NA.a", "b"], inputs={"b": [2, 1]})
 
-
     wf.add_nodes([na, nb])
     wf.connect(na, "out", nb, "a")
 
@@ -355,7 +354,7 @@ def test_workflow_3a(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_3b(plugin):
-    """using add(interface) method"""
+    """using add (function) method"""
     wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
 
     wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
@@ -376,7 +375,9 @@ def test_workflow_3b(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_4(plugin):
-    """using add(node) method"""
+    """ using add(node) method
+        using wf.connect to connect two nodes
+    """
     wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
@@ -412,7 +413,7 @@ def test_workflow_4(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_4a(plugin):
-    """using add(node) method with kwarg arg instead of connect """
+    """ using add(node) method with kwarg arg to connect nodes (instead of wf.connect) """
     wf = NewWorkflow(name="wf4a", workingdir="test_wf4a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
@@ -560,7 +561,7 @@ def test_workflow_6a(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_6b(plugin):
-    """using a map method for two nodes (specifying the node), usng kwarg arg instead of connect"""
+    """using a map method for two nodes (specifying the node), using kwarg arg instead of connect"""
     wf = NewWorkflow(name="wf6b", workingdir="test_wf6b_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")

From b93ef449967b00142bb8d2396aa0983491789557 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 29 Aug 2018 12:28:23 -0400
Subject: [PATCH 30/55] [wip] adding inner workflow if workflow has a mapper
 (doesnt work properly, everything is in written in one directory)

---
 nipype/pipeline/engine/state.py              |  8 +--
 nipype/pipeline/engine/tests/test_newnode.py | 23 +++----
 nipype/pipeline/engine/workflows.py          | 69 ++++++++++++++++++--
 3 files changed, 78 insertions(+), 22 deletions(-)

diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
index 38e3e687f5..c81f18e405 100644
--- a/nipype/pipeline/engine/state.py
+++ b/nipype/pipeline/engine/state.py
@@ -45,10 +45,10 @@ def prepare_state_input(self, state_inputs):
         self.index_generator = itertools.product(*self.all_elements)
 
 
-    def __getitem__(self, key):
-        if type(key) is int:
-            key = (key,)
-        return self.state_values(key)
+    def __getitem__(self, ind):
+        if type(ind) is int:
+            ind = (ind,)
+        return self.state_values(ind)
 
     # not used?
     #@property
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 02e3ee52e1..250486c731 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -819,21 +819,22 @@ def test_workflow_11(plugin):
 
 #@pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-@pytest.mark.xfail(reason="mapper in workflow still not impplemented")
+#@pytest.mark.xfail(reason="mapper in workflow still not implemented")
 def test_workflow_12(plugin="serial"):
     """using inputs for workflow and connect_workflow"""
-    wf = NewWorkflow(name="wf9", inputs={"a": [3, 5]}, mapper="a", workingdir="test_wf12_{}".format(plugin))
+    wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na)
-    wf.connect_workflow(na, "wf_a","a")
+    wf.connect_workflow(na, "wf_a", "a")
+    #pdb.set_trace()
     wf.run(plugin=plugin)
-
-    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
+    # TODO: doesn't work properly (writes everything in one dir
+    # expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
+    # key_sort = list(expected[0][0].keys())
+    # expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    # wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    # for i, res in enumerate(expected):
+    #     assert wf.nodes[0].result["out"][i][0] == res[0]
+    #     assert wf.nodes[0].result["out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index e82feafa44..5539ee23a7 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1122,6 +1122,8 @@ def inputs(self):
     def state_inputs(self):
         return self._state_inputs
 
+    def prepare_state_input(self):
+        self._state.prepare_state_input(state_inputs=self.state_inputs)
 
 
 
@@ -1302,10 +1304,6 @@ def _reading_results(self):
                     self._result[key_out].append(({}, eval(fout.readline())))
 
 
-    def prepare_state_input(self):
-        self._state.prepare_state_input(state_inputs=self.state_inputs)
-
-
     def run(self, plugin="serial"):
         self.prepare_state_input()
         submitter = sub.SubmitterNode(plugin, node=self)
@@ -1332,9 +1330,16 @@ def __init__(self, name, inputs=None, mapper=None, #join_by=None,
         # dj: not sure if this should be different than base_dir
         self.workingdir = os.path.join(os.getcwd(), workingdir)
 
-        #if self.mapper:
-        #    #dj: TODO have to implement mapper for workflow. should I create as many workflows??
-        #    pass
+        if self.mapper:
+            self.prepare_state_input()
+            self.inner_workflows = []
+            for ind in self.state.index_generator:
+                _input = {key.split(".")[1]: val for (key, val) in self.state.state_values(ind).items()}
+                inner_wf = NewWorkflow(name=self.name+str(ind), inputs=_input,
+                                       workingdir=os.path.join(self.workingdir, str(ind)))
+                self.inner_workflows.append(inner_wf)
+        else:
+            self.inner_workflows = None
 
         # dj not sure what was the motivation, wf_klasses gives an empty list
         #mro = self.__class__.mro()
@@ -1352,9 +1357,19 @@ def __init__(self, name, inputs=None, mapper=None, #join_by=None,
 
     @property
     def nodes(self):
+        if self.inner_workflows:
+            raise Exception("this workflow has inner workflows")
         return self._nodes
 
+    # TODO: use a decorator
     def add_nodes(self, nodes):
+        if self.inner_workflows:
+            for inner_wf in self.inner_workflows:
+                inner_wf._add_nodes(nodes)
+        else:
+            self._add_nodes(nodes)
+
+    def _add_nodes(self, nodes):
         """adding nodes without defining connections"""
         self.graph.add_nodes_from(nodes)
         for nn in nodes:
@@ -1367,6 +1382,14 @@ def add_nodes(self, nodes):
 
 
     def connect(self, from_node, from_socket, to_node, to_socket):
+        if self.inner_workflows:
+            for inner_wf in self.inner_workflows:
+                inner_wf._connect(from_node, from_socket, to_node, to_socket)
+        else:
+            self._connect(from_node, from_socket, to_node, to_socket)
+
+
+    def _connect(self, from_node, from_socket, to_node, to_socket):
         if from_node:
             self.graph.add_edges_from([(from_node, to_node)])
             if not to_node in self.nodes:
@@ -1379,6 +1402,14 @@ def connect(self, from_node, from_socket, to_node, to_socket):
 
 
     def connect_workflow(self, node, inp_wf, inp_nd):
+        if self.inner_workflows:
+            for inner_wf in self.inner_workflows:
+                inner_wf._connect_workflow(node, inp_wf, inp_nd)
+        else:
+            self._connect_workflow(node, inp_wf, inp_nd)
+
+
+    def _connect_workflow(self, node, inp_wf, inp_nd):
         if "{}.{}".format(self.name, inp_wf) in self.inputs:
             node.state_inputs.update({"{}.{}".format(node.name, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
             node.inputs.update({"{}.{}".format(node.name, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
@@ -1414,12 +1445,28 @@ def _preparing(self):
 
 
     def run(self, plugin="serial"):
+        if self.inner_workflows:
+            for inner_wf in self.inner_workflows:
+                inner_wf._run(plugin)
+        else:
+            self._run(plugin)
+
+    def _run(self, plugin="serial"):
         self._preparing()
         submitter = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
         submitter.run_workflow()
         submitter.close()
 
+
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None, **kwargs):
+        if self.inner_workflows:
+            for (ii, inner_wf) in enumerate(self.inner_workflows):
+                self.inner_workflows[ii] = inner_wf._add(runnable, name, base_dir, inputs, output_nm, mapper, **kwargs)
+        else:
+            return self._add(runnable, name, base_dir, inputs, output_nm, mapper, **kwargs)
+
+
+    def _add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None, **kwargs):
         # dj TODO: should I move this if checks to NewNode __init__?
         if is_function(runnable):
             if not output_nm:
@@ -1462,6 +1509,14 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
 
 
     def map(self, mapper, node=None, inputs=None):
+        if self.inner_workflows:
+            for inner_wf in self.inner_workflows:
+                inner_wf._map(mapper, node, inputs)
+        else:
+            self._map(mapper, node, inputs)
+
+
+    def _map(self, mapper, node=None, inputs=None):
         # if node is None:
         #     if '.' in field:
         #         node, field = field.rsplit('.', 1)

From c7737d0d2630cb25841fceb8230e97fbad141128 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 29 Aug 2018 12:47:44 -0400
Subject: [PATCH 31/55] [wip] adding some asserts to the test with inner
 workflows [skip ci]

---
 nipype/pipeline/engine/tests/test_newnode.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 250486c731..966031e7ce 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -828,13 +828,20 @@ def test_workflow_12(plugin="serial"):
 
     wf.add(na)
     wf.connect_workflow(na, "wf_a", "a")
-    #pdb.set_trace()
+
+    assert len(wf.inner_workflows) == 2
+    assert wf.inner_workflows[0].mapper is None
+    assert wf.inner_workflows[0].inputs == {'wf9(0,).wf_a': 3}
+    assert wf.inner_workflows[1].inputs == {'wf9(1,).wf_a': 5}
     wf.run(plugin=plugin)
-    # TODO: doesn't work properly (writes everything in one dir
-    # expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
-    # key_sort = list(expected[0][0].keys())
-    # expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    # wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.inner_workflows[0].nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    # TODO: doesn't work properly (writes everything in one dir and overwrites results
+    # so checking only for wf.inner_workflows[1]
+    wf.inner_workflows[1].nodes[0].result["out"][0][0] == expected[1][0]
+    wf.inner_workflows[1].nodes[0].result["out"][0][1] == expected[1][1]
     # for i, res in enumerate(expected):
     #     assert wf.nodes[0].result["out"][i][0] == res[0]
     #     assert wf.nodes[0].result["out"][i][1] == res[1]

From 2214404f7e677b4763d6ac2ddbec70a96965230e Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 29 Aug 2018 17:48:22 -0400
Subject: [PATCH 32/55] Adding comments

---
 nipype/pipeline/engine/tests/test_newnode.py | 50 +++++++++++++++-----
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 966031e7ce..59c9957ae5 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -34,6 +34,7 @@ def test_node_2():
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": 3})
     assert nn.mapper is None
+    # adding NA to the name of the variable
     assert nn.inputs == {"NA.a": 3}
     assert nn.state._mapper is None
 
@@ -94,12 +95,13 @@ def test_node_6(plugin):
     # testing if the node runs properly
     nn.run(plugin=plugin)
 
-    # checking teh results
+    # checking the results
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     # to be sure that there is the same order (not sure if node itself should keep the order)
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     nn.result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+
     for i, res in enumerate(expected):
         assert nn.result["out"][i][0] == res[0]
         assert nn.result["out"][i][1] == res[1]
@@ -111,6 +113,7 @@ def test_node_7(plugin):
     """Node with interface, inputs and scalar mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nn = NewNode(name="NA", interface=interf_addvar, base_dir="test_nd7_{}".format(plugin))
+    # scalar mapper
     nn.map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [2, 1]})
 
     assert nn.mapper == ("NA.a", "NA.b")
@@ -120,12 +123,13 @@ def test_node_7(plugin):
     # testing if the node runs properly
     nn.run(plugin=plugin)
 
-    # checking teh results
+    # checking the results
     expected = [({"NA.a": 3, "NA.b": 2}, 5), ({"NA.a": 5, "NA.b": 1}, 6)]
     # to be sure that there is the same order (not sure if node itself should keep the order)
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     nn.result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+
     for i, res in enumerate(expected):
         assert nn.result["out"][i][0] == res[0]
         assert nn.result["out"][i][1] == res[1]
@@ -137,6 +141,7 @@ def test_node_8(plugin):
     """Node with interface, inputs and vector mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nn = NewNode(name="NA", interface=interf_addvar, base_dir="test_nd8_{}".format(plugin))
+    # [] for outer product
     nn.map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [2, 1]})
 
     assert nn.mapper == ["NA.a", "NA.b"]
@@ -158,15 +163,17 @@ def test_node_8(plugin):
         assert nn.result["out"][i][1] == res[1]
 
 
-# tests for workflows that set mapper to node that are later added to a workflow
+# tests for workflows
 
 @python35_only
 def test_workflow_0(plugin="serial"):
     """workflow (without run) with one node with a mapper"""
     wf = NewWorkflow(name="wf0", workingdir="test_wf0_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    # defining a node with mapper and inputs first
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
+    # one of the way of adding nodes to the workflow
     wf.add_nodes([na])
     assert wf.nodes[0].mapper == "NA.a"
     assert (wf.nodes[0].inputs['NA.a'] == np.array([3, 5])).all()
@@ -202,9 +209,11 @@ def test_workflow_2(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
+    # the second node does not have explicit mapper (but keeps the mapper from the NA node)
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, base_dir="nb")
 
+    # adding 2 nodes and create a connection (as it is now)
     wf.add_nodes([na, nb])
     wf.connect(na, "out", nb, "a")
 
@@ -219,7 +228,8 @@ def test_workflow_2(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-
+    # results from NB keeps the "state input" from the first node
+    # two elements as in NA
     expected_B = [({"NA.a": 3, "NB.b": 10}, 15), ({"NA.a": 5, "NB.b": 10}, 17)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -240,6 +250,7 @@ def test_workflow_2a(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    # explicit scalar mapper between "a" from NA and b
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
 
     wf.add_nodes([na, nb])
@@ -257,6 +268,7 @@ def test_workflow_2a(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
+    # two elements (scalar mapper)
     expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
     key_sort = list(expected_B[0][0].keys())
     expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -277,6 +289,7 @@ def test_workflow_2b(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    # outer mapper
     nb.map(mapper=["NA.a", "b"], inputs={"b": [2, 1]})
 
     wf.add_nodes([na, nb])
@@ -294,7 +307,7 @@ def test_workflow_2b(plugin):
         assert wf.nodes[0].result["out"][i][0] == res[0]
         assert wf.nodes[0].result["out"][i][1] == res[1]
 
-
+    # four elements (outer product)
     expected_B = [({"NA.a": 3, "NB.b": 1}, 6), ({"NA.a": 3, "NB.b": 2}, 7),
                   ({"NA.a": 5, "NB.b": 1}, 8), ({"NA.a": 5, "NB.b": 2}, 9)]
     key_sort = list(expected_B[0][0].keys())
@@ -315,7 +328,7 @@ def test_workflow_3(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
-
+    # using add method (as in the Satra's example) with a node
     wf.add(na)
 
     assert wf.nodes[0].mapper == "NA.a"
@@ -337,6 +350,7 @@ def test_workflow_3a(plugin):
     wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
 
+    # using the add method with an interface
     wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
     assert wf.nodes[0].mapper == "NA.a"
@@ -356,7 +370,7 @@ def test_workflow_3a(plugin):
 def test_workflow_3b(plugin):
     """using add (function) method"""
     wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
-
+    # using the add method with a function
     wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
     assert wf.nodes[0].mapper == "NA.a"
@@ -386,9 +400,11 @@ def test_workflow_4(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    # explicit mapper with a variable from the previous node
+    # providing inputs with b
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.add(nb)
-
+    # connect method as it is in the current version
     wf.connect(na, "out", nb, "a")
 
     wf.run(plugin=plugin)
@@ -422,7 +438,9 @@ def test_workflow_4a(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    # explicit mapper with a variable from the previous node
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
+    # instead of "connect", using kwrg argument in the add method as in the example
     wf.add(nb, a="NA.out")
 
     wf.run(plugin=plugin)
@@ -456,6 +474,7 @@ def test_workflow_5(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na)
+    # using the map method after add (using mapper for the last added node as default)
     wf.map(mapper="a", inputs={"a": [3, 5]})
     wf.run(plugin=plugin)
 
@@ -498,7 +517,7 @@ def test_workflow_6(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-
+    # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
     wf.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(nb)
@@ -533,7 +552,7 @@ def test_workflow_6a(plugin):
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
-
+    # using the map method after add (specifying the node)
     wf.add(na)
     wf.add(nb)
     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
@@ -598,11 +617,13 @@ def test_workflow_6b(plugin):
 @python35_only
 def test_workflow_7(plugin):
     """using inputs for workflow and connect_workflow"""
+    # adding inputs to the workflow directly
     wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na)
+    # connecting the node with inputs from the workflow
     wf.connect_workflow(na, "wf_a","a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
@@ -620,11 +641,13 @@ def test_workflow_7(plugin):
 @python35_only
 def test_workflow_7a(plugin):
     """using inputs for workflow and connect(None...)"""
+    # adding inputs to the workflow directly
     wf = NewWorkflow(name="wf7a", inputs={"wf_a": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na)
+    # if connect has None as the first arg, it is the same as connect_workflow
     wf.connect(None, "wf_a", na, "a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
@@ -645,7 +668,7 @@ def test_workflow_7b(plugin):
     wf = NewWorkflow(name="wf7b", inputs={"wf_a": [3, 5]}, workingdir="test_wf7b_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
+    # using kwrg argument in the add method (instead of connect or connect_workflow
     wf.add(na, a="wf_a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
@@ -705,6 +728,7 @@ def test_workflow_9(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     wf.add(name="NA", runnable=interf_addtwo, base_dir="na").map(mapper="a", inputs={"a": [3, 5]})
     interf_addvar = Function_Interface(fun_addvar, ["out"])
+    # _NA means that I'm using mapper from the NA node, it's the same as ("NA.a", "b")
     wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
     wf.run(plugin=plugin)
 
@@ -733,6 +757,7 @@ def test_workflow_10(plugin):
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
     wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    # _NA means that I'm using mapper from the NA node, it's the same as (("NA.a", NA.b), "b")
     wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
     wf.run(plugin=plugin)
 
@@ -761,6 +786,7 @@ def test_workflow_10a(plugin):
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
     wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    # _NA means that I'm using mapper from the NA node, it's the same as (["NA.a", NA.b], "b")
     wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
     wf.run(plugin=plugin)
 
@@ -793,6 +819,7 @@ def test_workflow_11(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     wf.add(name="NB", runnable=interf_addtwo, base_dir="nb").map(mapper="a", inputs={"a": [2, 1]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    # _NA, _NB means that I'm using mappers from the NA/NB nodes, it's the same as [("NA.a", NA.b), "NB.a"]
     wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map(mapper=["_NA", "_NB"])
     wf.run(plugin=plugin)
 
@@ -816,6 +843,7 @@ def test_workflow_11(plugin):
 
 
 # tests for a workflow that have its own input and mapper
+#WIP
 
 #@pytest.mark.parametrize("plugin", Plugins)
 @python35_only

From d0e8c76b6531d36d19c547620b7e3f33567e61fd Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 4 Sep 2018 03:20:02 -0400
Subject: [PATCH 33/55] fixing inner_workflows (had to add deepcopy to node),
 changing syntax for connecting methods (using name of the node)

---
 nipype/pipeline/engine/tests/test_newnode.py | 33 ++++++--------
 nipype/pipeline/engine/workflows.py          | 48 +++++++++++++++-----
 2 files changed, 50 insertions(+), 31 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 59c9957ae5..b0dbda36d5 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -215,7 +215,7 @@ def test_workflow_2(plugin):
 
     # adding 2 nodes and create a connection (as it is now)
     wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
+    wf.connect("NA", "out", "NB", "a")
 
     assert wf.nodes[0].mapper == "NA.a"
     wf.run(plugin=plugin)
@@ -254,7 +254,7 @@ def test_workflow_2a(plugin):
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
 
     wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
+    wf.connect("NA", "out", "NB", "a")
 
     assert wf.nodes[0].mapper == "NA.a"
     assert wf.nodes[1].mapper == ("NA.a", "NB.b")
@@ -293,7 +293,7 @@ def test_workflow_2b(plugin):
     nb.map(mapper=["NA.a", "b"], inputs={"b": [2, 1]})
 
     wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
+    wf.connect("NA", "out", "NB", "a")
 
     assert wf.nodes[0].mapper == "NA.a"
     assert wf.nodes[1].mapper == ["NA.a", "NB.b"]
@@ -405,7 +405,7 @@ def test_workflow_4(plugin):
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.add(nb)
     # connect method as it is in the current version
-    wf.connect(na, "out", nb, "a")
+    wf.connect("NA", "out", "NB", "a")
 
     wf.run(plugin=plugin)
 
@@ -522,7 +522,7 @@ def test_workflow_6(plugin):
     wf.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(nb)
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
-    wf.connect(na, "out", nb, "a")
+    wf.connect("NA", "out", "NB", "a")
     wf.run(plugin=plugin)
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -557,7 +557,7 @@ def test_workflow_6a(plugin):
     wf.add(nb)
     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
-    wf.connect(na, "out", nb, "a")
+    wf.connect("NA", "out", "NB", "a")
     wf.run(plugin=plugin)
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -624,7 +624,7 @@ def test_workflow_7(plugin):
 
     wf.add(na)
     # connecting the node with inputs from the workflow
-    wf.connect_workflow(na, "wf_a","a")
+    wf.connect_workflow("NA", "wf_a","a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
@@ -648,7 +648,7 @@ def test_workflow_7a(plugin):
 
     wf.add(na)
     # if connect has None as the first arg, it is the same as connect_workflow
-    wf.connect(None, "wf_a", na, "a")
+    wf.connect(None, "wf_a", "NA", "a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
@@ -695,8 +695,8 @@ def test_workflow_8(plugin):
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
 
     wf.add_nodes([na, nb])
-    wf.connect(na, "out", nb, "a")
-    wf.connect_workflow(nb, "b", "b")
+    wf.connect("NA", "out", "NB", "a")
+    wf.connect_workflow("NB", "b", "b")
     assert wf.nodes[0].mapper == "NA.a"
     wf.run(plugin=plugin)
 
@@ -853,9 +853,8 @@ def test_workflow_12(plugin="serial"):
     wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
     wf.add(na)
-    wf.connect_workflow(na, "wf_a", "a")
+    wf.connect_workflow("NA", "wf_a", "a")
 
     assert len(wf.inner_workflows) == 2
     assert wf.inner_workflows[0].mapper is None
@@ -866,10 +865,6 @@ def test_workflow_12(plugin="serial"):
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.inner_workflows[0].nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    # TODO: doesn't work properly (writes everything in one dir and overwrites results
-    # so checking only for wf.inner_workflows[1]
-    wf.inner_workflows[1].nodes[0].result["out"][0][0] == expected[1][0]
-    wf.inner_workflows[1].nodes[0].result["out"][0][1] == expected[1][1]
-    # for i, res in enumerate(expected):
-    #     assert wf.nodes[0].result["out"][i][0] == res[0]
-    #     assert wf.nodes[0].result["out"][i][1] == res[1]
+    for i, res in enumerate(expected):
+        assert wf.inner_workflows[i].nodes[0].result["out"][0][0] == res[0]
+        assert wf.inner_workflows[i].nodes[0].result["out"][0][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 5539ee23a7..28dd67f51e 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -14,7 +14,7 @@
 import os.path as op
 import sys
 from datetime import datetime
-from copy import deepcopy
+from copy import copy, deepcopy
 import pickle
 import shutil
 
@@ -1160,6 +1160,24 @@ def interface(self):
         return self._interface
 
 
+    def __deepcopy__(self, memo): # memo is a dict of id's to copies
+        id_self = id(self)        # memoization avoids unnecesary recursion
+        _copy = memo.get(id_self)
+        if _copy is None:
+            # changing names of inputs and input_map, so it doesnt contain node.name
+            inputs_copy = dict((key[len(self.name)+1:], deepcopy(value))
+                               for (key, value) in self.inputs.items())
+            interface_copy = deepcopy(self.interface)
+            interface_copy.input_map = dict((key, val[len(self.name)+1:])
+                                            for (key, val) in interface_copy.input_map.items())
+            _copy = type(self)(
+                name=deepcopy(self.name), interface=interface_copy,
+                inputs=inputs_copy, mapper=deepcopy(self.mapper),
+                base_dir=deepcopy(self._nodedir), wf_mappers=deepcopy(self._wf_mappers))
+            memo[id_self] = _copy
+        return _copy
+
+
 
     def map(self, mapper, inputs=None):
         if self._mapper:
@@ -1389,16 +1407,18 @@ def connect(self, from_node, from_socket, to_node, to_socket):
             self._connect(from_node, from_socket, to_node, to_socket)
 
 
-    def _connect(self, from_node, from_socket, to_node, to_socket):
-        if from_node:
+    def _connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
+        if from_node_nm:
+            from_node = self._node_names[from_node_nm]
+            to_node = self._node_names[to_node_nm]
             self.graph.add_edges_from([(from_node, to_node)])
             if not to_node in self.nodes:
-                self.add_nodes(to_node)
+                self._add_nodes(to_node)
             self.connected_var[to_node][to_socket] = (from_node, from_socket)
             # from_node.sending_output.append((from_socket, to_node, to_socket))
             logger.debug('connecting {} and {}'.format(from_node, to_node))
         else:
-            self.connect_workflow(to_node, from_socket, to_socket)
+            self.connect_workflow(to_node_nm, from_socket, to_socket)
 
 
     def connect_workflow(self, node, inp_wf, inp_nd):
@@ -1409,10 +1429,11 @@ def connect_workflow(self, node, inp_wf, inp_nd):
             self._connect_workflow(node, inp_wf, inp_nd)
 
 
-    def _connect_workflow(self, node, inp_wf, inp_nd):
+    def _connect_workflow(self, node_nm, inp_wf, inp_nd):
+        node = self._node_names[node_nm]
         if "{}.{}".format(self.name, inp_wf) in self.inputs:
-            node.state_inputs.update({"{}.{}".format(node.name, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
-            node.inputs.update({"{}.{}".format(node.name, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
+            node.state_inputs.update({"{}.{}".format(node_nm, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
+            node.inputs.update({"{}.{}".format(node_nm, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
         else:
             raise Exception("{} not in the workflow inputs".format(inp_wf))
 
@@ -1461,7 +1482,9 @@ def _run(self, plugin="serial"):
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None, **kwargs):
         if self.inner_workflows:
             for (ii, inner_wf) in enumerate(self.inner_workflows):
-                self.inner_workflows[ii] = inner_wf._add(runnable, name, base_dir, inputs, output_nm, mapper, **kwargs)
+                self.inner_workflows[ii] = inner_wf._add(deepcopy(runnable), deepcopy(name), deepcopy(base_dir),
+                                                         deepcopy(inputs), deepcopy(output_nm), deepcopy(mapper),
+                                                         **deepcopy(kwargs))
         else:
             return self._add(runnable, name, base_dir, inputs, output_nm, mapper, **kwargs)
 
@@ -1500,10 +1523,11 @@ def _add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None,
         # connecting inputs to other nodes outputs
         for (inp, source) in kwargs.items():
             try:
-                from_node, from_socket = source.split(".")
-                self.connect(self._node_names[from_node], from_socket, node, inp)
+                from_node_nm, from_socket = source.split(".")
+                self.connect(from_node_nm, from_socket, node.name, inp)
+            # TODO not sure if i need it, just check if from_node_nm is not None??
             except(ValueError):
-                self.connect(None, source, node, inp)
+                self.connect(None, source, node.name, inp)
 
         return self
 

From dfe7bd5396cc051d388454a17cd8838fabee2ed5 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Tue, 4 Sep 2018 11:43:28 -0400
Subject: [PATCH 34/55] [skip ci] fixing _check_all_results (bug introduced in
 f1d603e, but worked for serial plugin anyway)

---
 nipype/pipeline/engine/tests/test_newnode.py | 2 +-
 nipype/pipeline/engine/workflows.py          | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index b0dbda36d5..0e627cca9e 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -8,7 +8,7 @@
 python35_only = pytest.mark.skipif(sys.version_info < (3, 5),
                                    reason="requires Python>3.4")
 
-Plugins = ["serial"]#, "mp", ""cf", "dask"]
+Plugins = ["serial", "mp", "cf", "dask"]
 
 def fun_addtwo(a):
     time.sleep(3)
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 28dd67f51e..108e4f884f 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -20,7 +20,7 @@
 
 import numpy as np
 import networkx as nx
-import collections
+import collections, itertools
 
 from ... import config, logging
 from ...exceptions import NodeError, WorkflowError, MappingError, JoinError
@@ -1282,9 +1282,9 @@ def global_done(self):
     # dj: version without join
     def _check_all_results(self):
         # checking if all files that should be created are present
-        for ind in self.state.index_generator:
+        for ind in itertools.product(*self.state.all_elements):
             state_dict = self.state.state_values(ind)
-            dir_nm_el = "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items())])
+            dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
             for key_out in self._out_nm:
                 if not os.path.isfile(os.path.join(self.nodedir, dir_nm_el, key_out+".txt")):
                     return False

From 409aa8487957ad8ab9452b93760cc880dc77522b Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 5 Sep 2018 05:34:25 -0400
Subject: [PATCH 35/55] [skip ci] adding example from #2539 (not running yet);
 small changes in deepcopy of node

---
 .../engine/tests/test_newnode_neuro.py        | 120 ++++++++++++++++++
 nipype/pipeline/engine/workflows.py           |  27 ++--
 2 files changed, 135 insertions(+), 12 deletions(-)
 create mode 100644 nipype/pipeline/engine/tests/test_newnode_neuro.py

diff --git a/nipype/pipeline/engine/tests/test_newnode_neuro.py b/nipype/pipeline/engine/tests/test_newnode_neuro.py
new file mode 100644
index 0000000000..48ec9cab1d
--- /dev/null
+++ b/nipype/pipeline/engine/tests/test_newnode_neuro.py
@@ -0,0 +1,120 @@
+from .. import NewNode, NewWorkflow
+from ..auxiliary import Function_Interface
+
+#dj niworkflows vs ...??
+from niworkflows.nipype.interfaces.utility import Rename
+import niworkflows.nipype.interfaces.freesurfer as fs
+
+from ...interfaces.freesurfer import PatchedConcatenateLTA as ConcatenateLTA
+
+Name = "example"
+DEFAULT_MEMORY_MIN_GB = None
+Inputs = {}
+
+# wf = Workflow(name, mem_gb_node=DEFAULT_MEMORY_MIN_GB,
+#               inputs=['source_file', 't1_preproc', 'subject_id',
+#                       'subjects_dir', 't1_2_fsnative_forward_transform',
+#                       'mem_gb', 'output_spaces', 'medial_surface_nan'],
+#               outputs='surfaces')
+#
+#dj: why do I need outputs?
+wf = NewWorkflow(name=Name, mem_gb_node=DEFAULT_MEMORY_MIN_GB, inputs=Inputs)
+
+
+# @interface
+# def select_target(subject_id, space):
+#     """ Given a source subject ID and a target space, get the target subject ID """
+#     return subject_id if space == 'fsnative' else space
+
+def select_target(subject_id, space):
+    """ Given a source subject ID and a target space, get the target subject ID """
+    return subject_id if space == 'fsnative' else space
+
+#select_target_interface = Function_Interface(select_target, ["out"])
+
+
+# wf.add('targets', select_target(subject_id=wf.inputs.subject_id))
+#   .map('space', space=[space for space in wf.inputs.output_spaces
+#                        if space.startswith('fs')])
+
+#dj: don't have option in map to connect with wf input
+
+wf.add(runnable=select_target, name="targets", subject_id="subject_id")\
+    .map(mapper="space", inputs={"space": [space for space in Inputs["output_spaces"]
+                                           if space.startswith("fs")]})
+
+
+# wf.add('rename_src', Rename(format_string='%(subject)s',
+#                             keep_ext=True,
+#                             in_file=wf.inputs.source_file))
+#   .map('subject')
+
+
+# dj TODO: how do we get subject? how do we use it in the Rename?
+wf.add(name='rename_src',
+       runnable=Rename(format_string='%(subject)s', keep_ext=True),
+                            in_file="source_file")\
+    .map('subject')
+
+
+# wf.add('resampling_xfm',
+#        fs.utils.LTAConvert(in_lta='identity.nofile',
+#                            out_lta=True,
+#                            source_file=wf.inputs.source_file,
+#                            target_file=wf.inputs.t1_preproc)
+#   .add('set_xfm_source', ConcatenateLTA(out_type='RAS2RAS',
+#                                         in_lta2=wf.inputs.t1_2_fsnative_forward_transform,
+#                                         in_lta1=wf.resampling_xfm.out_lta))
+
+
+wf.add(name='resampling_xfm',
+       runnable=fs.utils.LTAConvert(in_lta='identity.nofile', out_lta=True),
+       source_file="source_file", target_file="t1_preproc")\
+    .add(name='set_xfm_source', runnable=ConcatenateLTA(out_type='RAS2RAS'),
+         in_lta2="t1_2_fsnative_forward_transform", in_lta1="resampling_xfm.out_lta")
+
+
+# wf.add('sampler',
+#        fs.SampleToSurface(sampling_method='average', sampling_range=(0, 1, 0.2),
+#                           sampling_units='frac', interp_method='trilinear',
+#                           cortex_mask=True, override_reg_subj=True,
+#                           out_type='gii',
+#                           subjects_dir=wf.inputs.subjects_dir,
+#                           subject_id=wf.inputs.subject_id,
+#                           reg_file=wf.set_xfm_source.out_file,
+#                           target_subject=wf.targets.out,
+#                           source_file=wf.rename_src.out_file),
+#         mem_gb=mem_gb * 3)
+#        .map([('source_file', 'target_subject'), 'hemi'], hemi=['lh', 'rh'])
+
+
+wf.add(name='sampler',
+       runnable=fs.SampleToSurface(sampling_method='average', sampling_range=(0, 1, 0.2),
+                              sampling_units='frac', interp_method='trilinear',
+                              cortex_mask=True, override_reg_subj=True,
+                              out_type='gii'),
+       mem_gb=DEFAULT_MEMORY_MIN_GB * 3,
+       subjects_dir="subjects_dir", subject_id="subject_id", reg_file="set_xfm_source.out_file",
+       target_subject="targets.out", source_file="rename_src.out_file")\
+    .map(mapper=[('source_file', 'target_subject'), 'hemi'], inputs={"hemi": ['lh', 'rh']})
+
+
+# dj: no conditions
+# dj: no join for now
+
+# wf.add_cond('cond1',
+#             condition=wf.inputs.medial_surface_nan,
+#             iftrue=wf.add('medial_nans', MedialNaNs(subjects_dir=wf.inputs.subjects_dir,
+#                                              in_file=wf.sampler.out_file,
+#                                              target_subject=wf.targets.out))
+#                      .set_output('out', wf.median_nans.out),
+#             elseclause=wf.set_output('out', wf.sampler.out_file))
+#
+# wf.add('merger', niu.Merge(1, ravel_inputs=True,
+#                            in1=wf.cond1.out),
+#             run_without_submitting=True)
+#       .join('sampler.hemi')
+#
+# wf.add('update_metadata',
+#        GiftiSetAnatomicalStructure(in_file=wf.merger.out))
+# wf.outputs.surfaces = wf.update_metadata.out_file
\ No newline at end of file
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 108e4f884f..3de1868d3c 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1073,7 +1073,7 @@ class MapState(object):
 
 # dj ??: should I use EngineBase?
 class NewBase(object):
-    def __init__(self, name, mapper=None, inputs=None, wf_mappers=None, *args, **kwargs):
+    def __init__(self, name, mapper=None, inputs=None, wf_mappers=None, mem_gb_node=None, *args, **kwargs):
         self.name = name
         #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
         if inputs:
@@ -1129,9 +1129,10 @@ def prepare_state_input(self):
 
 class NewNode(NewBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, wf_mappers=None, *args, **kwargs):
+                 base_dir=None, wf_mappers=None, mem_gb_node=None, *args, **kwargs):
         super(NewNode, self).__init__(name=name, mapper=mapper, inputs=inputs,
-                                      wf_mappers=wf_mappers, *args, **kwargs)
+                                      wf_mappers=wf_mappers, mem_gb_node=mem_gb_node,
+                                      *args, **kwargs)
 
         self._nodedir = base_dir
         self._interface = interface
@@ -1331,9 +1332,9 @@ def run(self, plugin="serial"):
 
 class NewWorkflow(NewBase):
     def __init__(self, name, inputs=None, mapper=None, #join_by=None,
-                 nodes=None, workingdir=None, *args, **kwargs):
+                 nodes=None, workingdir=None, mem_gb_node=None, *args, **kwargs):
         super(NewWorkflow, self).__init__(name=name, mapper=mapper, inputs=inputs,
-                                          *args, **kwargs)
+                                          mem_gb_node=mem_gb_node, *args, **kwargs)
 
         self.graph = nx.DiGraph()
         self._nodes = []
@@ -1479,17 +1480,19 @@ def _run(self, plugin="serial"):
         submitter.close()
 
 
-    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None, **kwargs):
+    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
+            mem_gb=None, **kwargs):
         if self.inner_workflows:
             for (ii, inner_wf) in enumerate(self.inner_workflows):
-                self.inner_workflows[ii] = inner_wf._add(deepcopy(runnable), deepcopy(name), deepcopy(base_dir),
-                                                         deepcopy(inputs), deepcopy(output_nm), deepcopy(mapper),
-                                                         **deepcopy(kwargs))
+                self.inner_workflows[ii] = inner_wf._add(deepcopy(runnable), name, base_dir,
+                                                         deepcopy(inputs), output_nm, mapper,
+                                                         mem_gb, **deepcopy(kwargs))
         else:
             return self._add(runnable, name, base_dir, inputs, output_nm, mapper, **kwargs)
 
 
-    def _add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None, **kwargs):
+    def _add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
+             mem_gb=None, **kwargs):
         # dj TODO: should I move this if checks to NewNode __init__?
         if is_function(runnable):
             if not output_nm:
@@ -1500,14 +1503,14 @@ def _add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None,
             if not base_dir:
                 base_dir = name
             node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
-                           wf_mappers=self._node_mappers)
+                           wf_mappers=self._node_mappers, mem_gb_node=mem_gb)
         elif is_interface(runnable):
             if not name:
                 raise Exception("you have to specify name for the node")
             if not base_dir:
                 base_dir = name
             node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
-                           wf_mappers=self._node_mappers)
+                           wf_mappers=self._node_mappers, mem_gb_node=mem_gb)
         elif is_node(runnable):
             node = runnable
             #dj: dont have clonning right now

From 1bc5350cbe457a891000a21231b312acfde09e02 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 19 Sep 2018 10:37:11 -0400
Subject: [PATCH 36/55] small update to tests; adding DotDict to auxiliary (but
 dont use it right now, would have to change naming of the input)

---
 nipype/pipeline/engine/auxiliary.py            | 18 ++++++++++++++++++
 nipype/pipeline/engine/tests/test_newnode.py   |  4 +++-
 .../engine/tests/test_newnode_neuro.py         |  4 ++--
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index 5b5275a20b..c04afd1292 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -229,3 +229,21 @@ def run(self, input):
             raise Exception("output_nm doesnt match length of the function output")
 
         return fun_output
+
+
+# want to use to access input as dot,
+# but it doesnt work since im using "." within names (using my old syntax with - also cant work)
+# https://stackoverflow.com/questions/2352181/how-to-use-a-dot-to-access-members-of-dictionary
+class DotDict(dict):
+    """dot.notation access to dictionary attributes"""
+    def __getattr__(self, attr):
+        return self.get(attr)
+    __setattr__= dict.__setitem__
+    __delattr__= dict.__delitem__
+
+    def __getstate__(self):
+        return self
+
+    def __setstate__(self, state):
+        self.update(state)
+        self.__dict__ = self
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 0e627cca9e..d33d3edd1d 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -11,7 +11,9 @@
 Plugins = ["serial", "mp", "cf", "dask"]
 
 def fun_addtwo(a):
-    time.sleep(3)
+    time.sleep(1)
+    if a == 3:
+        time.sleep(2)
     return a + 2
 
 
diff --git a/nipype/pipeline/engine/tests/test_newnode_neuro.py b/nipype/pipeline/engine/tests/test_newnode_neuro.py
index 48ec9cab1d..8644b8d1fc 100644
--- a/nipype/pipeline/engine/tests/test_newnode_neuro.py
+++ b/nipype/pipeline/engine/tests/test_newnode_neuro.py
@@ -9,6 +9,7 @@
 
 Name = "example"
 DEFAULT_MEMORY_MIN_GB = None
+# TODO, adding fields to Inputs (subject_id)
 Inputs = {}
 
 # wf = Workflow(name, mem_gb_node=DEFAULT_MEMORY_MIN_GB,
@@ -50,10 +51,9 @@ def select_target(subject_id, space):
 #   .map('subject')
 
 
-# dj TODO: how do we get subject? how do we use it in the Rename?
 wf.add(name='rename_src',
        runnable=Rename(format_string='%(subject)s', keep_ext=True),
-                            in_file="source_file")\
+                            in_file="source_file", subject="subject_id")\
     .map('subject')
 
 

From a2d33b985ccf9d6935c8a0bbd72a156af53a3766 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 19 Sep 2018 12:53:02 -0400
Subject: [PATCH 37/55] adding tests for inner workflows

---
 nipype/pipeline/engine/tests/test_newnode.py | 35 +++++++++++++++++---
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index d33d3edd1d..1f53bfcda3 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -845,12 +845,10 @@ def test_workflow_11(plugin):
 
 
 # tests for a workflow that have its own input and mapper
-#WIP
 
-#@pytest.mark.parametrize("plugin", Plugins)
+@pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-#@pytest.mark.xfail(reason="mapper in workflow still not implemented")
-def test_workflow_12(plugin="serial"):
+def test_workflow_12(plugin):
     """using inputs for workflow and connect_workflow"""
     wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -863,6 +861,7 @@ def test_workflow_12(plugin="serial"):
     assert wf.inner_workflows[0].inputs == {'wf9(0,).wf_a': 3}
     assert wf.inner_workflows[1].inputs == {'wf9(1,).wf_a': 5}
     wf.run(plugin=plugin)
+
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
@@ -870,3 +869,31 @@ def test_workflow_12(plugin="serial"):
     for i, res in enumerate(expected):
         assert wf.inner_workflows[i].nodes[0].result["out"][0][0] == res[0]
         assert wf.inner_workflows[i].nodes[0].result["out"][0][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_12a(plugin):
+    """using inputs for workflow and connect_workflow"""
+    wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12a_{}".format(plugin))
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    na = NewNode(name="NA", interface=interf_addvar, base_dir="na", mapper="b", inputs={"b": [10, 20]})
+    wf.add(na)
+    wf.connect_workflow("NA", "wf_a", "a")
+
+    assert len(wf.inner_workflows) == 2
+    assert wf.inner_workflows[0].mapper is None
+    assert wf.inner_workflows[0].inputs == {'wf9(0,).wf_a': 3}
+    assert wf.inner_workflows[1].inputs == {'wf9(1,).wf_a': 5}
+    wf.run(plugin=plugin)
+
+    expected = [[({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)],
+                [({"NA.a": 5, "NA.b": 10}, 15), ({"NA.a": 5, "NA.b": 20}, 25)]]
+    key_sort = list(expected[0][0][0].keys())
+    for exp in expected:
+        exp.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.inner_workflows[0].nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res_l in enumerate(expected):
+        for j, res in enumerate(res_l):
+            assert wf.inner_workflows[i].nodes[0].result["out"][j][0] == res[0]
+            assert wf.inner_workflows[i].nodes[0].result["out"][j][1] == res[1]

From b87782cf67631116eeab47340c6febb09e415ec3 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 24 Sep 2018 11:01:27 -0400
Subject: [PATCH 38/55] small edits

---
 nipype/pipeline/engine/tests/test_newnode.py       | 7 ++++---
 nipype/pipeline/engine/tests/test_newnode_neuro.py | 9 ++++++++-
 nipype/pipeline/engine/workflows.py                | 5 -----
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 1f53bfcda3..6a9a96ed82 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -558,6 +558,7 @@ def test_workflow_6a(plugin):
     wf.add(na)
     wf.add(nb)
     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
+    # TODO: should we se ("a", "c") instead?? shold I forget "NA.a" value?
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
     wf.connect("NA", "out", "NB", "a")
     wf.run(plugin=plugin)
@@ -620,13 +621,13 @@ def test_workflow_6b(plugin):
 def test_workflow_7(plugin):
     """using inputs for workflow and connect_workflow"""
     # adding inputs to the workflow directly
-    wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
+    wf = NewWorkflow(name="wf7", workingdir="test_wf7_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na)
     # connecting the node with inputs from the workflow
-    wf.connect_workflow("NA", "wf_a","a")
+    wf.connect_workflow("NA", "wf_a", "a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
@@ -822,7 +823,7 @@ def test_workflow_11(plugin):
     wf.add(name="NB", runnable=interf_addtwo, base_dir="nb").map(mapper="a", inputs={"a": [2, 1]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA, _NB means that I'm using mappers from the NA/NB nodes, it's the same as [("NA.a", NA.b), "NB.a"]
-    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map(mapper=["_NA", "_NB"])
+    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map(mapper=["_NA", "_NB"]) # TODO: this should eb default?
     wf.run(plugin=plugin)
 
     expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 5, "NA.b": 10}, 15)]
diff --git a/nipype/pipeline/engine/tests/test_newnode_neuro.py b/nipype/pipeline/engine/tests/test_newnode_neuro.py
index 8644b8d1fc..5deed3807e 100644
--- a/nipype/pipeline/engine/tests/test_newnode_neuro.py
+++ b/nipype/pipeline/engine/tests/test_newnode_neuro.py
@@ -10,7 +10,13 @@
 Name = "example"
 DEFAULT_MEMORY_MIN_GB = None
 # TODO, adding fields to Inputs (subject_id)
-Inputs = {}
+Inputs = {"subject_id": [],
+          "output_spaces": [],
+          "source_file": [],
+          "t1_preproc": [],
+          "t1_2_fsnative_forward_transform": [],
+          "subjects_dir": []
+}
 
 # wf = Workflow(name, mem_gb_node=DEFAULT_MEMORY_MIN_GB,
 #               inputs=['source_file', 't1_preproc', 'subject_id',
@@ -27,6 +33,7 @@
 #     """ Given a source subject ID and a target space, get the target subject ID """
 #     return subject_id if space == 'fsnative' else space
 
+# TODO: shouldn't map with subject?
 def select_target(subject_id, space):
     """ Given a source subject ID and a target space, get the target subject ID """
     return subject_id if space == 'fsnative' else space
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 3de1868d3c..ecbad6b3e3 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1210,11 +1210,6 @@ def map(self, mapper, inputs=None):
 #        self._mappers[field] = values
 
 
-    @property
-    def global_done(self):
-        return self._global_done
-
-
     @property
     def needed_outputs(self):
         return self._needed_outputs

From fa8710b76bed8b617efa2ad41de99400f1b65561 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 24 Sep 2018 11:48:30 -0400
Subject: [PATCH 39/55] removing the concept of inner workflow from workflow
 with mapper

---
 nipype/pipeline/engine/tests/test_newnode.py |   6 +-
 nipype/pipeline/engine/workflows.py          | 102 ++++---------------
 2 files changed, 25 insertions(+), 83 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 6a9a96ed82..7496c2940e 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -621,7 +621,7 @@ def test_workflow_6b(plugin):
 def test_workflow_7(plugin):
     """using inputs for workflow and connect_workflow"""
     # adding inputs to the workflow directly
-    wf = NewWorkflow(name="wf7", workingdir="test_wf7_{}".format(plugin))
+    wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
@@ -846,7 +846,8 @@ def test_workflow_11(plugin):
 
 
 # tests for a workflow that have its own input and mapper
-
+# WIP
+@pytest.mark.xfail(reason="WIP")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_12(plugin):
@@ -872,6 +873,7 @@ def test_workflow_12(plugin):
         assert wf.inner_workflows[i].nodes[0].result["out"][0][1] == res[1]
 
 
+@pytest.mark.xfail(reason="WIP")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_12a(plugin):
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index ecbad6b3e3..ad9bdaea9a 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1160,23 +1160,23 @@ def nodedir(self, nodedir):
     def interface(self):
         return self._interface
 
-
-    def __deepcopy__(self, memo): # memo is a dict of id's to copies
-        id_self = id(self)        # memoization avoids unnecesary recursion
-        _copy = memo.get(id_self)
-        if _copy is None:
-            # changing names of inputs and input_map, so it doesnt contain node.name
-            inputs_copy = dict((key[len(self.name)+1:], deepcopy(value))
-                               for (key, value) in self.inputs.items())
-            interface_copy = deepcopy(self.interface)
-            interface_copy.input_map = dict((key, val[len(self.name)+1:])
-                                            for (key, val) in interface_copy.input_map.items())
-            _copy = type(self)(
-                name=deepcopy(self.name), interface=interface_copy,
-                inputs=inputs_copy, mapper=deepcopy(self.mapper),
-                base_dir=deepcopy(self._nodedir), wf_mappers=deepcopy(self._wf_mappers))
-            memo[id_self] = _copy
-        return _copy
+    # dj: not sure if I need it
+    # def __deepcopy__(self, memo): # memo is a dict of id's to copies
+    #     id_self = id(self)        # memoization avoids unnecesary recursion
+    #     _copy = memo.get(id_self)
+    #     if _copy is None:
+    #         # changing names of inputs and input_map, so it doesnt contain node.name
+    #         inputs_copy = dict((key[len(self.name)+1:], deepcopy(value))
+    #                            for (key, value) in self.inputs.items())
+    #         interface_copy = deepcopy(self.interface)
+    #         interface_copy.input_map = dict((key, val[len(self.name)+1:])
+    #                                         for (key, val) in interface_copy.input_map.items())
+    #         _copy = type(self)(
+    #             name=deepcopy(self.name), interface=interface_copy,
+    #             inputs=inputs_copy, mapper=deepcopy(self.mapper),
+    #             base_dir=deepcopy(self._nodedir), wf_mappers=deepcopy(self._wf_mappers))
+    #         memo[id_self] = _copy
+    #     return _copy
 
 
 
@@ -1345,15 +1345,7 @@ def __init__(self, name, inputs=None, mapper=None, #join_by=None,
         self.workingdir = os.path.join(os.getcwd(), workingdir)
 
         if self.mapper:
-            self.prepare_state_input()
-            self.inner_workflows = []
-            for ind in self.state.index_generator:
-                _input = {key.split(".")[1]: val for (key, val) in self.state.state_values(ind).items()}
-                inner_wf = NewWorkflow(name=self.name+str(ind), inputs=_input,
-                                       workingdir=os.path.join(self.workingdir, str(ind)))
-                self.inner_workflows.append(inner_wf)
-        else:
-            self.inner_workflows = None
+            pass #TODO
 
         # dj not sure what was the motivation, wf_klasses gives an empty list
         #mro = self.__class__.mro()
@@ -1368,22 +1360,12 @@ def __init__(self, name, inputs=None, mapper=None, #join_by=None,
         #    self.add(name, value)
 
 
-
     @property
     def nodes(self):
-        if self.inner_workflows:
-            raise Exception("this workflow has inner workflows")
         return self._nodes
 
-    # TODO: use a decorator
-    def add_nodes(self, nodes):
-        if self.inner_workflows:
-            for inner_wf in self.inner_workflows:
-                inner_wf._add_nodes(nodes)
-        else:
-            self._add_nodes(nodes)
 
-    def _add_nodes(self, nodes):
+    def add_nodes(self, nodes):
         """adding nodes without defining connections"""
         self.graph.add_nodes_from(nodes)
         for nn in nodes:
@@ -1395,15 +1377,7 @@ def _add_nodes(self, nodes):
             self._node_mappers[nn.name] = nn.mapper
 
 
-    def connect(self, from_node, from_socket, to_node, to_socket):
-        if self.inner_workflows:
-            for inner_wf in self.inner_workflows:
-                inner_wf._connect(from_node, from_socket, to_node, to_socket)
-        else:
-            self._connect(from_node, from_socket, to_node, to_socket)
-
-
-    def _connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
+    def connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
         if from_node_nm:
             from_node = self._node_names[from_node_nm]
             to_node = self._node_names[to_node_nm]
@@ -1417,15 +1391,7 @@ def _connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
             self.connect_workflow(to_node_nm, from_socket, to_socket)
 
 
-    def connect_workflow(self, node, inp_wf, inp_nd):
-        if self.inner_workflows:
-            for inner_wf in self.inner_workflows:
-                inner_wf._connect_workflow(node, inp_wf, inp_nd)
-        else:
-            self._connect_workflow(node, inp_wf, inp_nd)
-
-
-    def _connect_workflow(self, node_nm, inp_wf, inp_nd):
+    def connect_workflow(self, node_nm, inp_wf, inp_nd):
         node = self._node_names[node_nm]
         if "{}.{}".format(self.name, inp_wf) in self.inputs:
             node.state_inputs.update({"{}.{}".format(node_nm, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
@@ -1462,13 +1428,6 @@ def _preparing(self):
 
 
     def run(self, plugin="serial"):
-        if self.inner_workflows:
-            for inner_wf in self.inner_workflows:
-                inner_wf._run(plugin)
-        else:
-            self._run(plugin)
-
-    def _run(self, plugin="serial"):
         self._preparing()
         submitter = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
         submitter.run_workflow()
@@ -1477,17 +1436,6 @@ def _run(self, plugin="serial"):
 
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
             mem_gb=None, **kwargs):
-        if self.inner_workflows:
-            for (ii, inner_wf) in enumerate(self.inner_workflows):
-                self.inner_workflows[ii] = inner_wf._add(deepcopy(runnable), name, base_dir,
-                                                         deepcopy(inputs), output_nm, mapper,
-                                                         mem_gb, **deepcopy(kwargs))
-        else:
-            return self._add(runnable, name, base_dir, inputs, output_nm, mapper, **kwargs)
-
-
-    def _add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
-             mem_gb=None, **kwargs):
         # dj TODO: should I move this if checks to NewNode __init__?
         if is_function(runnable):
             if not output_nm:
@@ -1531,14 +1479,6 @@ def _add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None,
 
 
     def map(self, mapper, node=None, inputs=None):
-        if self.inner_workflows:
-            for inner_wf in self.inner_workflows:
-                inner_wf._map(mapper, node, inputs)
-        else:
-            self._map(mapper, node, inputs)
-
-
-    def _map(self, mapper, node=None, inputs=None):
         # if node is None:
         #     if '.' in field:
         #         node, field = field.rsplit('.', 1)

From 2cec4085d0c8361010eecf709cd0e2012adf9f10 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 24 Sep 2018 14:55:32 -0400
Subject: [PATCH 40/55] adding outputs_nm list for NewWorkflow, adding
 _reading_result for workflow

---
 nipype/pipeline/engine/tests/test_newnode.py | 71 +++++++++++++++++++-
 nipype/pipeline/engine/workflows.py          | 38 +++++++----
 2 files changed, 94 insertions(+), 15 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 7496c2940e..61d68d46ce 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -845,12 +845,79 @@ def test_workflow_11(plugin):
         assert wf.nodes[2].result["out"][i][1] == res[1]
 
 
+# checking workflow.result
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_12(plugin):
+    """testing if wf.result works (the same workflow as in test_workflow_6)"""
+    wf = NewWorkflow(name="wf12", workingdir="test_wf12_{}".format(plugin),
+                     outputs_nm=[("NA", "out", "NA_out"), ("NB", "out")])
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    # using the map methods after add (using mapper for the last added nodes as default)
+    wf.add(na)
+    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(nb)
+    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
+    wf.connect("NA", "out", "NB", "a")
+    wf.run(plugin=plugin)
+
+    # checking if workflow.results is the same as results of nodes
+    assert wf.result["NA_out"] == wf.nodes[0].result["out"]
+    assert wf.result["out"] == wf.nodes[1].result["out"]
+
+    # checking values of workflow.result
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.result["NA_out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected):
+        assert wf.result["NA_out"][i][0] == res[0]
+        assert wf.result["NA_out"][i][1] == res[1]
+
+    expected_B = [({"NA.a": 3, "NB.b": 2}, 7), ({"NA.a": 5, "NB.b": 1}, 8)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.result["out"][i][0] == res[0]
+        assert wf.result["out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_12a(plugin):
+    """testing if wf.result raises exceptione (the same workflow as in test_workflow_6)"""
+    wf = NewWorkflow(name="wf12a", workingdir="test_wf12a_{}".format(plugin),
+                     outputs_nm=[("NA", "out", "wf_out"), ("NB", "out", "wf_out")])
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    # using the map methods after add (using mapper for the last added nodes as default)
+    wf.add(na)
+    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(nb)
+    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
+    wf.connect("NA", "out", "NB", "a")
+    wf.run(plugin=plugin)
+
+    # wf_out can't be used twice in wf.result
+    with pytest.raises(Exception) as exinfo:
+        wf.result
+    assert str(exinfo.value) == "the key wf_out is already used in workflow.result"
+
 # tests for a workflow that have its own input and mapper
 # WIP
 @pytest.mark.xfail(reason="WIP")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_12(plugin):
+def test_workflow_13(plugin):
     """using inputs for workflow and connect_workflow"""
     wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -876,7 +943,7 @@ def test_workflow_12(plugin):
 @pytest.mark.xfail(reason="WIP")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_12a(plugin):
+def test_workflow_13a(plugin):
     """using inputs for workflow and connect_workflow"""
     wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12a_{}".format(plugin))
     interf_addvar = Function_Interface(fun_addvar, ["out"])
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index ad9bdaea9a..c8871662ff 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1122,6 +1122,13 @@ def inputs(self):
     def state_inputs(self):
         return self._state_inputs
 
+    @property
+    def result(self):
+        if not self._result:
+            self._reading_results()
+        return self._result
+
+
     def prepare_state_input(self):
         self._state.prepare_state_input(state_inputs=self.state_inputs)
 
@@ -1214,10 +1221,6 @@ def map(self, mapper, inputs=None):
     def needed_outputs(self):
         return self._needed_outputs
 
-    @property
-    def result(self):
-        return self.result
-
 
     def run_interface_el(self, i, ind):
         """ running interface one element generated from node_state."""
@@ -1288,14 +1291,6 @@ def _check_all_results(self):
         return True
 
 
-    # reading results (without join for now)
-    @property
-    def result(self):
-        if not self._result:
-            self._reading_results()
-        return self._result
-
-
     def _reading_results(self):
         """
         reading results from file,
@@ -1326,7 +1321,7 @@ def run(self, plugin="serial"):
 
 
 class NewWorkflow(NewBase):
-    def __init__(self, name, inputs=None, mapper=None, #join_by=None,
+    def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=None,
                  nodes=None, workingdir=None, mem_gb_node=None, *args, **kwargs):
         super(NewWorkflow, self).__init__(name=name, mapper=mapper, inputs=inputs,
                                           mem_gb_node=mem_gb_node, *args, **kwargs)
@@ -1347,6 +1342,9 @@ def __init__(self, name, inputs=None, mapper=None, #join_by=None,
         if self.mapper:
             pass #TODO
 
+        # list of (nodename, output name in the name, output name in wf) or (nodename, output name in the name)
+        self.outputs_nm = outputs_nm
+
         # dj not sure what was the motivation, wf_klasses gives an empty list
         #mro = self.__class__.mro()
         #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
@@ -1503,6 +1501,20 @@ def join(self, field, node=None):
         pass
 
 
+    def _reading_results(self):
+        for out in self.outputs_nm:
+            if len(out) == 2:
+                node_nm, out_nd_nm, out_wf_nm = out[0], out[1], out[1]
+            elif len(out) == 3:
+                node_nm, out_nd_nm, out_wf_nm = out
+            else:
+                raise Exception("outputs_nm should have 2 or 3 elements")
+            if out_wf_nm not in self._result.keys():
+                self._result[out_wf_nm] = self._node_names[node_nm].result[out_nd_nm]
+            else:
+                raise Exception("the key {} is already used in workflow.result".format(out_wf_nm))
+
+
 def is_function(obj):
     return hasattr(obj, '__call__')
 

From 14ebc78e0e9dc9594100caffeb7fa9ab40fe5ed0 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Sun, 30 Sep 2018 12:29:10 -0400
Subject: [PATCH 41/55] alloing for mapper in workflows (copying workflows in
 teh submitter); changing output/result in Node and Workflow

---
 nipype/pipeline/engine/auxiliary.py          |  32 +-
 nipype/pipeline/engine/state.py              |   4 +-
 nipype/pipeline/engine/submitter.py          |  90 ++++--
 nipype/pipeline/engine/tests/test_newnode.py |  73 ++---
 nipype/pipeline/engine/workers.py            |   7 +-
 nipype/pipeline/engine/workflows.py          | 322 +++++++++++--------
 6 files changed, 304 insertions(+), 224 deletions(-)

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index c04afd1292..50b2807c4f 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -7,44 +7,44 @@
 # dj: might create a new class or move to State
 
 # Function to change user provided mapper to "reverse polish notation" used in State
-def mapper2rpn(mapper, wf_mappers=None):
+def mapper2rpn(mapper, other_mappers=None):
     """ Functions that translate mapper to "reverse polish notation."""
     output_mapper = []
-    _ordering(mapper, i=0, output_mapper=output_mapper, wf_mappers=wf_mappers)
+    _ordering(mapper, i=0, output_mapper=output_mapper, other_mappers=other_mappers)
     return output_mapper
 
 
-def _ordering(el, i, output_mapper, current_sign=None, wf_mappers=None):
+def _ordering(el, i, output_mapper, current_sign=None, other_mappers=None):
     """ Used in the mapper2rpn to get a proper order of fields and signs. """
     if type(el) is tuple:
         # checking if the mapper dont contain mapper from previous nodes, i.e. has str "_NA", etc.
         if type(el[0]) is str and el[0].startswith("_"):
             node_nm = el[0][1:]
-            if node_nm not in wf_mappers:
+            if node_nm not in other_mappers:
                 raise Exception("can't ask for mapper from {}".format(node_nm))
-            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            mapper_mod = change_mapper(mapper=other_mappers[node_nm], name=node_nm)
             el = (mapper_mod, el[1])
         if type(el[1]) is str and el[1].startswith("_"):
             node_nm = el[1][1:]
-            if node_nm not in wf_mappers:
+            if node_nm not in other_mappers:
                 raise Exception("can't ask for mapper from {}".format(node_nm))
-            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            mapper_mod = change_mapper(mapper=other_mappers[node_nm], name=node_nm)
             el = (el[0], mapper_mod)
-        _iterate_list(el, ".", wf_mappers, output_mapper=output_mapper)
+        _iterate_list(el, ".", other_mappers, output_mapper=output_mapper)
     elif type(el) is list:
         if type(el[0]) is str and el[0].startswith("_"):
             node_nm = el[0][1:]
-            if node_nm not in wf_mappers:
+            if node_nm not in other_mappers:
                 raise Exception("can't ask for mapper from {}".format(node_nm))
-            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            mapper_mod = change_mapper(mapper=other_mappers[node_nm], name=node_nm)
             el[0] = mapper_mod
         if type(el[1]) is str and el[1].startswith("_"):
             node_nm = el[1][1:]
-            if node_nm not in wf_mappers:
+            if node_nm not in other_mappers:
                 raise Exception("can't ask for mapper from {}".format(node_nm))
-            mapper_mod = change_mapper(mapper=wf_mappers[node_nm], name=node_nm)
+            mapper_mod = change_mapper(mapper=other_mappers[node_nm], name=node_nm)
             el[1] = mapper_mod
-        _iterate_list(el, "*", wf_mappers, output_mapper=output_mapper)
+        _iterate_list(el, "*", other_mappers, output_mapper=output_mapper)
     elif type(el) is str:
         output_mapper.append(el)
     else:
@@ -54,10 +54,10 @@ def _ordering(el, i, output_mapper, current_sign=None, wf_mappers=None):
         output_mapper.append(current_sign)
 
 
-def _iterate_list(element, sign, wf_mappers, output_mapper):
+def _iterate_list(element, sign, other_mappers, output_mapper):
     """ Used in the mapper2rpn to get recursion. """
     for i, el in enumerate(element):
-        _ordering(el, i, current_sign=sign, wf_mappers=wf_mappers, output_mapper=output_mapper)
+        _ordering(el, i, current_sign=sign, other_mappers=other_mappers, output_mapper=output_mapper)
 
 
 # functions used in State to know which element should be used for a specific axis
@@ -68,7 +68,7 @@ def mapping_axis(state_inputs, mapper_rpn):
     stack = []
     current_axis = None
     current_shape = None
-
+    #pdb.set_trace()
     for el in mapper_rpn:
         if el == ".":
             right = stack.pop()
diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
index c81f18e405..58becdd23c 100644
--- a/nipype/pipeline/engine/state.py
+++ b/nipype/pipeline/engine/state.py
@@ -5,13 +5,13 @@
 from . import auxiliary as aux
 
 class State(object):
-    def __init__(self, node_name, mapper=None, wf_mappers=None):
+    def __init__(self, node_name, mapper=None, other_mappers=None):
         self._mapper = mapper
         self.node_name = node_name
         if self._mapper:
             # changing mapper (as in rpn), so I can read from left to right
             # e.g. if mapper=('d', ['e', 'r']), _mapper_rpn=['d', 'e', 'r', '*', '.']
-            self._mapper_rpn = aux.mapper2rpn(self._mapper, wf_mappers=wf_mappers)
+            self._mapper_rpn = aux.mapper2rpn(self._mapper, other_mappers=other_mappers)
             self._input_names_mapper = [i for i in self._mapper_rpn if i not in ["*", "."]]
         else:
             self._mapper_rpn = []
diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index ea10085aa1..9bc5273ad5 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -6,6 +6,7 @@
 standard_library.install_aliases()
 
 import os, pdb, time
+from copy import deepcopy
 
 from .workers import MpWorker, SerialWorker, DaskWorker, ConcurrentFuturesWorker
 
@@ -34,6 +35,7 @@ def submit_work(self, node):
 
     def _submit_work_el(self, node, i, ind):
         logger.debug("SUBMIT WORKER, node: {}, ind: {}".format(node, ind))
+        print("SUBMIT WORK", node.inputs)
         self.worker.run_el(node.run_interface_el, (i, ind))
 
 
@@ -49,39 +51,37 @@ def __init__(self, plugin, node):
 
     def run_node(self):
         self.submit_work(self.node)
-        while not self.node.global_done:
+        while not self.node.finished_all:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
 
 
 class SubmitterWorkflow(Submitter):
-    def __init__(self, graph, plugin):
+    def __init__(self, workflow, plugin):
         super(SubmitterWorkflow, self).__init__(plugin)
-        self.graph = graph
-        logger.debug('Initialize Submitter, graph: {}'.format(graph))
-        self._to_finish = list(self.graph)
+        self.workflow = workflow
+        logger.debug('Initialize Submitter, graph: {}'.format(self.workflow.graph_sorted))
+        #self._to_finish = list(self.workflow.graph)
+        self._to_finish = []
 
 
     def run_workflow(self):
-        for (i_n, node) in enumerate(self.graph):
-            # submitting all the nodes who are self sufficient (self.graph is already sorted)
-            if node.sufficient:
-                self.submit_work(node)
-            # if its not, its been added to a line
-            else:
-                break
-
-            # in case there is no element in the graph that goes to the break
-            # i want to be sure that not calculating the last node again in the next for loop
-            if i_n == len(self.graph) - 1:
-                i_n += 1
-
-        # all nodes that are not self sufficient will go to the line
-        # iterating over all elements
-        # (i think ordered list work well here, since it's more efficient to check within a specific order)
-        for nn in list(self.graph)[i_n:]:
-            for (i, ind) in enumerate(nn.state.index_generator):
-                self.node_line.append((nn, i, ind))
+        if self.workflow.mapper:
+            self.workflow.inner_nodes = {}
+            for key in self.workflow._node_names.keys():
+                self.workflow.inner_nodes[key] = []
+            for (i, ind) in enumerate(self.workflow.state.index_generator):
+                print("LOOP", i, self._to_finish, self.node_line)
+                wf_inputs = self.workflow.state.state_values(ind)
+                new_workflow = deepcopy(self.workflow)
+                #pdb.set_trace()
+                new_workflow.preparing(wf_inputs=wf_inputs)
+                #pdb.set_trace()
+                self.run_workflow_el(workflow=new_workflow)
+                print("LOOP END", i, self._to_finish, self.node_line)
+        else:
+            self.workflow.preparing(wf_inputs=self.workflow.inputs)
+            self.run_workflow_el(workflow=self.workflow)
 
         # this parts submits nodes that are waiting to be run
         # it should stop when nothing is waiting
@@ -89,15 +89,45 @@ def run_workflow(self):
             logger.debug("Submitter, in while, node_line: {}".format(self.node_line))
             time.sleep(3)
 
-        # TODO(?): combining two while together
         # this part simply waiting for all "last nodes" to finish
         while self._output_check():
             logger.debug("Submitter, in while, to_finish: {}".format(self._to_finish))
             time.sleep(3)
 
 
+    def run_workflow_el(self, workflow):
+        #pdb.set_trace()
+        for (i_n, node) in enumerate(workflow.graph_sorted):
+            if self.workflow.mapper:
+                self.workflow.inner_nodes[node.name].append(node)
+            node.prepare_state_input()
+
+
+            print("RUN WF", node.name, node.inputs)
+            self._to_finish.append(node)
+            # submitting all the nodes who are self sufficient (self.workflow.graph is already sorted)
+            if node.ready2run:
+
+                self.submit_work(node)
+            # if its not, its been added to a line
+            else:
+                break
+            # in case there is no element in the graph that goes to the break
+            # i want to be sure that not calculating the last node again in the next for loop
+            if i_n == len(workflow.graph_sorted) - 1:
+                i_n += 1
+
+        # all nodes that are not self sufficient (not ready to run) will go to the line
+        # iterating over all elements
+        for nn in list(workflow.graph_sorted)[i_n:]:
+            for (i, ind) in enumerate(nn.state.index_generator):
+                self._to_finish.append(nn)
+                self.node_line.append((nn, i, ind))
+
+
     # for now without callback, so checking all nodes (with ind) in some order
     def _nodes_check(self):
+        print("NODES CHECK BEG", self.node_line)
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
             if to_node.checking_input_el(ind):
@@ -108,17 +138,19 @@ def _nodes_check(self):
         # can't remove during iterating
         for rn in _to_remove:
             self.node_line.remove(rn)
+        print("NODES CHECK END", self.node_line)
         return self.node_line
 
 
     # this I believe can be done for entire node
     def _output_check(self):
         _to_remove = []
+        print("OUT CHECK", self._to_finish)
         for node in self._to_finish:
-            print("_output check node", node, node.global_done)
-            if node.global_done:
+            print("_output check node", node, node.finished_all)
+            if node.finished_all:
                 _to_remove.append(node)
         for rn in _to_remove:
             self._to_finish.remove(rn)
-        return self._to_finish
-
+        print("OUT CHECK END", self._to_finish)
+        return self._to_finish
\ No newline at end of file
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 61d68d46ce..76d7efe4a8 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -8,6 +8,7 @@
 python35_only = pytest.mark.skipif(sys.version_info < (3, 5),
                                    reason="requires Python>3.4")
 
+Plugins = ["serial"]
 Plugins = ["serial", "mp", "cf", "dask"]
 
 def fun_addtwo(a):
@@ -16,12 +17,10 @@ def fun_addtwo(a):
         time.sleep(2)
     return a + 2
 
-
 def fun_addvar(a, b):
     return a + b
 
 
-
 def test_node_1():
     """Node with mandatory arguments only"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -190,7 +189,6 @@ def test_workflow_1(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add_nodes([na])
-
     wf.run(plugin=plugin)
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -621,13 +619,13 @@ def test_workflow_6b(plugin):
 def test_workflow_7(plugin):
     """using inputs for workflow and connect_workflow"""
     # adding inputs to the workflow directly
-    wf = NewWorkflow(name="wf7", inputs={"wf_a": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
+    wf = NewWorkflow(name="wf7", inputs={"wfa": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na)
     # connecting the node with inputs from the workflow
-    wf.connect_workflow("NA", "wf_a", "a")
+    wf.connect_workflow("NA", "wfa", "a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
@@ -645,13 +643,13 @@ def test_workflow_7(plugin):
 def test_workflow_7a(plugin):
     """using inputs for workflow and connect(None...)"""
     # adding inputs to the workflow directly
-    wf = NewWorkflow(name="wf7a", inputs={"wf_a": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
+    wf = NewWorkflow(name="wf7a", inputs={"wfa": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na)
     # if connect has None as the first arg, it is the same as connect_workflow
-    wf.connect(None, "wf_a", "NA", "a")
+    wf.connect(None, "wfa", "NA", "a")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
@@ -668,11 +666,11 @@ def test_workflow_7a(plugin):
 @python35_only
 def test_workflow_7b(plugin):
     """using inputs for workflow and kwarg arg in add (instead of connect)"""
-    wf = NewWorkflow(name="wf7b", inputs={"wf_a": [3, 5]}, workingdir="test_wf7b_{}".format(plugin))
+    wf = NewWorkflow(name="wf7b", inputs={"wfa": [3, 5]}, workingdir="test_wf7b_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     # using kwrg argument in the add method (instead of connect or connect_workflow
-    wf.add(na, a="wf_a")
+    wf.add(na, a="wfa")
     wf.map(mapper="a")
     wf.run(plugin=plugin)
 
@@ -875,6 +873,7 @@ def test_workflow_12(plugin):
     key_sort = list(expected[0][0].keys())
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.result["NA_out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    #pdb.set_trace()
     for i, res in enumerate(expected):
         assert wf.result["NA_out"][i][0] == res[0]
         assert wf.result["NA_out"][i][1] == res[1]
@@ -905,65 +904,51 @@ def test_workflow_12a(plugin):
     wf.add(nb)
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
-    wf.run(plugin=plugin)
 
     # wf_out can't be used twice in wf.result
     with pytest.raises(Exception) as exinfo:
-        wf.result
+        wf.run(plugin=plugin)
     assert str(exinfo.value) == "the key wf_out is already used in workflow.result"
 
 # tests for a workflow that have its own input and mapper
-# WIP
-@pytest.mark.xfail(reason="WIP")
+
+
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_13(plugin):
     """using inputs for workflow and connect_workflow"""
-    wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12_{}".format(plugin))
+    wf = NewWorkflow(name="wf13", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13_{}".format(plugin),
+                     outputs_nm=[("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wf.add(na)
-    wf.connect_workflow("NA", "wf_a", "a")
-
-    assert len(wf.inner_workflows) == 2
-    assert wf.inner_workflows[0].mapper is None
-    assert wf.inner_workflows[0].inputs == {'wf9(0,).wf_a': 3}
-    assert wf.inner_workflows[1].inputs == {'wf9(1,).wf_a': 5}
+    wf.connect_workflow("NA", "wfa", "a")
     wf.run(plugin=plugin)
 
-    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.inner_workflows[0].nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    expected = [({"wf13.wfa": 3}, [({"NA.a": 3}, 5)]),
+                ({'wf13.wfa': 5}, [({"NA.a": 5}, 7)])]
     for i, res in enumerate(expected):
-        assert wf.inner_workflows[i].nodes[0].result["out"][0][0] == res[0]
-        assert wf.inner_workflows[i].nodes[0].result["out"][0][1] == res[1]
+        assert wf.result["NA_out"][i][0] == res[0]
+        assert wf.result["NA_out"][i][1][0][0] == res[1][0][0]
+        assert wf.result["NA_out"][i][1][0][1] == res[1][0][1]
 
 
-@pytest.mark.xfail(reason="WIP")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_13a(plugin):
     """using inputs for workflow and connect_workflow"""
-    wf = NewWorkflow(name="wf9", inputs={"wf_a": [3, 5]}, mapper="wf_a", workingdir="test_wf12a_{}".format(plugin))
+    wf = NewWorkflow(name="wf13a", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13a_{}".format(plugin),
+                     outputs_nm=[("NA", "out", "NA_out")])
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     na = NewNode(name="NA", interface=interf_addvar, base_dir="na", mapper="b", inputs={"b": [10, 20]})
     wf.add(na)
-    wf.connect_workflow("NA", "wf_a", "a")
-
-    assert len(wf.inner_workflows) == 2
-    assert wf.inner_workflows[0].mapper is None
-    assert wf.inner_workflows[0].inputs == {'wf9(0,).wf_a': 3}
-    assert wf.inner_workflows[1].inputs == {'wf9(1,).wf_a': 5}
+    wf.connect_workflow("NA", "wfa", "a")
     wf.run(plugin=plugin)
 
-    expected = [[({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)],
-                [({"NA.a": 5, "NA.b": 10}, 15), ({"NA.a": 5, "NA.b": 20}, 25)]]
-    key_sort = list(expected[0][0][0].keys())
-    for exp in expected:
-        exp.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.inner_workflows[0].nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res_l in enumerate(expected):
-        for j, res in enumerate(res_l):
-            assert wf.inner_workflows[i].nodes[0].result["out"][j][0] == res[0]
-            assert wf.inner_workflows[i].nodes[0].result["out"][j][1] == res[1]
+    expected = [({"wf13a.wfa": 3}, [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)]),
+                ({'wf13a.wfa': 5}, [({"NA.a": 5, "NA.b": 10}, 15), ({"NA.a": 5, "NA.b": 20}, 25)])]
+    for i, res in enumerate(expected):
+        assert wf.result["NA_out"][i][0] == res[0]
+        for j in range(len(res[1])):
+            assert wf.result["NA_out"][i][1][j][0] == res[1][j][0]
+            assert wf.result["NA_out"][i][1][j][1] == res[1][j][1]
\ No newline at end of file
diff --git a/nipype/pipeline/engine/workers.py b/nipype/pipeline/engine/workers.py
index d20a12725c..b7e35c99ad 100644
--- a/nipype/pipeline/engine/workers.py
+++ b/nipype/pipeline/engine/workers.py
@@ -85,12 +85,13 @@ def __init__(self):
 
 
     def run_el(self, interface, inp):
-        print("DASK, run_el: ", interface, inp)
-        x = self.client.submit(interface, inp[0], inp[1])
+        print("DASK, run_el: ", interface, inp, time.time())
+        # dask  doesn't copy the node second time, so it doesn't see that I change input in the meantime (??)
+        x = self.client.submit(interface, inp[0], inp[1], time.time())
         print("DASK, status: ", x.status)
         # this important, otherwise dask will not finish the job
         x.add_done_callback(lambda x: print("DONE ", interface, inp))
-        #print("res", x.result())
+        print("res", x.result())
 
 
     def close(self):
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index c8871662ff..a8fe3a98f8 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1071,9 +1071,10 @@ class Join(Node):
 class MapState(object):
     pass
 
+
 # dj ??: should I use EngineBase?
 class NewBase(object):
-    def __init__(self, name, mapper=None, inputs=None, wf_mappers=None, mem_gb_node=None, *args, **kwargs):
+    def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb_node=None, *args, **kwargs):
         self.name = name
         #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
         if inputs:
@@ -1089,10 +1090,11 @@ def __init__(self, name, mapper=None, inputs=None, wf_mappers=None, mem_gb_node=
             # adding name of the node to the input name within the mapper
             mapper = aux.change_mapper(mapper, self.name)
         self._mapper = mapper
-        self._wf_mappers = wf_mappers
+        # information about other nodes' mappers from workflow (in case the mapper from previous node is used)
+        self._other_mappers = other_mappers
         # create state (takes care of mapper, connects inputs with axes, so we can ask for specifc element)
-        self._state = state.State(mapper=self._mapper, node_name=self.name, wf_mappers=self._wf_mappers)
-
+        self._state = state.State(mapper=self._mapper, node_name=self.name, other_mappers=self._other_mappers)
+        self._output = {}
         self._result = {}
 
 
@@ -1112,16 +1114,21 @@ def mapper(self):
     def mapper(self, mapper):
         self._mapper = mapper
         # updating state
-        self._state = state.State(mapper=self._mapper, node_name=self.name, wf_mappers=self._wf_mappers)
-
-    @property
-    def inputs(self):
-        return self._inputs
+        self._state = state.State(mapper=self._mapper, node_name=self.name, other_mappers=self._other_mappers)
 
     @property
     def state_inputs(self):
         return self._state_inputs
 
+    @state_inputs.setter
+    def state_inputs(self, state_inputs):
+        self._state_inputs.update(state_inputs)
+
+
+    @property
+    def output(self):
+        return self._output
+
     @property
     def result(self):
         if not self._result:
@@ -1136,55 +1143,53 @@ def prepare_state_input(self):
 
 class NewNode(NewBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, wf_mappers=None, mem_gb_node=None, *args, **kwargs):
+                 base_dir=None, other_mappers=None, mem_gb_node=None, *args, **kwargs):
         super(NewNode, self).__init__(name=name, mapper=mapper, inputs=inputs,
-                                      wf_mappers=wf_mappers, mem_gb_node=mem_gb_node,
+                                      other_mappers=other_mappers, mem_gb_node=mem_gb_node,
                                       *args, **kwargs)
 
-        self._nodedir = base_dir
-        self._interface = interface
-        self._interface.input_map = dict((key, "{}.{}".format(self.name, value))
-                                         for (key, value) in self._interface.input_map.items())
-
-        self._needed_outputs = []
-        self._out_nm = self._interface._output_nm
-        self._global_done = False
-
-        self.sufficient = True
-
-
-    @property
-    def nodedir(self):
-        return self._nodedir
+        # working directory for node, will be change if node is a part of a wf
+        self.nodedir = base_dir
+        self.interface = interface
+        # adding node name to the interface's name mapping
+        self.interface.input_map = dict((key, "{}.{}".format(self.name, value))
+                                         for (key, value) in self.interface.input_map.items())
+        # needed outputs from other nodes if the node part of a wf
+        self.needed_outputs = []
+        # output names taken from interface output name
+        self.output_names = self.interface._output_nm
+        # flag that says if node finished all jobs
+        self._finished_all = False
+        # flag that says if the node is ready to run (has all input)
+        self.ready2run = True
 
 
-    @nodedir.setter
-    def nodedir(self, nodedir):
-        self._nodedir = nodedir
+    # dj: not sure if I need it
+    def __deepcopy__(self, memo): # memo is a dict of id's to copies
+        id_self = id(self)        # memoization avoids unnecesary recursion
+        _copy = memo.get(id_self)
+        if _copy is None:
+            # changing names of inputs and input_map, so it doesnt contain node.name
+            inputs_copy = dict((key[len(self.name)+1:], deepcopy(value))
+                               for (key, value) in self.inputs.items())
+            interface_copy = deepcopy(self.interface)
+            interface_copy.input_map = dict((key, val[len(self.name)+1:])
+                                            for (key, val) in interface_copy.input_map.items())
+            _copy = type(self)(
+                name=deepcopy(self.name), interface=interface_copy,
+                inputs=inputs_copy, mapper=deepcopy(self.mapper),
+                base_dir=deepcopy(self.nodedir), other_mappers=deepcopy(self._other_mappers))
+            memo[id_self] = _copy
+        return _copy
 
 
     @property
-    def interface(self):
-        return self._interface
-
-    # dj: not sure if I need it
-    # def __deepcopy__(self, memo): # memo is a dict of id's to copies
-    #     id_self = id(self)        # memoization avoids unnecesary recursion
-    #     _copy = memo.get(id_self)
-    #     if _copy is None:
-    #         # changing names of inputs and input_map, so it doesnt contain node.name
-    #         inputs_copy = dict((key[len(self.name)+1:], deepcopy(value))
-    #                            for (key, value) in self.inputs.items())
-    #         interface_copy = deepcopy(self.interface)
-    #         interface_copy.input_map = dict((key, val[len(self.name)+1:])
-    #                                         for (key, val) in interface_copy.input_map.items())
-    #         _copy = type(self)(
-    #             name=deepcopy(self.name), interface=interface_copy,
-    #             inputs=inputs_copy, mapper=deepcopy(self.mapper),
-    #             base_dir=deepcopy(self._nodedir), wf_mappers=deepcopy(self._wf_mappers))
-    #         memo[id_self] = _copy
-    #     return _copy
+    def inputs(self):
+        return self._inputs
 
+    @inputs.setter
+    def inputs(self, inputs):
+        self._inputs.update(inputs)
 
 
     def map(self, mapper, inputs=None):
@@ -1201,7 +1206,7 @@ def map(self, mapper, inputs=None):
             self._state_inputs.update(inputs)
         if mapper:
             # updating state if we have a new mapper
-            self._state = state.State(mapper=self._mapper, node_name=self.name, wf_mappers=self._wf_mappers)
+            self._state = state.State(mapper=self._mapper, node_name=self.name, other_mappers=self._other_mappers)
 
 #    def map_orig(self, field, values=None):
 #        if isinstance(field, list):
@@ -1217,21 +1222,18 @@ def map(self, mapper, inputs=None):
 #        self._mappers[field] = values
 
 
-    @property
-    def needed_outputs(self):
-        return self._needed_outputs
-
-
-    def run_interface_el(self, i, ind):
+    def run_interface_el(self, i, ind, test=None):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
         state_dict, inputs_dict = self._collecting_input_el(ind)
+        dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
+        print("Run interface el, dict={}".format(state_dict))
         logger.debug("Run interface el, name={}, inputs_dict={}, state_dict={}".format(
                                                             self.name, inputs_dict, state_dict))
-        res = self._interface.run(inputs_dict)
-        output = self._interface.output
+        res = self.interface.run(inputs_dict)
+        output = self.interface.output
+        print("Run interface el, output={}".format(output))
         logger.debug("Run interface el, output={}".format(output))
-        dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
         # TODO when join
         #if self._joinByKey:
         #    dir_join = "join_" + "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items()) if i not in self._joinByKey])
@@ -1240,14 +1242,33 @@ def run_interface_el(self, i, ind):
         #if self._joinByKey or self._join:
         #    os.makedirs(os.path.join(self.nodedir, dir_join), exist_ok=True)
         #    dir_nm_el = os.path.join(dir_join, dir_nm_el)
+        self._writting_results_tmp(state_dict, dir_nm_el, output)
+        return res
+
+
+    def _writting_results_tmp(self, state_dict, dir_nm_el, output):
+        """temporary method to write the results in the files (this is usually part of a interface)"""
         os.makedirs(os.path.join(self.nodedir, dir_nm_el), exist_ok=True)
-        for key_out in list(output.keys()):
+        for key_out, val_out in output.items():
             with open(os.path.join(self.nodedir, dir_nm_el, key_out+".txt"), "w") as fout:
-                fout.write(str(output[key_out]))
-        return res
+                fout.write(str(val_out))
+
+
+    def _collecting_output(self):
+        for key_out in self.output_names:
+            self._output[key_out] = {}
+            #pdb.set_trace()
+            for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
+                state_dict, inputs_dict = self._collecting_input_el(ind)
+                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
+                #pdb.set_trace()
+                self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.nodedir, dir_nm_el, key_out + ".txt"))
+        return self._output
+
 
     # dj: this is not used for a single node
     def _collecting_input_el(self, ind):
+        """collecting all inputs required to run the node (for specific state element)"""
         state_dict = self.state.state_values(ind)
         inputs_dict = {k: state_dict[k] for k in self._inputs.keys()}
         # reading extra inputs that come from previous nodes
@@ -1261,6 +1282,7 @@ def _collecting_input_el(self, ind):
 
 
     def checking_input_el(self, ind):
+        """checking if all inputs are available (for specific state element)"""
         try:
             self._collecting_input_el(ind)
             return True
@@ -1270,54 +1292,53 @@ def checking_input_el(self, ind):
 
     # checking if all outputs are saved
     @property
-    def global_done(self):
-        # once _global_done os True, this should not change
-        logger.debug('global_done {}'.format(self._global_done))
-        if self._global_done:
-            return self._global_done
+    def finished_all(self):
+        # once _finished_all os True, this should not change
+        print("in finished all", self.inputs, self.nodedir)
+        logger.debug('finished_all {}'.format(self._finished_all))
+        if self._finished_all:
+            return self._finished_all
         else:
             return self._check_all_results()
 
+
     # dj: version without join
     def _check_all_results(self):
-        # checking if all files that should be created are present
+        """checking if all files that should be created are present"""
         for ind in itertools.product(*self.state.all_elements):
             state_dict = self.state.state_values(ind)
             dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
-            for key_out in self._out_nm:
+            for key_out in self.output_names:
                 if not os.path.isfile(os.path.join(self.nodedir, dir_nm_el, key_out+".txt")):
                     return False
-        self._global_done = True
+        self._finished_all = True
         return True
 
 
     def _reading_results(self):
+        """temporary: reading results from output files
+            should be probably just reading output for self.output_names
         """
-        reading results from file,
-        doesn't check if everything is ready, i.e. if self.global_done"""
-        for key_out in self._out_nm:
+        for key_out in self.output_names:
             self._result[key_out] = []
             if self._state_inputs:
-                files = [name for name in glob.glob("{}/*/{}.txt".format(self.nodedir, key_out))]
-                for file in files:
-                    st_el = file.split(os.sep)[-2].split("_")
-                    st_dict = collections.OrderedDict([(el.split(":")[0], eval(el.split(":")[1]))
-                                                            for el in st_el])
-                    with open(file) as fout:
-                        logger.debug('Reading Results: file={}, st_dict={}'.format(file, st_dict))
+                for _, (st_dict, filename) in self._output[key_out].items():
+                    with open(filename) as fout:
                         self._result[key_out].append((st_dict, eval(fout.readline())))
-            # for nodes without input
             else:
-                files = [name for name in glob.glob("{}/{}.txt".format(self.nodedir, key_out))]
-                with open(files[0]) as fout:
+                # st_dict should be {}
+                (st_dict, filename) = self._output[key_out][None]
+                with open(filename) as fout:
                     self._result[key_out].append(({}, eval(fout.readline())))
 
 
     def run(self, plugin="serial"):
+        """preparing the node to run and run the interface"""
         self.prepare_state_input()
         submitter = sub.SubmitterNode(plugin, node=self)
         submitter.run_node()
         submitter.close()
+        self._collecting_output()
 
 
 class NewWorkflow(NewBase):
@@ -1329,6 +1350,7 @@ def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=Non
         self.graph = nx.DiGraph()
         self._nodes = []
         self.connected_var = {}
+        self.needed_inp_wf = []
         if nodes:
             self.add_nodes(nodes)
         for nn in self._nodes:
@@ -1339,9 +1361,6 @@ def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=Non
         # dj: not sure if this should be different than base_dir
         self.workingdir = os.path.join(os.getcwd(), workingdir)
 
-        if self.mapper:
-            pass #TODO
-
         # list of (nodename, output name in the name, output name in wf) or (nodename, output name in the name)
         self.outputs_nm = outputs_nm
 
@@ -1357,6 +1376,14 @@ def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=Non
 
         #    self.add(name, value)
 
+    @property
+    def inputs(self):
+        return self._inputs
+
+    @inputs.setter
+    def inputs(self, inputs):
+        self._inputs.update(dict(("{}.{}".format(self.name, key), value) for (key, value) in inputs.items()))
+
 
     @property
     def nodes(self):
@@ -1370,7 +1397,6 @@ def add_nodes(self, nodes):
             self._nodes.append(nn)
             #self._inputs.update(nn.inputs)
             self.connected_var[nn] = {}
-            nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
             self._node_names[nn.name] = nn
             self._node_mappers[nn.name] = nn.mapper
 
@@ -1381,15 +1407,18 @@ def connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
             to_node = self._node_names[to_node_nm]
             self.graph.add_edges_from([(from_node, to_node)])
             if not to_node in self.nodes:
-                self._add_nodes(to_node)
+                self.add_nodes(to_node)
             self.connected_var[to_node][to_socket] = (from_node, from_socket)
             # from_node.sending_output.append((from_socket, to_node, to_socket))
             logger.debug('connecting {} and {}'.format(from_node, to_node))
         else:
             self.connect_workflow(to_node_nm, from_socket, to_socket)
 
-
+    # TODO: change (at least the name)
     def connect_workflow(self, node_nm, inp_wf, inp_nd):
+        self.needed_inp_wf.append((node_nm, inp_wf, inp_nd))
+
+    def _connect_workflow(self, node_nm, inp_wf, inp_nd):
         node = self._node_names[node_nm]
         if "{}.{}".format(self.name, inp_wf) in self.inputs:
             node.state_inputs.update({"{}.{}".format(node_nm, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
@@ -1398,21 +1427,31 @@ def connect_workflow(self, node_nm, inp_wf, inp_nd):
             raise Exception("{} not in the workflow inputs".format(inp_wf))
 
 
-    def _preparing(self):
+    def preparing(self, wf_inputs=None):
         """preparing nodes which are connected: setting the final mapper and state_inputs"""
-        self.graph_sorted = list(nx.topological_sort(self.graph))
-        logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
+        #pdb.set_trace()
+        for node_nm, inp_wf, inp_nd in self.needed_inp_wf:
+            node = self._node_names[node_nm]
+            if "{}.{}".format(self.name, inp_wf) in wf_inputs:
+                node.state_inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
+                node.inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
+            else:
+                raise Exception("{}.{} not in the workflow inputs".format(self.name, inp_wf))
         for nn in self.graph_sorted:
-            nn.wfdir = self.workingdir
+            if self.mapper:
+                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
+                nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.nodedir)
+                nn._finished_all = False # helps when mp is used
+            else:
+                nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
             try:
                 for inp, (out_node, out_var) in self.connected_var[nn].items():
-                    nn.sufficient = False #it has some history (doesnt have to be in the loop)
+                    nn.ready2run = False #it has some history (doesnt have to be in the loop)
                     nn.state_inputs.update(out_node.state_inputs)
                     nn.needed_outputs.append((out_node, out_var, inp))
                     #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
                     if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
                         nn.mapper = out_node.mapper
-                        #nn._mapper = inp #not used
                     else:
                         pass
                     #TODO: implement inner mapper
@@ -1420,21 +1459,51 @@ def _preparing(self):
                 # tmp: we don't care about nn that are not in self.connected_var
                 pass
 
-            #nn.preparing_node() #dj: only one this needed?:
-            # do i need it at all?
             nn.prepare_state_input()
 
 
     def run(self, plugin="serial"):
-        self._preparing()
-        submitter = sub.SubmitterWorkflow(plugin=plugin, graph=self.graph)
+        self.graph_sorted = list(nx.topological_sort(self.graph))
+        #self.preparing(wf_inputs=self.inputs) # moved to submitter
+        self.prepare_state_input()
+        logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
+        submitter = sub.SubmitterWorkflow(workflow=self, plugin=plugin)
         submitter.run_workflow()
         submitter.close()
+        self._collecting_outpt()
+
+
+    def _collecting_outpt(self):
+        # not sure, if I should collecto output of all nodes or only the ones that are used in wf.output
+        self.node_outputs = {}
+        for nn in self.graph:
+            if self.mapper:
+                self.node_outputs[nn.name] = [ni._collecting_output() for ni in self.inner_nodes[nn.name]]
+            else:
+                self.node_outputs[nn.name] = nn._collecting_output()
+        if self.outputs_nm:
+            for out in self.outputs_nm:
+                if len(out) == 2:
+                    node_nm, out_nd_nm, out_wf_nm = out[0], out[1], out[1]
+                elif len(out) == 3:
+                    node_nm, out_nd_nm, out_wf_nm = out
+                else:
+                    raise Exception("outputs_nm should have 2 or 3 elements")
+                if out_wf_nm not in self._output.keys():
+                    if self.mapper:
+                        self._output[out_wf_nm] = {}
+                        for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
+                            wf_inputs_dict = self.state.state_values(ind)
+                            dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
+                            self._output[out_wf_nm][dir_nm_el] = self.node_outputs[node_nm][i][out_nd_nm]
+                    else:
+                        self._output[out_wf_nm] = self.node_outputs[node_nm][out_nd_nm]
+                else:
+                    raise Exception("the key {} is already used in workflow.result".format(out_wf_nm))
 
 
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
             mem_gb=None, **kwargs):
-        # dj TODO: should I move this if checks to NewNode __init__?
         if is_function(runnable):
             if not output_nm:
                 output_nm = ["out"]
@@ -1444,25 +1513,20 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
             if not base_dir:
                 base_dir = name
             node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
-                           wf_mappers=self._node_mappers, mem_gb_node=mem_gb)
+                           other_mappers=self._node_mappers, mem_gb_node=mem_gb)
         elif is_interface(runnable):
             if not name:
                 raise Exception("you have to specify name for the node")
             if not base_dir:
                 base_dir = name
             node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
-                           wf_mappers=self._node_mappers, mem_gb_node=mem_gb)
+                           other_mappers=self._node_mappers, mem_gb_node=mem_gb)
         elif is_node(runnable):
             node = runnable
-            #dj: dont have clonning right now
-            #node = runnable if runnable.name == name else runnable.clone(name=name)
         else:
             raise ValueError("Unknown workflow element: {!r}".format(runnable))
         self.add_nodes([node])
-        # dj: i'm using name as a name of a workflow
-        #setattr(self, name, node)
-        #self._nodes[name] = node
-        self._last_added = node #name
+        self._last_added = node
 
         # connecting inputs to other nodes outputs
         for (inp, source) in kwargs.items():
@@ -1477,17 +1541,6 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
 
 
     def map(self, mapper, node=None, inputs=None):
-        # if node is None:
-        #     if '.' in field:
-        #         node, field = field.rsplit('.', 1)
-        #     else:
-        #         node = self._last_added
-        #
-        # if '.' in node:
-        #     subwf, node = node.split('.', 1)
-        #     self._nodes[subwf].map(field, node, values)
-        #     return
-
         if not node:
             node = self._last_added
 
@@ -1502,17 +1555,26 @@ def join(self, field, node=None):
 
 
     def _reading_results(self):
-        for out in self.outputs_nm:
-            if len(out) == 2:
-                node_nm, out_nd_nm, out_wf_nm = out[0], out[1], out[1]
-            elif len(out) == 3:
-                node_nm, out_nd_nm, out_wf_nm = out
-            else:
-                raise Exception("outputs_nm should have 2 or 3 elements")
-            if out_wf_nm not in self._result.keys():
-                self._result[out_wf_nm] = self._node_names[node_nm].result[out_nd_nm]
-            else:
-                raise Exception("the key {} is already used in workflow.result".format(out_wf_nm))
+        if self.outputs_nm:
+            for out in self.outputs_nm:
+                key_out = out[2] if len(out)==3 else out[1]
+                self._result[key_out] = []
+                if self.mapper:
+                    for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
+                        wf_inputs_dict = self.state.state_values(ind)
+                        dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
+                        res_l= []
+                        for key, val in self.output[key_out][dir_nm_el].items():
+                            with open(val[1]) as fout:
+                                logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
+                                res_l.append((val[0], eval(fout.readline())))
+                        self._result[key_out].append((wf_inputs_dict, res_l))
+                else:
+                    for key, val in self.output[key_out].items():
+                        with open(val[1]) as fout:
+                            logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
+                            self._result[key_out].append((val[0], eval(fout.readline())))
+
 
 
 def is_function(obj):

From a9407b4544b7ae1a37d71cda7b06f3c80c920acd Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Sun, 30 Sep 2018 12:31:33 -0400
Subject: [PATCH 42/55] updating tests for mappers

---
 nipype/pipeline/engine/tests/test_auxiliary.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_auxiliary.py b/nipype/pipeline/engine/tests/test_auxiliary.py
index 1002eecfb5..4c5df1df2f 100644
--- a/nipype/pipeline/engine/tests/test_auxiliary.py
+++ b/nipype/pipeline/engine/tests/test_auxiliary.py
@@ -16,15 +16,15 @@ def test_mapper2rpn(mapper, rpn):
     assert aux.mapper2rpn(mapper) == rpn
 
 
-@pytest.mark.parametrize("mapper, wf_mappers, rpn",
+@pytest.mark.parametrize("mapper, other_mappers, rpn",
                          [
                             (["a", "_NA"],        {"NA": ("b", "c")}, ["a", "NA.b", "NA.c", ".", "*"]),
                             (["_NA", "c"],        {"NA": ("a", "b")}, ["NA.a", "NA.b", ".", "c", "*"]),
                             (["a", ("b", "_NA")], {"NA": ["c", "d"]}, ["a", "b", "NA.c", "NA.d", "*", ".", "*"])
                         ])
 
-def test_mapper2rpn_wf_mapper(mapper, wf_mappers, rpn):
-    assert aux.mapper2rpn(mapper, wf_mappers=wf_mappers) == rpn
+def test_mapper2rpn_wf_mapper(mapper, other_mappers, rpn):
+    assert aux.mapper2rpn(mapper, other_mappers=other_mappers) == rpn
 
 
 @pytest.mark.parametrize("mapper, mapper_changed",

From 63fe2f40df4100747ff69f1b19c7ea48bbd28556 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 1 Oct 2018 14:02:12 -0400
Subject: [PATCH 43/55] adding inputs and imports to newnode_nero

---
 .../engine/tests/test_newnode_neuro.py        | 240 +++++++++---------
 1 file changed, 123 insertions(+), 117 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode_neuro.py b/nipype/pipeline/engine/tests/test_newnode_neuro.py
index 5deed3807e..a35a18400c 100644
--- a/nipype/pipeline/engine/tests/test_newnode_neuro.py
+++ b/nipype/pipeline/engine/tests/test_newnode_neuro.py
@@ -1,127 +1,133 @@
-from .. import NewNode, NewWorkflow
+from nipype.pipeline.engine import NewNode, NewWorkflow
 from ..auxiliary import Function_Interface
 
 #dj niworkflows vs ...??
-from niworkflows.nipype.interfaces.utility import Rename
-import niworkflows.nipype.interfaces.freesurfer as fs
+from nipype.interfaces.utility import Rename
+import nipype.interfaces.freesurfer as fs
 
-from ...interfaces.freesurfer import PatchedConcatenateLTA as ConcatenateLTA
+from fmriprep.interfaces.freesurfer import PatchedConcatenateLTA as ConcatenateLTA
+
+import pdb
 
 Name = "example"
 DEFAULT_MEMORY_MIN_GB = None
 # TODO, adding fields to Inputs (subject_id)
-Inputs = {"subject_id": [],
-          "output_spaces": [],
-          "source_file": [],
-          "t1_preproc": [],
-          "t1_2_fsnative_forward_transform": [],
-          "subjects_dir": []
+Inputs = {"subject_id": "sub-01",
+          "output_spaces": ["fsaverage", "fsaverage5"],
+          "source_file": "/Users/dorota/fmriprep_test/workdir1/fmriprep_wf/single_subject_01_wf/func_preproc_ses_test_task_fingerfootlips_wf/bold_t1_trans_wf/merge/vol0000_xform-00000_merged.nii",
+          "t1_preproc": "/Users/dorota/fmriprep_test/output1/fmriprep/sub-01/anat/sub-01_T1w_preproc.nii.gz",
+          "t1_2_fsnative_forward_transform": "/Users/dorota/fmriprep_test/workdir1/fmriprep_wf/single_subject_01_wf/anat_preproc_wf/surface_recon_wf/t1_2_fsnative_xfm/out.lta",
+          "subjects_dir": "/Users/dorota/fmriprep_test/fmriprep_test/output1/freesurfer/"
 }
 
-# wf = Workflow(name, mem_gb_node=DEFAULT_MEMORY_MIN_GB,
-#               inputs=['source_file', 't1_preproc', 'subject_id',
-#                       'subjects_dir', 't1_2_fsnative_forward_transform',
-#                       'mem_gb', 'output_spaces', 'medial_surface_nan'],
-#               outputs='surfaces')
-#
-#dj: why do I need outputs?
-wf = NewWorkflow(name=Name, mem_gb_node=DEFAULT_MEMORY_MIN_GB, inputs=Inputs)
-
-
-# @interface
-# def select_target(subject_id, space):
-#     """ Given a source subject ID and a target space, get the target subject ID """
-#     return subject_id if space == 'fsnative' else space
-
-# TODO: shouldn't map with subject?
-def select_target(subject_id, space):
-    """ Given a source subject ID and a target space, get the target subject ID """
-    return subject_id if space == 'fsnative' else space
-
-#select_target_interface = Function_Interface(select_target, ["out"])
-
-
-# wf.add('targets', select_target(subject_id=wf.inputs.subject_id))
-#   .map('space', space=[space for space in wf.inputs.output_spaces
-#                        if space.startswith('fs')])
-
-#dj: don't have option in map to connect with wf input
-
-wf.add(runnable=select_target, name="targets", subject_id="subject_id")\
-    .map(mapper="space", inputs={"space": [space for space in Inputs["output_spaces"]
-                                           if space.startswith("fs")]})
-
-
-# wf.add('rename_src', Rename(format_string='%(subject)s',
-#                             keep_ext=True,
-#                             in_file=wf.inputs.source_file))
-#   .map('subject')
-
-
-wf.add(name='rename_src',
-       runnable=Rename(format_string='%(subject)s', keep_ext=True),
-                            in_file="source_file", subject="subject_id")\
-    .map('subject')
-
-
-# wf.add('resampling_xfm',
-#        fs.utils.LTAConvert(in_lta='identity.nofile',
-#                            out_lta=True,
-#                            source_file=wf.inputs.source_file,
-#                            target_file=wf.inputs.t1_preproc)
-#   .add('set_xfm_source', ConcatenateLTA(out_type='RAS2RAS',
-#                                         in_lta2=wf.inputs.t1_2_fsnative_forward_transform,
-#                                         in_lta1=wf.resampling_xfm.out_lta))
-
-
-wf.add(name='resampling_xfm',
-       runnable=fs.utils.LTAConvert(in_lta='identity.nofile', out_lta=True),
-       source_file="source_file", target_file="t1_preproc")\
-    .add(name='set_xfm_source', runnable=ConcatenateLTA(out_type='RAS2RAS'),
-         in_lta2="t1_2_fsnative_forward_transform", in_lta1="resampling_xfm.out_lta")
-
-
-# wf.add('sampler',
-#        fs.SampleToSurface(sampling_method='average', sampling_range=(0, 1, 0.2),
-#                           sampling_units='frac', interp_method='trilinear',
-#                           cortex_mask=True, override_reg_subj=True,
-#                           out_type='gii',
-#                           subjects_dir=wf.inputs.subjects_dir,
-#                           subject_id=wf.inputs.subject_id,
-#                           reg_file=wf.set_xfm_source.out_file,
-#                           target_subject=wf.targets.out,
-#                           source_file=wf.rename_src.out_file),
-#         mem_gb=mem_gb * 3)
-#        .map([('source_file', 'target_subject'), 'hemi'], hemi=['lh', 'rh'])
-
-
-wf.add(name='sampler',
-       runnable=fs.SampleToSurface(sampling_method='average', sampling_range=(0, 1, 0.2),
-                              sampling_units='frac', interp_method='trilinear',
-                              cortex_mask=True, override_reg_subj=True,
-                              out_type='gii'),
-       mem_gb=DEFAULT_MEMORY_MIN_GB * 3,
-       subjects_dir="subjects_dir", subject_id="subject_id", reg_file="set_xfm_source.out_file",
-       target_subject="targets.out", source_file="rename_src.out_file")\
-    .map(mapper=[('source_file', 'target_subject'), 'hemi'], inputs={"hemi": ['lh', 'rh']})
-
-
-# dj: no conditions
-# dj: no join for now
-
-# wf.add_cond('cond1',
-#             condition=wf.inputs.medial_surface_nan,
-#             iftrue=wf.add('medial_nans', MedialNaNs(subjects_dir=wf.inputs.subjects_dir,
-#                                              in_file=wf.sampler.out_file,
-#                                              target_subject=wf.targets.out))
-#                      .set_output('out', wf.median_nans.out),
-#             elseclause=wf.set_output('out', wf.sampler.out_file))
-#
-# wf.add('merger', niu.Merge(1, ravel_inputs=True,
-#                            in1=wf.cond1.out),
-#             run_without_submitting=True)
-#       .join('sampler.hemi')
-#
-# wf.add('update_metadata',
-#        GiftiSetAnatomicalStructure(in_file=wf.merger.out))
-# wf.outputs.surfaces = wf.update_metadata.out_file
\ No newline at end of file
+def test_neuro():
+
+    # wf = Workflow(name, mem_gb_node=DEFAULT_MEMORY_MIN_GB,
+    #               inputs=['source_file', 't1_preproc', 'subject_id',
+    #                       'subjects_dir', 't1_2_fsnative_forward_transform',
+    #                       'mem_gb', 'output_spaces', 'medial_surface_nan'],
+    #               outputs='surfaces')
+    #
+    #dj: why do I need outputs?
+
+    pdb.set_trace()
+    wf = NewWorkflow(name=Name, inputs=Inputs, workingdir="test_neuro")
+    pdb.set_trace()
+
+
+    # @interface
+    # def select_target(subject_id, space):
+    #     """ Given a source subject ID and a target space, get the target subject ID """
+    #     return subject_id if space == 'fsnative' else space
+
+    # TODO: shouldn't map with subject?
+    def select_target(subject_id, space):
+        """ Given a source subject ID and a target space, get the target subject ID """
+        return subject_id if space == 'fsnative' else space
+
+    #select_target_interface = Function_Interface(select_target, ["out"])
+
+
+    # wf.add('targets', select_target(subject_id=wf.inputs.subject_id))
+    #   .map('space', space=[space for space in wf.inputs.output_spaces
+    #                        if space.startswith('fs')])
+
+    #dj: don't have option in map to connect with wf input
+
+    wf.add(runnable=select_target, name="targets", subject_id="subject_id")\
+        .map(mapper="space", inputs={"space": [space for space in Inputs["output_spaces"]
+                                               if space.startswith("fs")]})
+
+
+    # wf.add('rename_src', Rename(format_string='%(subject)s',
+    #                             keep_ext=True,
+    #                             in_file=wf.inputs.source_file))
+    #   .map('subject')
+
+    pdb.set_trace()
+    wf.add(name='rename_src',
+           runnable=Rename(format_string='%(subject)s', keep_ext=True),
+                                in_file="source_file", subject="subject_id")\
+        .map('subject')
+    pdb.set_trace()
+
+    # wf.add('resampling_xfm',
+    #        fs.utils.LTAConvert(in_lta='identity.nofile',
+    #                            out_lta=True,
+    #                            source_file=wf.inputs.source_file,
+    #                            target_file=wf.inputs.t1_preproc)
+    #   .add('set_xfm_source', ConcatenateLTA(out_type='RAS2RAS',
+    #                                         in_lta2=wf.inputs.t1_2_fsnative_forward_transform,
+    #                                         in_lta1=wf.resampling_xfm.out_lta))
+
+
+    wf.add(name='resampling_xfm',
+           runnable=fs.utils.LTAConvert(in_lta='identity.nofile', out_lta=True),
+           source_file="source_file", target_file="t1_preproc")\
+        .add(name='set_xfm_source', runnable=ConcatenateLTA(out_type='RAS2RAS'),
+             in_lta2="t1_2_fsnative_forward_transform", in_lta1="resampling_xfm.out_lta")
+
+
+    # wf.add('sampler',
+    #        fs.SampleToSurface(sampling_method='average', sampling_range=(0, 1, 0.2),
+    #                           sampling_units='frac', interp_method='trilinear',
+    #                           cortex_mask=True, override_reg_subj=True,
+    #                           out_type='gii',
+    #                           subjects_dir=wf.inputs.subjects_dir,
+    #                           subject_id=wf.inputs.subject_id,
+    #                           reg_file=wf.set_xfm_source.out_file,
+    #                           target_subject=wf.targets.out,
+    #                           source_file=wf.rename_src.out_file),
+    #         mem_gb=mem_gb * 3)
+    #        .map([('source_file', 'target_subject'), 'hemi'], hemi=['lh', 'rh'])
+
+
+    wf.add(name='sampler',
+           runnable=fs.SampleToSurface(sampling_method='average', sampling_range=(0, 1, 0.2),
+                                  sampling_units='frac', interp_method='trilinear',
+                                  cortex_mask=True, override_reg_subj=True,
+                                  out_type='gii'),
+           subjects_dir="subjects_dir", subject_id="subject_id", reg_file="set_xfm_source.out_file",
+           target_subject="targets.out", source_file="rename_src.out_file")\
+        .map(mapper=[('source_file', 'target_subject'), 'hemi'], inputs={"hemi": ['lh', 'rh']})
+
+
+    # dj: no conditions
+    # dj: no join for now
+
+    # wf.add_cond('cond1',
+    #             condition=wf.inputs.medial_surface_nan,
+    #             iftrue=wf.add('medial_nans', MedialNaNs(subjects_dir=wf.inputs.subjects_dir,
+    #                                              in_file=wf.sampler.out_file,
+    #                                              target_subject=wf.targets.out))
+    #                      .set_output('out', wf.median_nans.out),
+    #             elseclause=wf.set_output('out', wf.sampler.out_file))
+    #
+    # wf.add('merger', niu.Merge(1, ravel_inputs=True,
+    #                            in1=wf.cond1.out),
+    #             run_without_submitting=True)
+    #       .join('sampler.hemi')
+    #
+    # wf.add('update_metadata',
+    #        GiftiSetAnatomicalStructure(in_file=wf.merger.out))
+    # wf.outputs.surfaces = wf.update_metadata.out_file
\ No newline at end of file

From 1c48416f754a8ae3342633e0ad900f0131ba6ffa Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 3 Oct 2018 17:41:38 -0400
Subject: [PATCH 44/55] introducing workflows that can be treated as a node:
 adding results, checking inputs/outputs, changing submitter; finishing mapper
 for workflow: adding inner nodes, parent workflow is used to collect results,
 changing submitter; reorganization of SubmitterWorkflow (still have both:
 SubmitterNode and SubmitterWorkflow)

---
 nipype/pipeline/engine/submitter.py          |  96 ++++++----
 nipype/pipeline/engine/tests/test_newnode.py | 179 ++++++++++++++++++-
 nipype/pipeline/engine/workflows.py          | 135 +++++++++-----
 3 files changed, 326 insertions(+), 84 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 9bc5273ad5..fceb540f94 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -29,11 +29,11 @@ def __init__(self, plugin):
             raise Exception("plugin {} not available".format(self.plugin))
 
 
-    def submit_work(self, node):
+    def submit_node(self, node):
         for (i, ind) in enumerate(node.state.index_generator):
-            self._submit_work_el(node, i, ind)
+            self._submit_node_el(node, i, ind)
 
-    def _submit_work_el(self, node, i, ind):
+    def _submit_node_el(self, node, i, ind):
         logger.debug("SUBMIT WORKER, node: {}, ind: {}".format(node, ind))
         print("SUBMIT WORK", node.inputs)
         self.worker.run_el(node.run_interface_el, (i, ind))
@@ -50,7 +50,7 @@ def __init__(self, plugin, node):
         self.node = node
 
     def run_node(self):
-        self.submit_work(self.node)
+        self.submit_node(self.node)
         while not self.node.finished_all:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
@@ -65,23 +65,27 @@ def __init__(self, workflow, plugin):
         self._to_finish = []
 
 
-    def run_workflow(self):
-        if self.workflow.mapper:
-            self.workflow.inner_nodes = {}
-            for key in self.workflow._node_names.keys():
-                self.workflow.inner_nodes[key] = []
-            for (i, ind) in enumerate(self.workflow.state.index_generator):
-                print("LOOP", i, self._to_finish, self.node_line)
-                wf_inputs = self.workflow.state.state_values(ind)
-                new_workflow = deepcopy(self.workflow)
-                #pdb.set_trace()
-                new_workflow.preparing(wf_inputs=wf_inputs)
-                #pdb.set_trace()
-                self.run_workflow_el(workflow=new_workflow)
-                print("LOOP END", i, self._to_finish, self.node_line)
+    def run_workflow(self, workflow=None, ready=True):
+        if not workflow:
+            workflow = self.workflow
+
+        # TODO: should I havve inner_nodes for all workflow (to avoid if wf.mapper)??
+        if workflow.mapper:
+            for key in workflow._node_names.keys():
+                workflow.inner_nodes[key] = []
+            for (i, ind) in enumerate(workflow.state.index_generator):
+                new_workflow = deepcopy(workflow)
+                new_workflow.parent_wf = workflow
+                if ready:
+                    self.run_workflow_el(new_workflow, i, ind)
+                else:
+                    self.node_line.append((new_workflow, i, ind))
         else:
-            self.workflow.preparing(wf_inputs=self.workflow.inputs)
-            self.run_workflow_el(workflow=self.workflow)
+            if ready:
+                workflow.preparing(wf_inputs=workflow.inputs)
+                self.run_workflow_nd(workflow=workflow)
+            else:
+                self.node_line.append((workflow, 0, ()))
 
         # this parts submits nodes that are waiting to be run
         # it should stop when nothing is waiting
@@ -94,21 +98,33 @@ def run_workflow(self):
             logger.debug("Submitter, in while, to_finish: {}".format(self._to_finish))
             time.sleep(3)
 
+    def run_workflow_el(self, workflow, i, ind, collect_inp=False):
+        print("LOOP", i, self._to_finish, self.node_line)
+        # TODO: can I simplify and remove collect inp? where should it be?
+        if collect_inp:
+            st_inputs, wf_inputs = workflow._collecting_input_el(ind)
+        else:
+            wf_inputs = workflow.state.state_values(ind)
+        workflow.preparing(wf_inputs=wf_inputs)
+        self.run_workflow_nd(workflow=workflow)
+        print("LOOP END", i, self._to_finish, self.node_line)
 
-    def run_workflow_el(self, workflow):
-        #pdb.set_trace()
+    def run_workflow_nd(self, workflow):
         for (i_n, node) in enumerate(workflow.graph_sorted):
-            if self.workflow.mapper:
-                self.workflow.inner_nodes[node.name].append(node)
-            node.prepare_state_input()
+            if workflow.parent_wf and workflow.parent_wf.mapper: # for now if parent_wf, parent_wf has to have mapper
+                workflow.parent_wf.inner_nodes[node.name].append(node)
 
+            node.prepare_state_input()
 
             print("RUN WF", node.name, node.inputs)
             self._to_finish.append(node)
             # submitting all the nodes who are self sufficient (self.workflow.graph is already sorted)
             if node.ready2run:
+                if hasattr(node, 'nodedir'):
+                    self.submit_node(node)
+                else:  # it's workflow
+                    self.run_workflow(workflow=node)
 
-                self.submit_work(node)
             # if its not, its been added to a line
             else:
                 break
@@ -117,12 +133,16 @@ def run_workflow_el(self, workflow):
             if i_n == len(workflow.graph_sorted) - 1:
                 i_n += 1
 
+
         # all nodes that are not self sufficient (not ready to run) will go to the line
         # iterating over all elements
         for nn in list(workflow.graph_sorted)[i_n:]:
-            for (i, ind) in enumerate(nn.state.index_generator):
-                self._to_finish.append(nn)
-                self.node_line.append((nn, i, ind))
+            if hasattr(nn, 'nodedir'):
+                for (i, ind) in enumerate(nn.state.index_generator):
+                    self._to_finish.append(nn)
+                    self.node_line.append((nn, i, ind))
+            else: #wf
+                self.run_workflow(workflow=nn, ready=False)
 
 
     # for now without callback, so checking all nodes (with ind) in some order
@@ -130,11 +150,19 @@ def _nodes_check(self):
         print("NODES CHECK BEG", self.node_line)
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
-            if to_node.checking_input_el(ind):
-                self._submit_work_el(to_node, i, ind)
-                _to_remove.append((to_node, i, ind))
-            else:
-                pass
+            if hasattr(to_node, 'nodedir'):
+                if to_node.checking_input_el(ind):
+                    self._submit_node_el(to_node, i, ind)
+                    _to_remove.append((to_node, i, ind))
+                else:
+                    pass
+            else: #wf
+                if to_node.checking_input_el(ind):
+                    self.run_workflow_el(workflow=to_node, i=i, ind=ind, collect_inp=True)
+                    _to_remove.append((to_node, i, ind))
+                else:
+                    pass
+
         # can't remove during iterating
         for rn in _to_remove:
             self.node_line.remove(rn)
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 76d7efe4a8..0b7746436b 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -874,6 +874,7 @@ def test_workflow_12(plugin):
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.result["NA_out"].sort(key=lambda t: [t[0][key] for key in key_sort])
     #pdb.set_trace()
+    wf.finished_all
     for i, res in enumerate(expected):
         assert wf.result["NA_out"][i][0] == res[0]
         assert wf.result["NA_out"][i][1] == res[1]
@@ -925,6 +926,7 @@ def test_workflow_13(plugin):
     wf.connect_workflow("NA", "wfa", "a")
     wf.run(plugin=plugin)
 
+    assert wf.finished_all
     expected = [({"wf13.wfa": 3}, [({"NA.a": 3}, 5)]),
                 ({'wf13.wfa': 5}, [({"NA.a": 5}, 7)])]
     for i, res in enumerate(expected):
@@ -945,10 +947,185 @@ def test_workflow_13a(plugin):
     wf.connect_workflow("NA", "wfa", "a")
     wf.run(plugin=plugin)
 
+    assert wf.finished_all
     expected = [({"wf13a.wfa": 3}, [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)]),
                 ({'wf13a.wfa': 5}, [({"NA.a": 5, "NA.b": 10}, 15), ({"NA.a": 5, "NA.b": 20}, 25)])]
     for i, res in enumerate(expected):
         assert wf.result["NA_out"][i][0] == res[0]
         for j in range(len(res[1])):
             assert wf.result["NA_out"][i][1][j][0] == res[1][j][0]
-            assert wf.result["NA_out"][i][1][j][1] == res[1][j][1]
\ No newline at end of file
+            assert wf.result["NA_out"][i][1][j][1] == res[1][j][1]
+
+
+# workflow as a node
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_14(plugin):
+    """workflow with a workflow as a node"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na", inputs={"a": 3})
+    wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
+                      outputs_nm=[("NA", "out", "NA_out")])
+    wfa.add(na)
+
+    wf = NewWorkflow(name="wf14", workingdir="test_wf14_{}".format(plugin),
+                     outputs_nm=[("wfa", "NA_out", "wfa_out")])
+    wf.add(wfa)
+    wf.run(plugin=plugin)
+
+    assert wf.finished_all
+    expected = [({"NA.a": 3}, 5)]
+    for i, res in enumerate(expected):
+        assert wf.result["wfa_out"][i][0] == res[0]
+        assert wf.result["wfa_out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_14a(plugin):
+    """workflow with a workflow as a node (using connect_workflow in wfa)"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    wfa = NewWorkflow(name="wfa", workingdir="test_wfa", inputs={"a": 3},
+                      outputs_nm=[("NA", "out", "NA_out")])
+    wfa.add(na)
+    wfa.connect_workflow("NA", "a", "a")
+
+    wf = NewWorkflow(name="wf14a", workingdir="test_wf14a_{}".format(plugin),
+                     outputs_nm=[("wfa", "NA_out", "wfa_out")])
+    wf.add(wfa)
+    wf.run(plugin=plugin)
+
+    assert wf.finished_all
+    expected = [({"NA.a": 3}, 5)]
+    for i, res in enumerate(expected):
+        assert wf.result["wfa_out"][i][0] == res[0]
+        assert wf.result["wfa_out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_14b(plugin):
+    """workflow with a workflow as a node (using connect_workflow in wfa and wf)"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
+                      outputs_nm=[("NA", "out", "NA_out")])
+    wfa.add(na)
+    wfa.connect_workflow("NA", "a", "a")
+
+    wf = NewWorkflow(name="wf14b", workingdir="test_wf14b_{}".format(plugin),
+                     outputs_nm=[("wfa", "NA_out", "wfa_out")], inputs={"a": 3})
+    wf.add(wfa)
+    wf.connect_workflow("wfa", "a", "a")
+    wf.run(plugin=plugin)
+
+    assert wf.finished_all
+    expected = [({"NA.a": 3}, 5)]
+    for i, res in enumerate(expected):
+        assert wf.result["wfa_out"][i][0] == res[0]
+        assert wf.result["wfa_out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_15(plugin):
+    """workflow with a workflow as a node with mapper (like 14 but with mapper)"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na",
+                 inputs={"a": [3, 5]}, mapper="a")
+    wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
+                      outputs_nm=[("NA", "out", "NA_out")])
+    wfa.add(na)
+
+    wf = NewWorkflow(name="wf15", workingdir="test_wf15_{}".format(plugin),
+                     outputs_nm=[("wfa", "NA_out", "wfa_out")])
+    wf.add(wfa)
+    wf.run(plugin=plugin)
+    assert wf.finished_all
+    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
+    for i, res in enumerate(expected):
+        assert wf.result["wfa_out"][i][0] == res[0]
+        assert wf.result["wfa_out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_16(plugin):
+    """workflow with two nodes, second node without mapper"""
+    wf = NewWorkflow(name="wf16", workingdir="test_wf16_{}".format(plugin),
+                     outputs_nm=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na", inputs={"a": 3})
+    wf.add(na)
+
+    # the second node does not have explicit mapper (but keeps the mapper from the NA node)
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
+                      outputs_nm=[("NB", "out", "NB_out")])
+    wfb.add(nb)
+    wfb.connect_workflow("NB", "b", "b")
+    wfb.connect_workflow("NB", "a", "a")
+
+    wf.add(wfb)
+    wf.connect("NA", "out", "wfb", "a")
+    wf.run(plugin=plugin)
+
+    assert wf.finished_all
+    expected_A = [({"NA.a": 3}, 5)]
+    for i, res in enumerate(expected_A):
+        assert wf.result["NA_out"][i][0] == res[0]
+        assert wf.result["NA_out"][i][1] == res[1]
+
+    # TODO: the naming rememebrs only the node, doesnt remember that a came from NA...
+    # the naming should have names with workflows??
+    expected_B = [({"NB.a": 5, "NB.b": 10}, 15)]
+    for i, res in enumerate(expected_B):
+        assert wf.result["NB_out"][i][0] == res[0]
+        assert wf.result["NB_out"][i][1] == res[1]
+
+
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_16a(plugin):
+    """workflow with two nodes, second node without mapper"""
+    wf = NewWorkflow(name="wf16a", workingdir="test_wf16a_{}".format(plugin),
+                     outputs_nm=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na.map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(na)
+
+    # the second node does not have explicit mapper (but keeps the mapper from the NA node)
+    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
+                      outputs_nm=[("NB", "out", "NB_out")])
+    wfb.add(nb)
+    wfb.connect_workflow("NB", "b", "b")
+    wfb.connect_workflow("NB", "a", "a")
+
+    # adding 2 nodes and create a connection (as it is now)
+    wf.add(wfb)
+    wf.connect("NA", "out", "wfb", "a")
+    assert wf.nodes[0].mapper == "NA.a"
+    wf.run(plugin=plugin)
+
+    assert wf.finished_all
+
+    expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
+    for i, res in enumerate(expected_A):
+        assert wf.result["NA_out"][i][0] == res[0]
+        assert wf.result["NA_out"][i][1] == res[1]
+
+    # TODO: the naming rememebrs only the node, doesnt remember that a came from NA...
+    # the naming should have names with workflows??
+    expected_B = [({"NB.a": 5, "NB.b": 10}, 15), ({"NB.a": 7, "NB.b": 10}, 17)]
+    key_sort = list(expected_B[0][0].keys())
+    expected_B.sort(key=lambda t: [t[0][key] for key in key_sort])
+    wf.result["NB_out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+    for i, res in enumerate(expected_B):
+        assert wf.result["NB_out"][i][0] == res[0]
+        assert wf.result["NB_out"][i][1] == res[1]
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index a8fe3a98f8..061cf0de69 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1096,6 +1096,12 @@ def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb_no
         self._state = state.State(mapper=self._mapper, node_name=self.name, other_mappers=self._other_mappers)
         self._output = {}
         self._result = {}
+        # flag that says if the node/wf is ready to run (has all input)
+        self.ready2run = True
+        # needed outputs from other nodes if the node part of a wf
+        self.needed_outputs = []
+        # flag that says if node finished all jobs
+        self._finished_all = False
 
 
     # TBD
@@ -1140,6 +1146,40 @@ def prepare_state_input(self):
         self._state.prepare_state_input(state_inputs=self.state_inputs)
 
 
+    # dj: this is not used for a single node
+    def _collecting_input_el(self, ind):
+        """collecting all inputs required to run the node (for specific state element)"""
+        state_dict = self.state.state_values(ind)
+        inputs_dict = {k: state_dict[k] for k in self._inputs.keys()}
+        # reading extra inputs that come from previous nodes
+        for (from_node, from_socket, to_socket) in self.needed_outputs:
+            dir_nm_el_from = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())
+                                       if i in list(from_node._state_inputs.keys())])
+            file_from = os.path.join(from_node.nodedir, dir_nm_el_from, from_socket+".txt")
+            with open(file_from) as f:
+                inputs_dict["{}.{}".format(self.name, to_socket)] = eval(f.readline())
+        return state_dict, inputs_dict
+
+
+    def checking_input_el(self, ind):
+        """checking if all inputs are available (for specific state element)"""
+        try:
+            self._collecting_input_el(ind)
+            return True
+        except: #TODO specify
+            return False
+
+   # checking if all outputs are saved
+    @property
+    def finished_all(self):
+        # once _finished_all os True, this should not change
+        logger.debug('finished_all {}'.format(self._finished_all))
+        if self._finished_all:
+            return self._finished_all
+        else:
+            return self._check_all_results()
+
+
 
 class NewNode(NewBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
@@ -1154,14 +1194,8 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
         # adding node name to the interface's name mapping
         self.interface.input_map = dict((key, "{}.{}".format(self.name, value))
                                          for (key, value) in self.interface.input_map.items())
-        # needed outputs from other nodes if the node part of a wf
-        self.needed_outputs = []
         # output names taken from interface output name
         self.output_names = self.interface._output_nm
-        # flag that says if node finished all jobs
-        self._finished_all = False
-        # flag that says if the node is ready to run (has all input)
-        self.ready2run = True
 
 
     # dj: not sure if I need it
@@ -1257,7 +1291,6 @@ def _writting_results_tmp(self, state_dict, dir_nm_el, output):
     def _collecting_output(self):
         for key_out in self.output_names:
             self._output[key_out] = {}
-            #pdb.set_trace()
             for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
                 state_dict, inputs_dict = self._collecting_input_el(ind)
                 dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
@@ -1266,42 +1299,6 @@ def _collecting_output(self):
         return self._output
 
 
-    # dj: this is not used for a single node
-    def _collecting_input_el(self, ind):
-        """collecting all inputs required to run the node (for specific state element)"""
-        state_dict = self.state.state_values(ind)
-        inputs_dict = {k: state_dict[k] for k in self._inputs.keys()}
-        # reading extra inputs that come from previous nodes
-        for (from_node, from_socket, to_socket) in self.needed_outputs:
-            dir_nm_el_from = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())
-                                       if i in list(from_node._state_inputs.keys())])
-            file_from = os.path.join(from_node.nodedir, dir_nm_el_from, from_socket+".txt")
-            with open(file_from) as f:
-                inputs_dict["{}.{}".format(self.name, to_socket)] = eval(f.readline())
-        return state_dict, inputs_dict
-
-
-    def checking_input_el(self, ind):
-        """checking if all inputs are available (for specific state element)"""
-        try:
-            self._collecting_input_el(ind)
-            return True
-        except: #TODO specify
-            return False
-
-
-    # checking if all outputs are saved
-    @property
-    def finished_all(self):
-        # once _finished_all os True, this should not change
-        print("in finished all", self.inputs, self.nodedir)
-        logger.debug('finished_all {}'.format(self._finished_all))
-        if self._finished_all:
-            return self._finished_all
-        else:
-            return self._check_all_results()
-
-
     # dj: version without join
     def _check_all_results(self):
         """checking if all files that should be created are present"""
@@ -1364,6 +1361,10 @@ def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=Non
         # list of (nodename, output name in the name, output name in wf) or (nodename, output name in the name)
         self.outputs_nm = outputs_nm
 
+        # nodes that are created when the workflow has mapper (key: node name, value: list of nodes)
+        self.inner_nodes = {}
+        # in case of inner workflow this points to the main/parent workflow
+        self.parent_wf = None
         # dj not sure what was the motivation, wf_klasses gives an empty list
         #mro = self.__class__.mro()
         #wf_klasses = mro[:mro.index(NewWorkflow)][::-1]
@@ -1389,6 +1390,10 @@ def inputs(self, inputs):
     def nodes(self):
         return self._nodes
 
+    @property
+    def graph_sorted(self):
+        # TODO: should I always update the graph?
+        return list(nx.topological_sort(self.graph))
 
     def add_nodes(self, nodes):
         """adding nodes without defining connections"""
@@ -1437,13 +1442,22 @@ def preparing(self, wf_inputs=None):
                 node.inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
             else:
                 raise Exception("{}.{} not in the workflow inputs".format(self.name, inp_wf))
+        # TODO if should be later, should unify more workingdir and nodedir
         for nn in self.graph_sorted:
             if self.mapper:
                 dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
-                nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.nodedir)
+                if is_node(nn):
+                    # TODO: should be just nn.name?
+                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.nodedir)
+                elif is_workflow(nn):
+                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.name)
                 nn._finished_all = False # helps when mp is used
             else:
-                nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
+                if is_node(nn):
+                    #TODO: should be just nn.name?
+                    nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
+                elif is_workflow(nn):
+                    nn.workingdir = os.path.join(self.workingdir, nn.name)
             try:
                 for inp, (out_node, out_var) in self.connected_var[nn].items():
                     nn.ready2run = False #it has some history (doesnt have to be in the loop)
@@ -1463,17 +1477,16 @@ def preparing(self, wf_inputs=None):
 
 
     def run(self, plugin="serial"):
-        self.graph_sorted = list(nx.topological_sort(self.graph))
         #self.preparing(wf_inputs=self.inputs) # moved to submitter
         self.prepare_state_input()
         logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
         submitter = sub.SubmitterWorkflow(workflow=self, plugin=plugin)
         submitter.run_workflow()
         submitter.close()
-        self._collecting_outpt()
+        self._collecting_output()
 
 
-    def _collecting_outpt(self):
+    def _collecting_output(self):
         # not sure, if I should collecto output of all nodes or only the ones that are used in wf.output
         self.node_outputs = {}
         for nn in self.graph:
@@ -1500,6 +1513,7 @@ def _collecting_outpt(self):
                         self._output[out_wf_nm] = self.node_outputs[node_nm][out_nd_nm]
                 else:
                     raise Exception("the key {} is already used in workflow.result".format(out_wf_nm))
+        return self._output
 
 
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
@@ -1523,6 +1537,8 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
                            other_mappers=self._node_mappers, mem_gb_node=mem_gb)
         elif is_node(runnable):
             node = runnable
+        elif is_workflow(runnable):
+            node = runnable
         else:
             raise ValueError("Unknown workflow element: {!r}".format(runnable))
         self.add_nodes([node])
@@ -1554,6 +1570,7 @@ def join(self, field, node=None):
         pass
 
 
+    # TODO: should try to merge with the function from Node
     def _reading_results(self):
         if self.outputs_nm:
             for out in self.outputs_nm:
@@ -1571,19 +1588,39 @@ def _reading_results(self):
                         self._result[key_out].append((wf_inputs_dict, res_l))
                 else:
                     for key, val in self.output[key_out].items():
+                        #TODO: I think that val shouldn't be dict here...
+                        # TMP solution
+                        if type(val) is dict:
+                            val = [v for k,v in val.items()][0]
                         with open(val[1]) as fout:
                             logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
                             self._result[key_out].append((val[0], eval(fout.readline())))
 
 
+    # dj: version without join
+    # TODO: might merge with the function from Node
+    def _check_all_results(self):
+        """checking if all files that should be created are present"""
+        for nn in self.graph_sorted:
+            if nn.name in self.inner_nodes.keys():
+                if not all([ni.finished_all for ni in self.inner_nodes[nn.name]]):
+                    return False
+            else:
+                if not nn.finished_all:
+                    return False
+        self._finished_all = True
+        return True
+
 
 def is_function(obj):
     return hasattr(obj, '__call__')
 
-
 def is_interface(obj):
     return type(obj) is aux.Function_Interface
 
 
 def is_node(obj):
     return type(obj) is NewNode
+
+def is_workflow(obj):
+    return type(obj) is NewWorkflow

From ef2b4cf4d14fee065223b410e85416b6fd3f0255 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 3 Oct 2018 22:26:11 -0400
Subject: [PATCH 45/55] moving run node/wf to tests and removing run from
 Node/Workflow (at least temporarily)

---
 nipype/pipeline/engine/submitter.py          |   8 +
 nipype/pipeline/engine/tests/test_newnode.py | 178 ++++++++++++++-----
 nipype/pipeline/engine/workflows.py          |  34 ++--
 3 files changed, 162 insertions(+), 58 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index fceb540f94..2f1f243e2b 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -50,10 +50,12 @@ def __init__(self, plugin, node):
         self.node = node
 
     def run_node(self):
+        self.node.prepare_state_input()
         self.submit_node(self.node)
         while not self.node.finished_all:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
+        self.node._collecting_output()
 
 
 class SubmitterWorkflow(Submitter):
@@ -68,6 +70,7 @@ def __init__(self, workflow, plugin):
     def run_workflow(self, workflow=None, ready=True):
         if not workflow:
             workflow = self.workflow
+        workflow.prepare_state_input()
 
         # TODO: should I havve inner_nodes for all workflow (to avoid if wf.mapper)??
         if workflow.mapper:
@@ -98,6 +101,11 @@ def run_workflow(self, workflow=None, ready=True):
             logger.debug("Submitter, in while, to_finish: {}".format(self._to_finish))
             time.sleep(3)
 
+        # calling only for the main wf (other wf will be called inside the function)
+        if workflow is self.workflow:
+            workflow._collecting_output()
+
+
     def run_workflow_el(self, workflow, i, ind, collect_inp=False):
         print("LOOP", i, self._to_finish, self.node_line)
         # TODO: can I simplify and remove collect inp? where should it be?
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 0b7746436b..6a52921d87 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1,5 +1,6 @@
 from .. import NewNode, NewWorkflow
 from ..auxiliary import Function_Interface
+from ..submitter import SubmitterNode, SubmitterWorkflow
 
 import sys, time, os
 import numpy as np
@@ -93,8 +94,9 @@ def test_node_6(plugin):
     assert nn.mapper == "NA.a"
     assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
 
-    # testing if the node runs properly
-    nn.run(plugin=plugin)
+    sub = SubmitterNode(plugin=plugin, node=nn)
+    sub.run_node()
+    sub.close()
 
     # checking the results
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -121,8 +123,9 @@ def test_node_7(plugin):
     assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
     assert (nn.inputs["NA.b"] == np.array([2, 1])).all()
 
-    # testing if the node runs properly
-    nn.run(plugin=plugin)
+    sub = SubmitterNode(plugin=plugin, node=nn)
+    sub.run_node()
+    sub.close()
 
     # checking the results
     expected = [({"NA.a": 3, "NA.b": 2}, 5), ({"NA.a": 5, "NA.b": 1}, 6)]
@@ -149,8 +152,9 @@ def test_node_8(plugin):
     assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
     assert (nn.inputs["NA.b"] == np.array([2, 1])).all()
 
-    # testing if the node runs properly
-    nn.run(plugin=plugin)
+    sub = SubmitterNode(plugin=plugin, node=nn)
+    sub.run_node()
+    sub.close()
 
     # checking teh results
     expected = [({"NA.a": 3, "NA.b": 1}, 4), ({"NA.a": 3, "NA.b": 2}, 5),
@@ -189,7 +193,10 @@ def test_workflow_1(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add_nodes([na])
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -216,9 +223,11 @@ def test_workflow_2(plugin):
     # adding 2 nodes and create a connection (as it is now)
     wf.add_nodes([na, nb])
     wf.connect("NA", "out", "NB", "a")
-
     assert wf.nodes[0].mapper == "NA.a"
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
@@ -258,7 +267,10 @@ def test_workflow_2a(plugin):
 
     assert wf.nodes[0].mapper == "NA.a"
     assert wf.nodes[1].mapper == ("NA.a", "NB.b")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
@@ -297,7 +309,10 @@ def test_workflow_2b(plugin):
 
     assert wf.nodes[0].mapper == "NA.a"
     assert wf.nodes[1].mapper == ["NA.a", "NB.b"]
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
@@ -332,7 +347,10 @@ def test_workflow_3(plugin):
     wf.add(na)
 
     assert wf.nodes[0].mapper == "NA.a"
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -354,7 +372,10 @@ def test_workflow_3a(plugin):
     wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
     assert wf.nodes[0].mapper == "NA.a"
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -374,7 +395,10 @@ def test_workflow_3b(plugin):
     wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
     assert wf.nodes[0].mapper == "NA.a"
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -407,7 +431,9 @@ def test_workflow_4(plugin):
     # connect method as it is in the current version
     wf.connect("NA", "out", "NB", "a")
 
-    wf.run(plugin=plugin)
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -443,7 +469,9 @@ def test_workflow_4a(plugin):
     # instead of "connect", using kwrg argument in the add method as in the example
     wf.add(nb, a="NA.out")
 
-    wf.run(plugin=plugin)
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -476,7 +504,10 @@ def test_workflow_5(plugin):
     wf.add(na)
     # using the map method after add (using mapper for the last added node as default)
     wf.map(mapper="a", inputs={"a": [3, 5]})
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -496,7 +527,10 @@ def test_workflow_5a(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
     wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -523,7 +557,10 @@ def test_workflow_6(plugin):
     wf.add(nb)
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -559,7 +596,10 @@ def test_workflow_6a(plugin):
     # TODO: should we se ("a", "c") instead?? shold I forget "NA.a" value?
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
     wf.connect("NA", "out", "NB", "a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -593,7 +633,10 @@ def test_workflow_6b(plugin):
     wf.add(nb, a="NA.out")
     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -627,7 +670,10 @@ def test_workflow_7(plugin):
     # connecting the node with inputs from the workflow
     wf.connect_workflow("NA", "wfa", "a")
     wf.map(mapper="a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -651,7 +697,10 @@ def test_workflow_7a(plugin):
     # if connect has None as the first arg, it is the same as connect_workflow
     wf.connect(None, "wfa", "NA", "a")
     wf.map(mapper="a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -672,7 +721,10 @@ def test_workflow_7b(plugin):
     # using kwrg argument in the add method (instead of connect or connect_workflow
     wf.add(na, a="wfa")
     wf.map(mapper="a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -699,7 +751,10 @@ def test_workflow_8(plugin):
     wf.connect("NA", "out", "NB", "a")
     wf.connect_workflow("NB", "b", "b")
     assert wf.nodes[0].mapper == "NA.a"
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected_A[0][0].keys())
@@ -731,7 +786,10 @@ def test_workflow_9(plugin):
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as ("NA.a", "b")
     wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     key_sort = list(expected[0][0].keys())
@@ -760,7 +818,10 @@ def test_workflow_10(plugin):
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (("NA.a", NA.b), "b")
     wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 5, "NA.b": 10}, 15)]
     key_sort = list(expected[0][0].keys())
@@ -789,7 +850,10 @@ def test_workflow_10a(plugin):
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (["NA.a", NA.b], "b")
     wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 3, "NA.b": 10}, 13),
                 ({"NA.a": 5, "NA.b": 0}, 5), ({"NA.a": 5, "NA.b": 10}, 15)]
@@ -822,7 +886,10 @@ def test_workflow_11(plugin):
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA, _NB means that I'm using mappers from the NA/NB nodes, it's the same as [("NA.a", NA.b), "NB.a"]
     wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map(mapper=["_NA", "_NB"]) # TODO: this should eb default?
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 5, "NA.b": 10}, 15)]
     key_sort = list(expected[0][0].keys())
@@ -862,7 +929,10 @@ def test_workflow_12(plugin):
     wf.add(nb)
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     # checking if workflow.results is the same as results of nodes
     assert wf.result["NA_out"] == wf.nodes[0].result["out"]
@@ -874,7 +944,7 @@ def test_workflow_12(plugin):
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.result["NA_out"].sort(key=lambda t: [t[0][key] for key in key_sort])
     #pdb.set_trace()
-    wf.finished_all
+    assert wf.finished_all
     for i, res in enumerate(expected):
         assert wf.result["NA_out"][i][0] == res[0]
         assert wf.result["NA_out"][i][1] == res[1]
@@ -906,9 +976,10 @@ def test_workflow_12a(plugin):
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
 
-    # wf_out can't be used twice in wf.result
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    # wf_out can't be used twice
     with pytest.raises(Exception) as exinfo:
-        wf.run(plugin=plugin)
+        sub.run_workflow()
     assert str(exinfo.value) == "the key wf_out is already used in workflow.result"
 
 # tests for a workflow that have its own input and mapper
@@ -924,7 +995,10 @@ def test_workflow_13(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wf.add(na)
     wf.connect_workflow("NA", "wfa", "a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     assert wf.finished_all
     expected = [({"wf13.wfa": 3}, [({"NA.a": 3}, 5)]),
@@ -945,7 +1019,10 @@ def test_workflow_13a(plugin):
     na = NewNode(name="NA", interface=interf_addvar, base_dir="na", mapper="b", inputs={"b": [10, 20]})
     wf.add(na)
     wf.connect_workflow("NA", "wfa", "a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     assert wf.finished_all
     expected = [({"wf13a.wfa": 3}, [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)]),
@@ -972,7 +1049,10 @@ def test_workflow_14(plugin):
     wf = NewWorkflow(name="wf14", workingdir="test_wf14_{}".format(plugin),
                      outputs_nm=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     assert wf.finished_all
     expected = [({"NA.a": 3}, 5)]
@@ -995,7 +1075,10 @@ def test_workflow_14a(plugin):
     wf = NewWorkflow(name="wf14a", workingdir="test_wf14a_{}".format(plugin),
                      outputs_nm=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     assert wf.finished_all
     expected = [({"NA.a": 3}, 5)]
@@ -1019,7 +1102,10 @@ def test_workflow_14b(plugin):
                      outputs_nm=[("wfa", "NA_out", "wfa_out")], inputs={"a": 3})
     wf.add(wfa)
     wf.connect_workflow("wfa", "a", "a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     assert wf.finished_all
     expected = [({"NA.a": 3}, 5)]
@@ -1042,7 +1128,11 @@ def test_workflow_15(plugin):
     wf = NewWorkflow(name="wf15", workingdir="test_wf15_{}".format(plugin),
                      outputs_nm=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
+
     assert wf.finished_all
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     for i, res in enumerate(expected):
@@ -1071,7 +1161,10 @@ def test_workflow_16(plugin):
 
     wf.add(wfb)
     wf.connect("NA", "out", "wfb", "a")
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     assert wf.finished_all
     expected_A = [({"NA.a": 3}, 5)]
@@ -1111,7 +1204,10 @@ def test_workflow_16a(plugin):
     wf.add(wfb)
     wf.connect("NA", "out", "wfb", "a")
     assert wf.nodes[0].mapper == "NA.a"
-    wf.run(plugin=plugin)
+
+    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub.run_workflow()
+    sub.close()
 
     assert wf.finished_all
 
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 061cf0de69..df3930de4d 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1328,14 +1328,14 @@ def _reading_results(self):
                 with open(filename) as fout:
                     self._result[key_out].append(({}, eval(fout.readline())))
 
-
-    def run(self, plugin="serial"):
-        """preparing the node to run and run the interface"""
-        self.prepare_state_input()
-        submitter = sub.SubmitterNode(plugin, node=self)
-        submitter.run_node()
-        submitter.close()
-        self._collecting_output()
+    # dj: removing temp. from NewNode class
+    # def run(self, plugin="serial"):
+    #     """preparing the node to run and run the interface"""
+    #     self.prepare_state_input()
+    #     submitter = sub.SubmitterNode(plugin, node=self)
+    #     submitter.run_node()
+    #     submitter.close()
+    #     self._collecting_output()
 
 
 class NewWorkflow(NewBase):
@@ -1475,15 +1475,15 @@ def preparing(self, wf_inputs=None):
 
             nn.prepare_state_input()
 
-
-    def run(self, plugin="serial"):
-        #self.preparing(wf_inputs=self.inputs) # moved to submitter
-        self.prepare_state_input()
-        logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
-        submitter = sub.SubmitterWorkflow(workflow=self, plugin=plugin)
-        submitter.run_workflow()
-        submitter.close()
-        self._collecting_output()
+    # removing temp. from NewWorkflow
+    # def run(self, plugin="serial"):
+    #     #self.preparing(wf_inputs=self.inputs) # moved to submitter
+    #     self.prepare_state_input()
+    #     logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
+    #     submitter = sub.SubmitterWorkflow(workflow=self, plugin=plugin)
+    #     submitter.run_workflow()
+    #     submitter.close()
+    #     self._collecting_output()
 
 
     def _collecting_output(self):

From 935c8ea551d7a113acd045b1f0036c65653065b7 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Wed, 3 Oct 2018 23:39:03 -0400
Subject: [PATCH 46/55] merging all submitter classes

---
 nipype/pipeline/engine/submitter.py          |  99 +++++++------
 nipype/pipeline/engine/tests/test_newnode.py | 142 +++++++++----------
 2 files changed, 120 insertions(+), 121 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 2f1f243e2b..23e142e5ed 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -1,6 +1,5 @@
 from __future__ import print_function, division, unicode_literals, absolute_import
 from builtins import object
-from collections import defaultdict
 
 from future import standard_library
 standard_library.install_aliases()
@@ -13,10 +12,13 @@
 from ... import config, logging
 logger = logging.getLogger('nipype.workflow')
 
+
 class Submitter(object):
-    def __init__(self, plugin):
+    # TODO: runnable in init or run
+    def __init__(self, plugin, runnable):
         self.plugin = plugin
         self.node_line = []
+        self._to_finish = [] # used only for wf
         if self.plugin == "mp":
             self.worker = MpWorker()
         elif self.plugin == "serial":
@@ -28,46 +30,46 @@ def __init__(self, plugin):
         else:
             raise Exception("plugin {} not available".format(self.plugin))
 
-
-    def submit_node(self, node):
-        for (i, ind) in enumerate(node.state.index_generator):
-            self._submit_node_el(node, i, ind)
-
-    def _submit_node_el(self, node, i, ind):
-        logger.debug("SUBMIT WORKER, node: {}, ind: {}".format(node, ind))
-        print("SUBMIT WORK", node.inputs)
-        self.worker.run_el(node.run_interface_el, (i, ind))
-
-
-    def close(self):
-        self.worker.close()
+        if hasattr(runnable, 'interface'): # a node
+            self.node = runnable
+        elif hasattr(runnable, "graph"): # a workflow
+            self.workflow = runnable
+        else:
+            raise Exception("runnable has to be a Node or Workflow")
 
 
+    def run(self):
+        """main running method, checks if submitter id for Node or Workflow"""
+        if hasattr(self, "node"):
+            self.run_node()
+        elif hasattr(self, "workflow"):
+            self.run_workflow()
 
-class SubmitterNode(Submitter):
-    def __init__(self, plugin, node):
-        super(SubmitterNode, self).__init__(plugin)
-        self.node = node
 
     def run_node(self):
+        """the main method to run a Node"""
         self.node.prepare_state_input()
-        self.submit_node(self.node)
+        self._submit_node(self.node)
         while not self.node.finished_all:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
         self.node._collecting_output()
 
 
-class SubmitterWorkflow(Submitter):
-    def __init__(self, workflow, plugin):
-        super(SubmitterWorkflow, self).__init__(plugin)
-        self.workflow = workflow
-        logger.debug('Initialize Submitter, graph: {}'.format(self.workflow.graph_sorted))
-        #self._to_finish = list(self.workflow.graph)
-        self._to_finish = []
+    def _submit_node(self, node):
+        """submitting nodes's interface for all states"""
+        for (i, ind) in enumerate(node.state.index_generator):
+            self._submit_node_el(node, i, ind)
+
+    def _submit_node_el(self, node, i, ind):
+        """submitting node's interface for one element of states"""
+        logger.debug("SUBMIT WORKER, node: {}, ind: {}".format(node, ind))
+        print("SUBMIT WORK", node.inputs)
+        self.worker.run_el(node.run_interface_el, (i, ind))
 
 
     def run_workflow(self, workflow=None, ready=True):
+        """the main function to run Workflow"""
         if not workflow:
             workflow = self.workflow
         workflow.prepare_state_input()
@@ -80,13 +82,13 @@ def run_workflow(self, workflow=None, ready=True):
                 new_workflow = deepcopy(workflow)
                 new_workflow.parent_wf = workflow
                 if ready:
-                    self.run_workflow_el(new_workflow, i, ind)
+                    self._run_workflow_el(new_workflow, i, ind)
                 else:
                     self.node_line.append((new_workflow, i, ind))
         else:
             if ready:
                 workflow.preparing(wf_inputs=workflow.inputs)
-                self.run_workflow_nd(workflow=workflow)
+                self._run_workflow_nd(workflow=workflow)
             else:
                 self.node_line.append((workflow, 0, ()))
 
@@ -106,33 +108,30 @@ def run_workflow(self, workflow=None, ready=True):
             workflow._collecting_output()
 
 
-    def run_workflow_el(self, workflow, i, ind, collect_inp=False):
-        print("LOOP", i, self._to_finish, self.node_line)
+    def _run_workflow_el(self, workflow, i, ind, collect_inp=False):
+        """running one internal workflow (if workflow has a mapper)"""
         # TODO: can I simplify and remove collect inp? where should it be?
         if collect_inp:
             st_inputs, wf_inputs = workflow._collecting_input_el(ind)
         else:
             wf_inputs = workflow.state.state_values(ind)
         workflow.preparing(wf_inputs=wf_inputs)
-        self.run_workflow_nd(workflow=workflow)
-        print("LOOP END", i, self._to_finish, self.node_line)
+        self._run_workflow_nd(workflow=workflow)
 
-    def run_workflow_nd(self, workflow):
+
+    def _run_workflow_nd(self, workflow):
+        """iterating over all nodes from a workflow and submitting them or adding to the node_line"""
         for (i_n, node) in enumerate(workflow.graph_sorted):
             if workflow.parent_wf and workflow.parent_wf.mapper: # for now if parent_wf, parent_wf has to have mapper
                 workflow.parent_wf.inner_nodes[node.name].append(node)
-
             node.prepare_state_input()
-
-            print("RUN WF", node.name, node.inputs)
             self._to_finish.append(node)
             # submitting all the nodes who are self sufficient (self.workflow.graph is already sorted)
             if node.ready2run:
-                if hasattr(node, 'nodedir'):
-                    self.submit_node(node)
+                if hasattr(node, 'interface'):
+                    self._submit_node(node)
                 else:  # it's workflow
                     self.run_workflow(workflow=node)
-
             # if its not, its been added to a line
             else:
                 break
@@ -141,11 +140,10 @@ def run_workflow_nd(self, workflow):
             if i_n == len(workflow.graph_sorted) - 1:
                 i_n += 1
 
-
         # all nodes that are not self sufficient (not ready to run) will go to the line
         # iterating over all elements
         for nn in list(workflow.graph_sorted)[i_n:]:
-            if hasattr(nn, 'nodedir'):
+            if hasattr(nn, 'interface'):
                 for (i, ind) in enumerate(nn.state.index_generator):
                     self._to_finish.append(nn)
                     self.node_line.append((nn, i, ind))
@@ -153,12 +151,11 @@ def run_workflow_nd(self, workflow):
                 self.run_workflow(workflow=nn, ready=False)
 
 
-    # for now without callback, so checking all nodes (with ind) in some order
     def _nodes_check(self):
-        print("NODES CHECK BEG", self.node_line)
+        """checking which nodes-states are ready to run and running the ones that are ready"""
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
-            if hasattr(to_node, 'nodedir'):
+            if hasattr(to_node, 'interface'):
                 if to_node.checking_input_el(ind):
                     self._submit_node_el(to_node, i, ind)
                     _to_remove.append((to_node, i, ind))
@@ -166,7 +163,7 @@ def _nodes_check(self):
                     pass
             else: #wf
                 if to_node.checking_input_el(ind):
-                    self.run_workflow_el(workflow=to_node, i=i, ind=ind, collect_inp=True)
+                    self._run_workflow_el(workflow=to_node, i=i, ind=ind, collect_inp=True)
                     _to_remove.append((to_node, i, ind))
                 else:
                     pass
@@ -174,19 +171,21 @@ def _nodes_check(self):
         # can't remove during iterating
         for rn in _to_remove:
             self.node_line.remove(rn)
-        print("NODES CHECK END", self.node_line)
         return self.node_line
 
 
     # this I believe can be done for entire node
     def _output_check(self):
+        """"checking if all nodes are done"""
         _to_remove = []
-        print("OUT CHECK", self._to_finish)
         for node in self._to_finish:
             print("_output check node", node, node.finished_all)
             if node.finished_all:
                 _to_remove.append(node)
         for rn in _to_remove:
             self._to_finish.remove(rn)
-        print("OUT CHECK END", self._to_finish)
-        return self._to_finish
\ No newline at end of file
+        return self._to_finish
+
+
+    def close(self):
+        self.worker.close()
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 6a52921d87..fed1690bc2 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1,6 +1,6 @@
 from .. import NewNode, NewWorkflow
 from ..auxiliary import Function_Interface
-from ..submitter import SubmitterNode, SubmitterWorkflow
+from ..submitter import Submitter
 
 import sys, time, os
 import numpy as np
@@ -94,8 +94,8 @@ def test_node_6(plugin):
     assert nn.mapper == "NA.a"
     assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
 
-    sub = SubmitterNode(plugin=plugin, node=nn)
-    sub.run_node()
+    sub = Submitter(plugin=plugin, runnable=nn)
+    sub.run()
     sub.close()
 
     # checking the results
@@ -123,8 +123,8 @@ def test_node_7(plugin):
     assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
     assert (nn.inputs["NA.b"] == np.array([2, 1])).all()
 
-    sub = SubmitterNode(plugin=plugin, node=nn)
-    sub.run_node()
+    sub = Submitter(plugin=plugin, runnable=nn)
+    sub.run()
     sub.close()
 
     # checking the results
@@ -152,8 +152,8 @@ def test_node_8(plugin):
     assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
     assert (nn.inputs["NA.b"] == np.array([2, 1])).all()
 
-    sub = SubmitterNode(plugin=plugin, node=nn)
-    sub.run_node()
+    sub = Submitter(plugin=plugin, runnable=nn)
+    sub.run()
     sub.close()
 
     # checking teh results
@@ -194,8 +194,8 @@ def test_workflow_1(plugin):
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add_nodes([na])
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -225,8 +225,8 @@ def test_workflow_2(plugin):
     wf.connect("NA", "out", "NB", "a")
     assert wf.nodes[0].mapper == "NA.a"
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -268,8 +268,8 @@ def test_workflow_2a(plugin):
     assert wf.nodes[0].mapper == "NA.a"
     assert wf.nodes[1].mapper == ("NA.a", "NB.b")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -310,8 +310,8 @@ def test_workflow_2b(plugin):
     assert wf.nodes[0].mapper == "NA.a"
     assert wf.nodes[1].mapper == ["NA.a", "NB.b"]
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -348,8 +348,8 @@ def test_workflow_3(plugin):
 
     assert wf.nodes[0].mapper == "NA.a"
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -373,8 +373,8 @@ def test_workflow_3a(plugin):
 
     assert wf.nodes[0].mapper == "NA.a"
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -396,8 +396,8 @@ def test_workflow_3b(plugin):
 
     assert wf.nodes[0].mapper == "NA.a"
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -431,8 +431,8 @@ def test_workflow_4(plugin):
     # connect method as it is in the current version
     wf.connect("NA", "out", "NB", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -469,8 +469,8 @@ def test_workflow_4a(plugin):
     # instead of "connect", using kwrg argument in the add method as in the example
     wf.add(nb, a="NA.out")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -505,8 +505,8 @@ def test_workflow_5(plugin):
     # using the map method after add (using mapper for the last added node as default)
     wf.map(mapper="a", inputs={"a": [3, 5]})
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -528,8 +528,8 @@ def test_workflow_5a(plugin):
 
     wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -558,8 +558,8 @@ def test_workflow_6(plugin):
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -597,8 +597,8 @@ def test_workflow_6a(plugin):
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
     wf.connect("NA", "out", "NB", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -634,8 +634,8 @@ def test_workflow_6b(plugin):
     wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -671,8 +671,8 @@ def test_workflow_7(plugin):
     wf.connect_workflow("NA", "wfa", "a")
     wf.map(mapper="a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -698,8 +698,8 @@ def test_workflow_7a(plugin):
     wf.connect(None, "wfa", "NA", "a")
     wf.map(mapper="a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -722,8 +722,8 @@ def test_workflow_7b(plugin):
     wf.add(na, a="wfa")
     wf.map(mapper="a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -752,8 +752,8 @@ def test_workflow_8(plugin):
     wf.connect_workflow("NB", "b", "b")
     assert wf.nodes[0].mapper == "NA.a"
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -787,8 +787,8 @@ def test_workflow_9(plugin):
     # _NA means that I'm using mapper from the NA node, it's the same as ("NA.a", "b")
     wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
@@ -819,8 +819,8 @@ def test_workflow_10(plugin):
     # _NA means that I'm using mapper from the NA node, it's the same as (("NA.a", NA.b), "b")
     wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 5, "NA.b": 10}, 15)]
@@ -851,8 +851,8 @@ def test_workflow_10a(plugin):
     # _NA means that I'm using mapper from the NA node, it's the same as (["NA.a", NA.b], "b")
     wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 3, "NA.b": 10}, 13),
@@ -887,8 +887,8 @@ def test_workflow_11(plugin):
     # _NA, _NB means that I'm using mappers from the NA/NB nodes, it's the same as [("NA.a", NA.b), "NB.a"]
     wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map(mapper=["_NA", "_NB"]) # TODO: this should eb default?
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     expected = [({"NA.a": 3, "NA.b": 0}, 3), ({"NA.a": 5, "NA.b": 10}, 15)]
@@ -930,8 +930,8 @@ def test_workflow_12(plugin):
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     # checking if workflow.results is the same as results of nodes
@@ -976,10 +976,10 @@ def test_workflow_12a(plugin):
     wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
+    sub = Submitter(runnable=wf, plugin=plugin)
     # wf_out can't be used twice
     with pytest.raises(Exception) as exinfo:
-        sub.run_workflow()
+        sub.run()
     assert str(exinfo.value) == "the key wf_out is already used in workflow.result"
 
 # tests for a workflow that have its own input and mapper
@@ -996,8 +996,8 @@ def test_workflow_13(plugin):
     wf.add(na)
     wf.connect_workflow("NA", "wfa", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all
@@ -1020,8 +1020,8 @@ def test_workflow_13a(plugin):
     wf.add(na)
     wf.connect_workflow("NA", "wfa", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all
@@ -1050,8 +1050,8 @@ def test_workflow_14(plugin):
                      outputs_nm=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all
@@ -1076,8 +1076,8 @@ def test_workflow_14a(plugin):
                      outputs_nm=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all
@@ -1103,8 +1103,8 @@ def test_workflow_14b(plugin):
     wf.add(wfa)
     wf.connect_workflow("wfa", "a", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all
@@ -1129,8 +1129,8 @@ def test_workflow_15(plugin):
                      outputs_nm=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all
@@ -1162,8 +1162,8 @@ def test_workflow_16(plugin):
     wf.add(wfb)
     wf.connect("NA", "out", "wfb", "a")
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all
@@ -1205,8 +1205,8 @@ def test_workflow_16a(plugin):
     wf.connect("NA", "out", "wfb", "a")
     assert wf.nodes[0].mapper == "NA.a"
 
-    sub = SubmitterWorkflow(workflow=wf, plugin=plugin)
-    sub.run_workflow()
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
     sub.close()
 
     assert wf.finished_all

From 8c8aa299521fc15099071d5a22e044a690d53260 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Thu, 4 Oct 2018 00:35:39 -0400
Subject: [PATCH 47/55] adding comments; cleaning connect methods

---
 nipype/pipeline/engine/tests/test_newnode.py | 71 ++++++--------------
 nipype/pipeline/engine/workflows.py          | 39 +++++------
 2 files changed, 38 insertions(+), 72 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index fed1690bc2..06f1bc63d9 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -668,7 +668,7 @@ def test_workflow_7(plugin):
 
     wf.add(na)
     # connecting the node with inputs from the workflow
-    wf.connect_workflow("NA", "wfa", "a")
+    wf.connect_wf_input("wfa", "NA", "a")
     wf.map(mapper="a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
@@ -687,38 +687,11 @@ def test_workflow_7(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_7a(plugin):
-    """using inputs for workflow and connect(None...)"""
-    # adding inputs to the workflow directly
-    wf = NewWorkflow(name="wf7a", inputs={"wfa": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-
-    wf.add(na)
-    # if connect has None as the first arg, it is the same as connect_workflow
-    wf.connect(None, "wfa", "NA", "a")
-    wf.map(mapper="a")
-
-    sub = Submitter(runnable=wf, plugin=plugin)
-    sub.run()
-    sub.close()
-
-    expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
-    key_sort = list(expected[0][0].keys())
-    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
-    wf.nodes[0].result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
-    for i, res in enumerate(expected):
-        assert wf.nodes[0].result["out"][i][0] == res[0]
-        assert wf.nodes[0].result["out"][i][1] == res[1]
-
-
-@pytest.mark.parametrize("plugin", Plugins)
-@python35_only
-def test_workflow_7b(plugin):
     """using inputs for workflow and kwarg arg in add (instead of connect)"""
-    wf = NewWorkflow(name="wf7b", inputs={"wfa": [3, 5]}, workingdir="test_wf7b_{}".format(plugin))
+    wf = NewWorkflow(name="wf7a", inputs={"wfa": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
-    # using kwrg argument in the add method (instead of connect or connect_workflow
+    # using kwrg argument in the add method (instead of connect or connect_wf_input
     wf.add(na, a="wfa")
     wf.map(mapper="a")
 
@@ -738,7 +711,7 @@ def test_workflow_7b(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_8(plugin):
-    """using inputs for workflow and connect_workflow for the second node"""
+    """using inputs for workflow and connect_wf_input for the second node"""
     wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
@@ -749,7 +722,7 @@ def test_workflow_8(plugin):
 
     wf.add_nodes([na, nb])
     wf.connect("NA", "out", "NB", "a")
-    wf.connect_workflow("NB", "b", "b")
+    wf.connect_wf_input("b", "NB", "b")
     assert wf.nodes[0].mapper == "NA.a"
 
     sub = Submitter(runnable=wf, plugin=plugin)
@@ -988,13 +961,13 @@ def test_workflow_12a(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_13(plugin):
-    """using inputs for workflow and connect_workflow"""
+    """using inputs for workflow and connect_wf_input"""
     wf = NewWorkflow(name="wf13", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13_{}".format(plugin),
                      outputs_nm=[("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wf.add(na)
-    wf.connect_workflow("NA", "wfa", "a")
+    wf.connect_wf_input("wfa", "NA", "a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -1012,13 +985,13 @@ def test_workflow_13(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_13a(plugin):
-    """using inputs for workflow and connect_workflow"""
+    """using inputs for workflow and connect_wf_input"""
     wf = NewWorkflow(name="wf13a", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13a_{}".format(plugin),
                      outputs_nm=[("NA", "out", "NA_out")])
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     na = NewNode(name="NA", interface=interf_addvar, base_dir="na", mapper="b", inputs={"b": [10, 20]})
     wf.add(na)
-    wf.connect_workflow("NA", "wfa", "a")
+    wf.connect_wf_input("wfa", "NA", "a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -1039,7 +1012,7 @@ def test_workflow_13a(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_14(plugin):
-    """workflow with a workflow as a node"""
+    """workflow with a workflow as a node (no mapper)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na", inputs={"a": 3})
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
@@ -1064,13 +1037,13 @@ def test_workflow_14(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_14a(plugin):
-    """workflow with a workflow as a node (using connect_workflow in wfa)"""
+    """workflow with a workflow as a node (no mapper, using connect_wf_input in wfa)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa", inputs={"a": 3},
                       outputs_nm=[("NA", "out", "NA_out")])
     wfa.add(na)
-    wfa.connect_workflow("NA", "a", "a")
+    wfa.connect_wf_input("a", "NA", "a")
 
     wf = NewWorkflow(name="wf14a", workingdir="test_wf14a_{}".format(plugin),
                      outputs_nm=[("wfa", "NA_out", "wfa_out")])
@@ -1090,18 +1063,18 @@ def test_workflow_14a(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_14b(plugin):
-    """workflow with a workflow as a node (using connect_workflow in wfa and wf)"""
+    """workflow with a workflow as a node (no mapper, using connect_wf_input in wfa and wf)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
                       outputs_nm=[("NA", "out", "NA_out")])
     wfa.add(na)
-    wfa.connect_workflow("NA", "a", "a")
+    wfa.connect_wf_input("a", "NA", "a")
 
     wf = NewWorkflow(name="wf14b", workingdir="test_wf14b_{}".format(plugin),
                      outputs_nm=[("wfa", "NA_out", "wfa_out")], inputs={"a": 3})
     wf.add(wfa)
-    wf.connect_workflow("wfa", "a", "a")
+    wf.connect_wf_input("a", "wfa", "a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -1117,7 +1090,7 @@ def test_workflow_14b(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_15(plugin):
-    """workflow with a workflow as a node with mapper (like 14 but with mapper)"""
+    """workflow with a workflow as a node with mapper (like 14 but with a mapper)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na",
                  inputs={"a": [3, 5]}, mapper="a")
@@ -1143,7 +1116,7 @@ def test_workflow_15(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_16(plugin):
-    """workflow with two nodes, second node without mapper"""
+    """workflow with two nodes, and one is a workflow (no mapper)"""
     wf = NewWorkflow(name="wf16", workingdir="test_wf16_{}".format(plugin),
                      outputs_nm=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -1156,8 +1129,8 @@ def test_workflow_16(plugin):
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
                       outputs_nm=[("NB", "out", "NB_out")])
     wfb.add(nb)
-    wfb.connect_workflow("NB", "b", "b")
-    wfb.connect_workflow("NB", "a", "a")
+    wfb.connect_wf_input("b", "NB", "b")
+    wfb.connect_wf_input("a", "NB", "a")
 
     wf.add(wfb)
     wf.connect("NA", "out", "wfb", "a")
@@ -1183,7 +1156,7 @@ def test_workflow_16(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_16a(plugin):
-    """workflow with two nodes, second node without mapper"""
+    """workflow with two nodes, and one is a workflow (with mapper)"""
     wf = NewWorkflow(name="wf16a", workingdir="test_wf16a_{}".format(plugin),
                      outputs_nm=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
@@ -1197,8 +1170,8 @@ def test_workflow_16a(plugin):
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
                       outputs_nm=[("NB", "out", "NB_out")])
     wfb.add(nb)
-    wfb.connect_workflow("NB", "b", "b")
-    wfb.connect_workflow("NB", "a", "a")
+    wfb.connect_wf_input("b", "NB", "b")
+    wfb.connect_wf_input("a", "NB", "a")
 
     # adding 2 nodes and create a connection (as it is now)
     wf.add(wfb)
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index df3930de4d..a62aa2dea1 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1345,19 +1345,22 @@ def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=Non
                                           mem_gb_node=mem_gb_node, *args, **kwargs)
 
         self.graph = nx.DiGraph()
+        # all nodes in the workflow (probably will be removed)
         self._nodes = []
+        # saving all connection between nodes
         self.connected_var = {}
+        # input that are expected by nodes to get from wf.inputs
         self.needed_inp_wf = []
         if nodes:
             self.add_nodes(nodes)
         for nn in self._nodes:
             self.connected_var[nn] = {}
+        # key: name of a node, value: the node
         self._node_names = {}
+        # key: name of a node, value: mapper of the node
         self._node_mappers = {}
-
         # dj: not sure if this should be different than base_dir
         self.workingdir = os.path.join(os.getcwd(), workingdir)
-
         # list of (nodename, output name in the name, output name in wf) or (nodename, output name in the name)
         self.outputs_nm = outputs_nm
 
@@ -1407,30 +1410,20 @@ def add_nodes(self, nodes):
 
 
     def connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
-        if from_node_nm:
-            from_node = self._node_names[from_node_nm]
-            to_node = self._node_names[to_node_nm]
-            self.graph.add_edges_from([(from_node, to_node)])
-            if not to_node in self.nodes:
-                self.add_nodes(to_node)
-            self.connected_var[to_node][to_socket] = (from_node, from_socket)
-            # from_node.sending_output.append((from_socket, to_node, to_socket))
-            logger.debug('connecting {} and {}'.format(from_node, to_node))
-        else:
-            self.connect_workflow(to_node_nm, from_socket, to_socket)
+        from_node = self._node_names[from_node_nm]
+        to_node = self._node_names[to_node_nm]
+        self.graph.add_edges_from([(from_node, to_node)])
+        if not to_node in self.nodes:
+            self.add_nodes(to_node)
+        self.connected_var[to_node][to_socket] = (from_node, from_socket)
+        # from_node.sending_output.append((from_socket, to_node, to_socket))
+        logger.debug('connecting {} and {}'.format(from_node, to_node))
+
 
     # TODO: change (at least the name)
-    def connect_workflow(self, node_nm, inp_wf, inp_nd):
+    def connect_wf_input(self, inp_wf, node_nm, inp_nd):
         self.needed_inp_wf.append((node_nm, inp_wf, inp_nd))
 
-    def _connect_workflow(self, node_nm, inp_wf, inp_nd):
-        node = self._node_names[node_nm]
-        if "{}.{}".format(self.name, inp_wf) in self.inputs:
-            node.state_inputs.update({"{}.{}".format(node_nm, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
-            node.inputs.update({"{}.{}".format(node_nm, inp_nd): self.inputs["{}.{}".format(self.name, inp_wf)]})
-        else:
-            raise Exception("{} not in the workflow inputs".format(inp_wf))
-
 
     def preparing(self, wf_inputs=None):
         """preparing nodes which are connected: setting the final mapper and state_inputs"""
@@ -1551,7 +1544,7 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
                 self.connect(from_node_nm, from_socket, node.name, inp)
             # TODO not sure if i need it, just check if from_node_nm is not None??
             except(ValueError):
-                self.connect(None, source, node.name, inp)
+                self.connect_wf_input(source, node.name, inp)
 
         return self
 

From 970eb168d481900578d00282205b7d51520e8846 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Thu, 4 Oct 2018 12:13:25 -0400
Subject: [PATCH 48/55] changing orders of the methods

---
 nipype/pipeline/engine/submitter.py |   4 +-
 nipype/pipeline/engine/workflows.py | 314 ++++++++++++++--------------
 2 files changed, 164 insertions(+), 154 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 23e142e5ed..153f23f23c 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -53,7 +53,7 @@ def run_node(self):
         while not self.node.finished_all:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
-        self.node._collecting_output()
+        self.node.collecting_output()
 
 
     def _submit_node(self, node):
@@ -105,7 +105,7 @@ def run_workflow(self, workflow=None, ready=True):
 
         # calling only for the main wf (other wf will be called inside the function)
         if workflow is self.workflow:
-            workflow._collecting_output()
+            workflow.collecting_output()
 
 
     def _run_workflow_el(self, workflow, i, ind, collect_inp=False):
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index a62aa2dea1..c231a8c106 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1146,6 +1146,15 @@ def prepare_state_input(self):
         self._state.prepare_state_input(state_inputs=self.state_inputs)
 
 
+    def checking_input_el(self, ind):
+        """checking if all inputs are available (for specific state element)"""
+        try:
+            self._collecting_input_el(ind)
+            return True
+        except: #TODO specify
+            return False
+
+
     # dj: this is not used for a single node
     def _collecting_input_el(self, ind):
         """collecting all inputs required to run the node (for specific state element)"""
@@ -1161,14 +1170,6 @@ def _collecting_input_el(self, ind):
         return state_dict, inputs_dict
 
 
-    def checking_input_el(self, ind):
-        """checking if all inputs are available (for specific state element)"""
-        try:
-            self._collecting_input_el(ind)
-            return True
-        except: #TODO specify
-            return False
-
    # checking if all outputs are saved
     @property
     def finished_all(self):
@@ -1199,22 +1200,22 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
 
 
     # dj: not sure if I need it
-    def __deepcopy__(self, memo): # memo is a dict of id's to copies
-        id_self = id(self)        # memoization avoids unnecesary recursion
-        _copy = memo.get(id_self)
-        if _copy is None:
-            # changing names of inputs and input_map, so it doesnt contain node.name
-            inputs_copy = dict((key[len(self.name)+1:], deepcopy(value))
-                               for (key, value) in self.inputs.items())
-            interface_copy = deepcopy(self.interface)
-            interface_copy.input_map = dict((key, val[len(self.name)+1:])
-                                            for (key, val) in interface_copy.input_map.items())
-            _copy = type(self)(
-                name=deepcopy(self.name), interface=interface_copy,
-                inputs=inputs_copy, mapper=deepcopy(self.mapper),
-                base_dir=deepcopy(self.nodedir), other_mappers=deepcopy(self._other_mappers))
-            memo[id_self] = _copy
-        return _copy
+    # def __deepcopy__(self, memo): # memo is a dict of id's to copies
+    #     id_self = id(self)        # memoization avoids unnecesary recursion
+    #     _copy = memo.get(id_self)
+    #     if _copy is None:
+    #         # changing names of inputs and input_map, so it doesnt contain node.name
+    #         inputs_copy = dict((key[len(self.name)+1:], deepcopy(value))
+    #                            for (key, value) in self.inputs.items())
+    #         interface_copy = deepcopy(self.interface)
+    #         interface_copy.input_map = dict((key, val[len(self.name)+1:])
+    #                                         for (key, val) in interface_copy.input_map.items())
+    #         _copy = type(self)(
+    #             name=deepcopy(self.name), interface=interface_copy,
+    #             inputs=inputs_copy, mapper=deepcopy(self.mapper),
+    #             base_dir=deepcopy(self.nodedir), other_mappers=deepcopy(self._other_mappers))
+    #         memo[id_self] = _copy
+    #     return _copy
 
 
     @property
@@ -1255,6 +1256,10 @@ def map(self, mapper, inputs=None):
 #                raise MappingError('Cannot map unassigned input field')
 #        self._mappers[field] = values
 
+    def join(self, field, node=None):
+        # TBD
+        pass
+
 
     def run_interface_el(self, i, ind, test=None):
         """ running interface one element generated from node_state."""
@@ -1288,7 +1293,7 @@ def _writting_results_tmp(self, state_dict, dir_nm_el, output):
                 fout.write(str(val_out))
 
 
-    def _collecting_output(self):
+    def collecting_output(self):
         for key_out in self.output_names:
             self._output[key_out] = {}
             for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
@@ -1335,7 +1340,7 @@ def _reading_results(self):
     #     submitter = sub.SubmitterNode(plugin, node=self)
     #     submitter.run_node()
     #     submitter.close()
-    #     self._collecting_output()
+    #     self.collecting_output()
 
 
 class NewWorkflow(NewBase):
@@ -1398,95 +1403,36 @@ def graph_sorted(self):
         # TODO: should I always update the graph?
         return list(nx.topological_sort(self.graph))
 
-    def add_nodes(self, nodes):
-        """adding nodes without defining connections"""
-        self.graph.add_nodes_from(nodes)
-        for nn in nodes:
-            self._nodes.append(nn)
-            #self._inputs.update(nn.inputs)
-            self.connected_var[nn] = {}
-            self._node_names[nn.name] = nn
-            self._node_mappers[nn.name] = nn.mapper
-
-
-    def connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
-        from_node = self._node_names[from_node_nm]
-        to_node = self._node_names[to_node_nm]
-        self.graph.add_edges_from([(from_node, to_node)])
-        if not to_node in self.nodes:
-            self.add_nodes(to_node)
-        self.connected_var[to_node][to_socket] = (from_node, from_socket)
-        # from_node.sending_output.append((from_socket, to_node, to_socket))
-        logger.debug('connecting {} and {}'.format(from_node, to_node))
-
 
-    # TODO: change (at least the name)
-    def connect_wf_input(self, inp_wf, node_nm, inp_nd):
-        self.needed_inp_wf.append((node_nm, inp_wf, inp_nd))
+    def map(self, mapper, node=None, inputs=None):
+        """this is setting a mapper to the wf's nodes (and not to the wf)"""
+        if not node:
+            node = self._last_added
+        if node.mapper:
+            raise WorkflowError("Cannot assign two mappings to the same input")
+        node.map(mapper=mapper, inputs=inputs)
+        self._node_mappers[node.name] = node.mapper
 
 
-    def preparing(self, wf_inputs=None):
-        """preparing nodes which are connected: setting the final mapper and state_inputs"""
-        #pdb.set_trace()
-        for node_nm, inp_wf, inp_nd in self.needed_inp_wf:
-            node = self._node_names[node_nm]
-            if "{}.{}".format(self.name, inp_wf) in wf_inputs:
-                node.state_inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
-                node.inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
-            else:
-                raise Exception("{}.{} not in the workflow inputs".format(self.name, inp_wf))
-        # TODO if should be later, should unify more workingdir and nodedir
-        for nn in self.graph_sorted:
-            if self.mapper:
-                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
-                if is_node(nn):
-                    # TODO: should be just nn.name?
-                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.nodedir)
-                elif is_workflow(nn):
-                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.name)
-                nn._finished_all = False # helps when mp is used
-            else:
-                if is_node(nn):
-                    #TODO: should be just nn.name?
-                    nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
-                elif is_workflow(nn):
-                    nn.workingdir = os.path.join(self.workingdir, nn.name)
-            try:
-                for inp, (out_node, out_var) in self.connected_var[nn].items():
-                    nn.ready2run = False #it has some history (doesnt have to be in the loop)
-                    nn.state_inputs.update(out_node.state_inputs)
-                    nn.needed_outputs.append((out_node, out_var, inp))
-                    #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
-                    if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
-                        nn.mapper = out_node.mapper
-                    else:
-                        pass
-                    #TODO: implement inner mapper
-            except(KeyError):
-                # tmp: we don't care about nn that are not in self.connected_var
-                pass
+    def map_workflow(self, mapper, inputs=None):
+        # TODO:should thi be also implemented?
+        # probably this should be called map, and the other method can be called map_nodes
+        pass
 
-            nn.prepare_state_input()
 
-    # removing temp. from NewWorkflow
-    # def run(self, plugin="serial"):
-    #     #self.preparing(wf_inputs=self.inputs) # moved to submitter
-    #     self.prepare_state_input()
-    #     logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
-    #     submitter = sub.SubmitterWorkflow(workflow=self, plugin=plugin)
-    #     submitter.run_workflow()
-    #     submitter.close()
-    #     self._collecting_output()
+    def join(self, field, node=None):
+        # TBD
+        pass
 
 
-    def _collecting_output(self):
+    def collecting_output(self):
         # not sure, if I should collecto output of all nodes or only the ones that are used in wf.output
         self.node_outputs = {}
         for nn in self.graph:
             if self.mapper:
-                self.node_outputs[nn.name] = [ni._collecting_output() for ni in self.inner_nodes[nn.name]]
+                self.node_outputs[nn.name] = [ni.collecting_output() for ni in self.inner_nodes[nn.name]]
             else:
-                self.node_outputs[nn.name] = nn._collecting_output()
+                self.node_outputs[nn.name] = nn.collecting_output()
         if self.outputs_nm:
             for out in self.outputs_nm:
                 if len(out) == 2:
@@ -1509,6 +1455,58 @@ def _collecting_output(self):
         return self._output
 
 
+    # dj: version without join
+    # TODO: might merge with the function from Node
+    def _check_all_results(self):
+        """checking if all files that should be created are present"""
+        for nn in self.graph_sorted:
+            if nn.name in self.inner_nodes.keys():
+                if not all([ni.finished_all for ni in self.inner_nodes[nn.name]]):
+                    return False
+            else:
+                if not nn.finished_all:
+                    return False
+        self._finished_all = True
+        return True
+
+    # TODO: should try to merge with the function from Node
+    def _reading_results(self):
+        if self.outputs_nm:
+            for out in self.outputs_nm:
+                key_out = out[2] if len(out)==3 else out[1]
+                self._result[key_out] = []
+                if self.mapper:
+                    for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
+                        wf_inputs_dict = self.state.state_values(ind)
+                        dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
+                        res_l= []
+                        for key, val in self.output[key_out][dir_nm_el].items():
+                            with open(val[1]) as fout:
+                                logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
+                                res_l.append((val[0], eval(fout.readline())))
+                        self._result[key_out].append((wf_inputs_dict, res_l))
+                else:
+                    for key, val in self.output[key_out].items():
+                        #TODO: I think that val shouldn't be dict here...
+                        # TMP solution
+                        if type(val) is dict:
+                            val = [v for k,v in val.items()][0]
+                        with open(val[1]) as fout:
+                            logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
+                            self._result[key_out].append((val[0], eval(fout.readline())))
+
+
+    def add_nodes(self, nodes):
+        """adding nodes without defining connections"""
+        self.graph.add_nodes_from(nodes)
+        for nn in nodes:
+            self._nodes.append(nn)
+            #self._inputs.update(nn.inputs)
+            self.connected_var[nn] = {}
+            self._node_names[nn.name] = nn
+            self._node_mappers[nn.name] = nn.mapper
+
+
     def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
             mem_gb=None, **kwargs):
         if is_function(runnable):
@@ -1545,64 +1543,76 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
             # TODO not sure if i need it, just check if from_node_nm is not None??
             except(ValueError):
                 self.connect_wf_input(source, node.name, inp)
-
         return self
 
 
-    def map(self, mapper, node=None, inputs=None):
-        if not node:
-            node = self._last_added
-
-        if node.mapper:
-            raise WorkflowError("Cannot assign two mappings to the same input")
-        node.map(mapper=mapper, inputs=inputs)
-        self._node_mappers[node.name] = node.mapper
-
-
-    def join(self, field, node=None):
-        pass
+    def connect(self, from_node_nm, from_socket, to_node_nm, to_socket):
+        from_node = self._node_names[from_node_nm]
+        to_node = self._node_names[to_node_nm]
+        self.graph.add_edges_from([(from_node, to_node)])
+        if not to_node in self.nodes:
+            self.add_nodes(to_node)
+        self.connected_var[to_node][to_socket] = (from_node, from_socket)
+        # from_node.sending_output.append((from_socket, to_node, to_socket))
+        logger.debug('connecting {} and {}'.format(from_node, to_node))
 
 
-    # TODO: should try to merge with the function from Node
-    def _reading_results(self):
-        if self.outputs_nm:
-            for out in self.outputs_nm:
-                key_out = out[2] if len(out)==3 else out[1]
-                self._result[key_out] = []
-                if self.mapper:
-                    for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
-                        wf_inputs_dict = self.state.state_values(ind)
-                        dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
-                        res_l= []
-                        for key, val in self.output[key_out][dir_nm_el].items():
-                            with open(val[1]) as fout:
-                                logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
-                                res_l.append((val[0], eval(fout.readline())))
-                        self._result[key_out].append((wf_inputs_dict, res_l))
-                else:
-                    for key, val in self.output[key_out].items():
-                        #TODO: I think that val shouldn't be dict here...
-                        # TMP solution
-                        if type(val) is dict:
-                            val = [v for k,v in val.items()][0]
-                        with open(val[1]) as fout:
-                            logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
-                            self._result[key_out].append((val[0], eval(fout.readline())))
+    def connect_wf_input(self, inp_wf, node_nm, inp_nd):
+        self.needed_inp_wf.append((node_nm, inp_wf, inp_nd))
 
 
-    # dj: version without join
-    # TODO: might merge with the function from Node
-    def _check_all_results(self):
-        """checking if all files that should be created are present"""
+    def preparing(self, wf_inputs=None):
+        """preparing nodes which are connected: setting the final mapper and state_inputs"""
+        #pdb.set_trace()
+        for node_nm, inp_wf, inp_nd in self.needed_inp_wf:
+            node = self._node_names[node_nm]
+            if "{}.{}".format(self.name, inp_wf) in wf_inputs:
+                node.state_inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
+                node.inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
+            else:
+                raise Exception("{}.{} not in the workflow inputs".format(self.name, inp_wf))
+        # TODO if should be later, should unify more workingdir and nodedir
         for nn in self.graph_sorted:
-            if nn.name in self.inner_nodes.keys():
-                if not all([ni.finished_all for ni in self.inner_nodes[nn.name]]):
-                    return False
+            if self.mapper:
+                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
+                if is_node(nn):
+                    # TODO: should be just nn.name?
+                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.nodedir)
+                elif is_workflow(nn):
+                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.name)
+                nn._finished_all = False # helps when mp is used
             else:
-                if not nn.finished_all:
-                    return False
-        self._finished_all = True
-        return True
+                if is_node(nn):
+                    #TODO: should be just nn.name?
+                    nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
+                elif is_workflow(nn):
+                    nn.workingdir = os.path.join(self.workingdir, nn.name)
+            try:
+                for inp, (out_node, out_var) in self.connected_var[nn].items():
+                    nn.ready2run = False #it has some history (doesnt have to be in the loop)
+                    nn.state_inputs.update(out_node.state_inputs)
+                    nn.needed_outputs.append((out_node, out_var, inp))
+                    #if there is no mapper provided, i'm assuming that mapper is taken from the previous node
+                    if (not nn.mapper or nn.mapper == out_node.mapper) and out_node.mapper:
+                        nn.mapper = out_node.mapper
+                    else:
+                        pass
+                    #TODO: implement inner mapper
+            except(KeyError):
+                # tmp: we don't care about nn that are not in self.connected_var
+                pass
+
+            nn.prepare_state_input()
+
+    # removing temp. from NewWorkflow
+    # def run(self, plugin="serial"):
+    #     #self.preparing(wf_inputs=self.inputs) # moved to submitter
+    #     self.prepare_state_input()
+    #     logger.debug('the sorted graph is: {}'.format(self.graph_sorted))
+    #     submitter = sub.SubmitterWorkflow(workflow=self, plugin=plugin)
+    #     submitter.run_workflow()
+    #     submitter.close()
+    #     self.collecting_output()
 
 
 def is_function(obj):

From 89395f2ad505eea04e8160e6282fe7e2969abadc Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Thu, 4 Oct 2018 14:45:45 -0400
Subject: [PATCH 49/55] changing directory format for node without mapper (one
 layer less, similar as for wf)

---
 nipype/pipeline/engine/submitter.py          |  4 +--
 nipype/pipeline/engine/tests/test_newnode.py | 28 +++++++++++++++++++-
 nipype/pipeline/engine/workers.py            |  2 +-
 nipype/pipeline/engine/workflows.py          | 21 ++++++++++-----
 4 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 153f23f23c..3f54046d27 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -53,7 +53,7 @@ def run_node(self):
         while not self.node.finished_all:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
-        self.node.collecting_output()
+        self.node.get_output()
 
 
     def _submit_node(self, node):
@@ -105,7 +105,7 @@ def run_workflow(self, workflow=None, ready=True):
 
         # calling only for the main wf (other wf will be called inside the function)
         if workflow is self.workflow:
-            workflow.collecting_output()
+            workflow.get_output()
 
 
     def _run_workflow_el(self, workflow, i, ind, collect_inp=False):
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 06f1bc63d9..993959cb1d 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -69,7 +69,7 @@ def test_node_4():
     assert nn.state.state_values([1]) == {"NA.a": 5}
 
 
-def test_node_5():
+def test_node_4a():
     """Node with interface, mapper and inputs set with the map method"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo)
@@ -83,6 +83,32 @@ def test_node_5():
     assert nn.state.state_values([1]) == {"NA.a": 5}
 
 
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_node_5(plugin):
+    """Node with interface and inputs, no mapper, running interface"""
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    nn = NewNode(name="NA", inputs={"a": 3}, interface=interf_addtwo,
+                 base_dir="test_nd5_{}".format(plugin))
+
+    assert (nn.inputs["NA.a"] == np.array([3])).all()
+
+    sub = Submitter(plugin=plugin, runnable=nn)
+    sub.run()
+    sub.close()
+
+    # checking the results
+    expected = [({"NA.a": 3}, 5)]
+    # to be sure that there is the same order (not sure if node itself should keep the order)
+    key_sort = list(expected[0][0].keys())
+    expected.sort(key=lambda t: [t[0][key] for key in key_sort])
+    nn.result["out"].sort(key=lambda t: [t[0][key] for key in key_sort])
+
+    for i, res in enumerate(expected):
+        assert nn.result["out"][i][0] == res[0]
+        assert nn.result["out"][i][1] == res[1]
+
+
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_node_6(plugin):
diff --git a/nipype/pipeline/engine/workers.py b/nipype/pipeline/engine/workers.py
index b7e35c99ad..456aed49ff 100644
--- a/nipype/pipeline/engine/workers.py
+++ b/nipype/pipeline/engine/workers.py
@@ -87,7 +87,7 @@ def __init__(self):
     def run_el(self, interface, inp):
         print("DASK, run_el: ", interface, inp, time.time())
         # dask  doesn't copy the node second time, so it doesn't see that I change input in the meantime (??)
-        x = self.client.submit(interface, inp[0], inp[1], time.time())
+        x = self.client.submit(interface, inp[0], inp[1])
         print("DASK, status: ", x.status)
         # this important, otherwise dask will not finish the job
         x.add_done_callback(lambda x: print("DONE ", interface, inp))
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index c231a8c106..ceada6feee 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1164,6 +1164,8 @@ def _collecting_input_el(self, ind):
         for (from_node, from_socket, to_socket) in self.needed_outputs:
             dir_nm_el_from = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())
                                        if i in list(from_node._state_inputs.keys())])
+            if not from_node.mapper:
+                dir_nm_el_from = ""
             file_from = os.path.join(from_node.nodedir, dir_nm_el_from, from_socket+".txt")
             with open(file_from) as f:
                 inputs_dict["{}.{}".format(self.name, to_socket)] = eval(f.readline())
@@ -1261,7 +1263,7 @@ def join(self, field, node=None):
         pass
 
 
-    def run_interface_el(self, i, ind, test=None):
+    def run_interface_el(self, i, ind):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
         state_dict, inputs_dict = self._collecting_input_el(ind)
@@ -1287,19 +1289,22 @@ def run_interface_el(self, i, ind, test=None):
 
     def _writting_results_tmp(self, state_dict, dir_nm_el, output):
         """temporary method to write the results in the files (this is usually part of a interface)"""
+        if not self.mapper:
+            dir_nm_el = ''
         os.makedirs(os.path.join(self.nodedir, dir_nm_el), exist_ok=True)
         for key_out, val_out in output.items():
             with open(os.path.join(self.nodedir, dir_nm_el, key_out+".txt"), "w") as fout:
                 fout.write(str(val_out))
 
 
-    def collecting_output(self):
+    def get_output(self):
         for key_out in self.output_names:
             self._output[key_out] = {}
             for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
-                state_dict, inputs_dict = self._collecting_input_el(ind)
+                state_dict = self.state.state_values(ind)
                 dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
-                #pdb.set_trace()
+                if not self.mapper:
+                    dir_nm_el = ""
                 self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.nodedir, dir_nm_el, key_out + ".txt"))
         return self._output
 
@@ -1310,6 +1315,8 @@ def _check_all_results(self):
         for ind in itertools.product(*self.state.all_elements):
             state_dict = self.state.state_values(ind)
             dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
+            if not self.mapper:
+                dir_nm_el = ""
             for key_out in self.output_names:
                 if not os.path.isfile(os.path.join(self.nodedir, dir_nm_el, key_out+".txt")):
                     return False
@@ -1425,14 +1432,14 @@ def join(self, field, node=None):
         pass
 
 
-    def collecting_output(self):
+    def get_output(self):
         # not sure, if I should collecto output of all nodes or only the ones that are used in wf.output
         self.node_outputs = {}
         for nn in self.graph:
             if self.mapper:
-                self.node_outputs[nn.name] = [ni.collecting_output() for ni in self.inner_nodes[nn.name]]
+                self.node_outputs[nn.name] = [ni.get_output() for ni in self.inner_nodes[nn.name]]
             else:
-                self.node_outputs[nn.name] = nn.collecting_output()
+                self.node_outputs[nn.name] = nn.get_output()
         if self.outputs_nm:
             for out in self.outputs_nm:
                 if len(out) == 2:

From a0cdf31f622202461242d99eb34ad3eaea842016 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Thu, 4 Oct 2018 17:58:30 -0400
Subject: [PATCH 50/55] wf has map the same as a node and map_node to change a
 mapper for the wf nodes; changing some names

---
 nipype/pipeline/engine/submitter.py           |   6 +-
 nipype/pipeline/engine/tests/test_newnode.py  | 146 +++++++++++------
 .../engine/tests/test_newnode_neuro.py        |   6 +-
 nipype/pipeline/engine/workflows.py           | 151 +++++++++---------
 4 files changed, 182 insertions(+), 127 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 3f54046d27..78b4e0e6fb 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -50,7 +50,7 @@ def run_node(self):
         """the main method to run a Node"""
         self.node.prepare_state_input()
         self._submit_node(self.node)
-        while not self.node.finished_all:
+        while not self.node.is_complete:
             logger.debug("Submitter, in while, to_finish: {}".format(self.node))
             time.sleep(3)
         self.node.get_output()
@@ -179,8 +179,8 @@ def _output_check(self):
         """"checking if all nodes are done"""
         _to_remove = []
         for node in self._to_finish:
-            print("_output check node", node, node.finished_all)
-            if node.finished_all:
+            print("_output check node", node, node.is_complete)
+            if node.is_complete:
                 _to_remove.append(node)
         for rn in _to_remove:
             self._to_finish.remove(rn)
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 993959cb1d..713b3f1ca7 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -529,7 +529,7 @@ def test_workflow_5(plugin):
 
     wf.add(na)
     # using the map method after add (using mapper for the last added node as default)
-    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.map_node(mapper="a", inputs={"a": [3, 5]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -552,7 +552,7 @@ def test_workflow_5a(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
-    wf.add(na).map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(na).map_node(mapper="a", inputs={"a": [3, 5]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -579,9 +579,9 @@ def test_workflow_6(plugin):
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
-    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.map_node(mapper="a", inputs={"a": [3, 5]})
     wf.add(nb)
-    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
+    wf.map_node(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
@@ -618,9 +618,9 @@ def test_workflow_6a(plugin):
     # using the map method after add (specifying the node)
     wf.add(na)
     wf.add(nb)
-    wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
+    wf.map_node(mapper="a", inputs={"a": [3, 5]}, node=na)
     # TODO: should we se ("a", "c") instead?? shold I forget "NA.a" value?
-    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
+    wf.map_node(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
     wf.connect("NA", "out", "NB", "a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
@@ -657,8 +657,8 @@ def test_workflow_6b(plugin):
 
     wf.add(na)
     wf.add(nb, a="NA.out")
-    wf.map(mapper="a", inputs={"a": [3, 5]}, node=na)
-    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
+    wf.map_node(mapper="a", inputs={"a": [3, 5]}, node=na)
+    wf.map_node(mapper=("NA.a", "b"), inputs={"b": [2, 1]}, node=nb)
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -695,7 +695,7 @@ def test_workflow_7(plugin):
     wf.add(na)
     # connecting the node with inputs from the workflow
     wf.connect_wf_input("wfa", "NA", "a")
-    wf.map(mapper="a")
+    wf.map_node(mapper="a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -719,7 +719,7 @@ def test_workflow_7a(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     # using kwrg argument in the add method (instead of connect or connect_wf_input
     wf.add(na, a="wfa")
-    wf.map(mapper="a")
+    wf.map_node(mapper="a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -781,10 +781,10 @@ def test_workflow_9(plugin):
     """using add(interface) method and mapper from previous nodes"""
     wf = NewWorkflow(name="wf9", workingdir="test_wf9_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    wf.add(name="NA", runnable=interf_addtwo, base_dir="na").map(mapper="a", inputs={"a": [3, 5]})
+    wf.add(name="NA", runnable=interf_addtwo, base_dir="na").map_node(mapper="a", inputs={"a": [3, 5]})
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as ("NA.a", "b")
-    wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
+    wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -813,10 +813,10 @@ def test_workflow_10(plugin):
     """using add(interface) method and scalar mapper from previous nodes"""
     wf = NewWorkflow(name="wf10", workingdir="test_wf10_{}".format(plugin))
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
+    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (("NA.a", NA.b), "b")
-    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [2, 1]})
+    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -845,10 +845,10 @@ def test_workflow_10a(plugin):
     """using add(interface) method and vector mapper from previous nodes"""
     wf = NewWorkflow(name="wf10a", workingdir="test_wf10a_{}".format(plugin))
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
+    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map_node(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (["NA.a", NA.b], "b")
-    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
+    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -879,12 +879,12 @@ def test_workflow_11(plugin):
     """using add(interface) method and vector mapper from previous two nodes"""
     wf = NewWorkflow(name="wf11", workingdir="test_wf11_{}".format(plugin))
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
+    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    wf.add(name="NB", runnable=interf_addtwo, base_dir="nb").map(mapper="a", inputs={"a": [2, 1]})
+    wf.add(name="NB", runnable=interf_addtwo, base_dir="nb").map_node(mapper="a", inputs={"a": [2, 1]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA, _NB means that I'm using mappers from the NA/NB nodes, it's the same as [("NA.a", NA.b), "NB.a"]
-    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map(mapper=["_NA", "_NB"]) # TODO: this should eb default?
+    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map_node(mapper=["_NA", "_NB"]) # TODO: this should eb default?
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -916,7 +916,7 @@ def test_workflow_11(plugin):
 def test_workflow_12(plugin):
     """testing if wf.result works (the same workflow as in test_workflow_6)"""
     wf = NewWorkflow(name="wf12", workingdir="test_wf12_{}".format(plugin),
-                     outputs_nm=[("NA", "out", "NA_out"), ("NB", "out")])
+                     wf_output_names=[("NA", "out", "NA_out"), ("NB", "out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
@@ -924,9 +924,9 @@ def test_workflow_12(plugin):
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
-    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.map_node(mapper="a", inputs={"a": [3, 5]})
     wf.add(nb)
-    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
+    wf.map_node(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
@@ -943,7 +943,7 @@ def test_workflow_12(plugin):
     expected.sort(key=lambda t: [t[0][key] for key in key_sort])
     wf.result["NA_out"].sort(key=lambda t: [t[0][key] for key in key_sort])
     #pdb.set_trace()
-    assert wf.finished_all
+    assert wf.is_complete
     for i, res in enumerate(expected):
         assert wf.result["NA_out"][i][0] == res[0]
         assert wf.result["NA_out"][i][1] == res[1]
@@ -962,7 +962,7 @@ def test_workflow_12(plugin):
 def test_workflow_12a(plugin):
     """testing if wf.result raises exceptione (the same workflow as in test_workflow_6)"""
     wf = NewWorkflow(name="wf12a", workingdir="test_wf12a_{}".format(plugin),
-                     outputs_nm=[("NA", "out", "wf_out"), ("NB", "out", "wf_out")])
+                     wf_output_names=[("NA", "out", "wf_out"), ("NB", "out", "wf_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
 
@@ -970,9 +970,9 @@ def test_workflow_12a(plugin):
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
-    wf.map(mapper="a", inputs={"a": [3, 5]})
+    wf.map_node(mapper="a", inputs={"a": [3, 5]})
     wf.add(nb)
-    wf.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
+    wf.map_node(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     wf.connect("NA", "out", "NB", "a")
 
     sub = Submitter(runnable=wf, plugin=plugin)
@@ -989,7 +989,7 @@ def test_workflow_12a(plugin):
 def test_workflow_13(plugin):
     """using inputs for workflow and connect_wf_input"""
     wf = NewWorkflow(name="wf13", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13_{}".format(plugin),
-                     outputs_nm=[("NA", "out", "NA_out")])
+                     wf_output_names=[("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wf.add(na)
@@ -999,7 +999,7 @@ def test_workflow_13(plugin):
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
     expected = [({"wf13.wfa": 3}, [({"NA.a": 3}, 5)]),
                 ({'wf13.wfa': 5}, [({"NA.a": 5}, 7)])]
     for i, res in enumerate(expected):
@@ -1011,9 +1011,9 @@ def test_workflow_13(plugin):
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_workflow_13a(plugin):
-    """using inputs for workflow and connect_wf_input"""
+    """using inputs for workflow and connect_wf_input (the node has 2 inputs)"""
     wf = NewWorkflow(name="wf13a", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13a_{}".format(plugin),
-                     outputs_nm=[("NA", "out", "NA_out")])
+                     wf_output_names=[("NA", "out", "NA_out")])
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     na = NewNode(name="NA", interface=interf_addvar, base_dir="na", mapper="b", inputs={"b": [10, 20]})
     wf.add(na)
@@ -1023,7 +1023,7 @@ def test_workflow_13a(plugin):
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
     expected = [({"wf13a.wfa": 3}, [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)]),
                 ({'wf13a.wfa': 5}, [({"NA.a": 5, "NA.b": 10}, 15), ({"NA.a": 5, "NA.b": 20}, 25)])]
     for i, res in enumerate(expected):
@@ -1033,6 +1033,54 @@ def test_workflow_13a(plugin):
             assert wf.result["NA_out"][i][1][j][1] == res[1][j][1]
 
 
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_workflow_13c(plugin):
+    """using inputs for workflow and connect_wf_input, using wf.map(mapper, inputs)"""
+    wf = NewWorkflow(name="wf13c", workingdir="test_wf13c_{}".format(plugin),
+                     wf_output_names=[("NA", "out", "NA_out")])
+    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    wf.add(na).map(mapper="wfa", inputs={"wfa": [3, 5]})
+    wf.connect_wf_input("wfa", "NA", "a")
+
+    sub = Submitter(runnable=wf, plugin=plugin)
+    sub.run()
+    sub.close()
+
+    assert wf.is_complete
+    expected = [({"wf13c.wfa": 3}, [({"NA.a": 3}, 5)]),
+                ({'wf13c.wfa': 5}, [({"NA.a": 5}, 7)])]
+    for i, res in enumerate(expected):
+        assert wf.result["NA_out"][i][0] == res[0]
+        assert wf.result["NA_out"][i][1][0][0] == res[1][0][0]
+        assert wf.result["NA_out"][i][1][0][1] == res[1][0][1]
+
+    @pytest.mark.parametrize("plugin", Plugins)
+    @python35_only
+    def test_workflow_13b(plugin):
+        """using inputs for workflow and connect_wf_input, using wf.map(mapper)"""
+        wf = NewWorkflow(name="wf13b", inputs={"wfa": [3, 5]},
+                         workingdir="test_wf13b_{}".format(plugin),
+                         wf_output_names=[("NA", "out", "NA_out")])
+        interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+        na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+        wf.add(na).map(mapper="wfa")
+        wf.connect_wf_input("wfa", "NA", "a")
+
+        sub = Submitter(runnable=wf, plugin=plugin)
+        sub.run()
+        sub.close()
+
+        assert wf.is_complete
+        expected = [({"wf13b.wfa": 3}, [({"NA.a": 3}, 5)]),
+                    ({'wf13b.wfa': 5}, [({"NA.a": 5}, 7)])]
+        for i, res in enumerate(expected):
+            assert wf.result["NA_out"][i][0] == res[0]
+            assert wf.result["NA_out"][i][1][0][0] == res[1][0][0]
+            assert wf.result["NA_out"][i][1][0][1] == res[1][0][1]
+
+
 # workflow as a node
 
 @pytest.mark.parametrize("plugin", Plugins)
@@ -1042,18 +1090,18 @@ def test_workflow_14(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na", inputs={"a": 3})
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
-                      outputs_nm=[("NA", "out", "NA_out")])
+                      wf_output_names=[("NA", "out", "NA_out")])
     wfa.add(na)
 
     wf = NewWorkflow(name="wf14", workingdir="test_wf14_{}".format(plugin),
-                     outputs_nm=[("wfa", "NA_out", "wfa_out")])
+                     wf_output_names=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
     expected = [({"NA.a": 3}, 5)]
     for i, res in enumerate(expected):
         assert wf.result["wfa_out"][i][0] == res[0]
@@ -1067,19 +1115,19 @@ def test_workflow_14a(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa", inputs={"a": 3},
-                      outputs_nm=[("NA", "out", "NA_out")])
+                      wf_output_names=[("NA", "out", "NA_out")])
     wfa.add(na)
     wfa.connect_wf_input("a", "NA", "a")
 
     wf = NewWorkflow(name="wf14a", workingdir="test_wf14a_{}".format(plugin),
-                     outputs_nm=[("wfa", "NA_out", "wfa_out")])
+                     wf_output_names=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
     expected = [({"NA.a": 3}, 5)]
     for i, res in enumerate(expected):
         assert wf.result["wfa_out"][i][0] == res[0]
@@ -1093,12 +1141,12 @@ def test_workflow_14b(plugin):
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
-                      outputs_nm=[("NA", "out", "NA_out")])
+                      wf_output_names=[("NA", "out", "NA_out")])
     wfa.add(na)
     wfa.connect_wf_input("a", "NA", "a")
 
     wf = NewWorkflow(name="wf14b", workingdir="test_wf14b_{}".format(plugin),
-                     outputs_nm=[("wfa", "NA_out", "wfa_out")], inputs={"a": 3})
+                     wf_output_names=[("wfa", "NA_out", "wfa_out")], inputs={"a": 3})
     wf.add(wfa)
     wf.connect_wf_input("a", "wfa", "a")
 
@@ -1106,7 +1154,7 @@ def test_workflow_14b(plugin):
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
     expected = [({"NA.a": 3}, 5)]
     for i, res in enumerate(expected):
         assert wf.result["wfa_out"][i][0] == res[0]
@@ -1121,18 +1169,18 @@ def test_workflow_15(plugin):
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na",
                  inputs={"a": [3, 5]}, mapper="a")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
-                      outputs_nm=[("NA", "out", "NA_out")])
+                      wf_output_names=[("NA", "out", "NA_out")])
     wfa.add(na)
 
     wf = NewWorkflow(name="wf15", workingdir="test_wf15_{}".format(plugin),
-                     outputs_nm=[("wfa", "NA_out", "wfa_out")])
+                     wf_output_names=[("wfa", "NA_out", "wfa_out")])
     wf.add(wfa)
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
     expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     for i, res in enumerate(expected):
         assert wf.result["wfa_out"][i][0] == res[0]
@@ -1144,7 +1192,7 @@ def test_workflow_15(plugin):
 def test_workflow_16(plugin):
     """workflow with two nodes, and one is a workflow (no mapper)"""
     wf = NewWorkflow(name="wf16", workingdir="test_wf16_{}".format(plugin),
-                     outputs_nm=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
+                     wf_output_names=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na", inputs={"a": 3})
     wf.add(na)
@@ -1153,7 +1201,7 @@ def test_workflow_16(plugin):
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
-                      outputs_nm=[("NB", "out", "NB_out")])
+                      wf_output_names=[("NB", "out", "NB_out")])
     wfb.add(nb)
     wfb.connect_wf_input("b", "NB", "b")
     wfb.connect_wf_input("a", "NB", "a")
@@ -1165,7 +1213,7 @@ def test_workflow_16(plugin):
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
     expected_A = [({"NA.a": 3}, 5)]
     for i, res in enumerate(expected_A):
         assert wf.result["NA_out"][i][0] == res[0]
@@ -1184,7 +1232,7 @@ def test_workflow_16(plugin):
 def test_workflow_16a(plugin):
     """workflow with two nodes, and one is a workflow (with mapper)"""
     wf = NewWorkflow(name="wf16a", workingdir="test_wf16a_{}".format(plugin),
-                     outputs_nm=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
+                     wf_output_names=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
@@ -1194,7 +1242,7 @@ def test_workflow_16a(plugin):
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
-                      outputs_nm=[("NB", "out", "NB_out")])
+                      wf_output_names=[("NB", "out", "NB_out")])
     wfb.add(nb)
     wfb.connect_wf_input("b", "NB", "b")
     wfb.connect_wf_input("a", "NB", "a")
@@ -1208,7 +1256,7 @@ def test_workflow_16a(plugin):
     sub.run()
     sub.close()
 
-    assert wf.finished_all
+    assert wf.is_complete
 
     expected_A = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)]
     for i, res in enumerate(expected_A):
diff --git a/nipype/pipeline/engine/tests/test_newnode_neuro.py b/nipype/pipeline/engine/tests/test_newnode_neuro.py
index a35a18400c..ab70a35c17 100644
--- a/nipype/pipeline/engine/tests/test_newnode_neuro.py
+++ b/nipype/pipeline/engine/tests/test_newnode_neuro.py
@@ -55,7 +55,7 @@ def select_target(subject_id, space):
     #dj: don't have option in map to connect with wf input
 
     wf.add(runnable=select_target, name="targets", subject_id="subject_id")\
-        .map(mapper="space", inputs={"space": [space for space in Inputs["output_spaces"]
+        .map_node(mapper="space", inputs={"space": [space for space in Inputs["output_spaces"]
                                                if space.startswith("fs")]})
 
 
@@ -68,7 +68,7 @@ def select_target(subject_id, space):
     wf.add(name='rename_src',
            runnable=Rename(format_string='%(subject)s', keep_ext=True),
                                 in_file="source_file", subject="subject_id")\
-        .map('subject')
+        .map_node('subject')
     pdb.set_trace()
 
     # wf.add('resampling_xfm',
@@ -109,7 +109,7 @@ def select_target(subject_id, space):
                                   out_type='gii'),
            subjects_dir="subjects_dir", subject_id="subject_id", reg_file="set_xfm_source.out_file",
            target_subject="targets.out", source_file="rename_src.out_file")\
-        .map(mapper=[('source_file', 'target_subject'), 'hemi'], inputs={"hemi": ['lh', 'rh']})
+        .map_node(mapper=[('source_file', 'target_subject'), 'hemi'], inputs={"hemi": ['lh', 'rh']})
 
 
     # dj: no conditions
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index ceada6feee..9f78d5b6d1 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1101,7 +1101,7 @@ def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb_no
         # needed outputs from other nodes if the node part of a wf
         self.needed_outputs = []
         # flag that says if node finished all jobs
-        self._finished_all = False
+        self._is_complete = False
 
 
     # TBD
@@ -1146,6 +1146,28 @@ def prepare_state_input(self):
         self._state.prepare_state_input(state_inputs=self.state_inputs)
 
 
+    def map(self, mapper, inputs=None):
+        if self._mapper:
+            raise Exception("mapper is already set")
+        else:
+            self._mapper = aux.change_mapper(mapper, self.name)
+
+        if inputs:
+            inputs = dict(("{}.{}".format(self.name, key), value) for (key, value) in inputs.items())
+            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
+                          for (key, val) in inputs.items())
+            self._inputs.update(inputs)
+            self._state_inputs.update(inputs)
+        if mapper:
+            # updating state if we have a new mapper
+            self._state = state.State(mapper=self._mapper, node_name=self.name, other_mappers=self._other_mappers)
+
+
+    def join(self, field, node=None):
+        # TBD
+        pass
+
+
     def checking_input_el(self, ind):
         """checking if all inputs are available (for specific state element)"""
         try:
@@ -1174,15 +1196,34 @@ def _collecting_input_el(self, ind):
 
    # checking if all outputs are saved
     @property
-    def finished_all(self):
-        # once _finished_all os True, this should not change
-        logger.debug('finished_all {}'.format(self._finished_all))
-        if self._finished_all:
-            return self._finished_all
+    def is_complete(self):
+        # once _is_complete os True, this should not change
+        logger.debug('is_complete {}'.format(self._is_complete))
+        if self._is_complete:
+            return self._is_complete
         else:
             return self._check_all_results()
 
 
+    def get_output(self):
+        raise NotImplementedError
+
+    def _check_all_results(self):
+        raise NotImplementedError
+
+    def _reading_results(self):
+        raise NotImplementedError
+
+
+    def _dict_tuple2list(self, container):
+        if type(container) is dict:
+            val_l = [val for (_, val) in container.items()]
+        elif type(container) is tuple:
+            val_l = [container]
+        else:
+            raise Exception("{} has to be dict or tuple".format(container))
+        return val_l
+
 
 class NewNode(NewBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
@@ -1229,40 +1270,6 @@ def inputs(self, inputs):
         self._inputs.update(inputs)
 
 
-    def map(self, mapper, inputs=None):
-        if self._mapper:
-            raise Exception("mapper is already set")
-        else:
-            self._mapper = aux.change_mapper(mapper, self.name)
-
-        if inputs:
-            inputs = dict(("{}.{}".format(self.name, key), value) for (key, value) in inputs.items())
-            inputs = dict((key, np.array(val)) if type(val) is list else (key, val)
-                          for (key, val) in inputs.items())
-            self._inputs.update(inputs)
-            self._state_inputs.update(inputs)
-        if mapper:
-            # updating state if we have a new mapper
-            self._state = state.State(mapper=self._mapper, node_name=self.name, other_mappers=self._other_mappers)
-
-#    def map_orig(self, field, values=None):
-#        if isinstance(field, list):
-#            for field_
-#            if values is not None:
-#                if len(values != len(field)):
-#        elif isinstance(field, tuple):
-#            pass
-#        if values is None:
-#            values = getattr(self._inputs, field)
-#            if values is None:
-#                raise MappingError('Cannot map unassigned input field')
-#        self._mappers[field] = values
-
-    def join(self, field, node=None):
-        # TBD
-        pass
-
-
     def run_interface_el(self, i, ind):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
@@ -1303,9 +1310,10 @@ def get_output(self):
             for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
                 state_dict = self.state.state_values(ind)
                 dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
-                if not self.mapper:
-                    dir_nm_el = ""
-                self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.nodedir, dir_nm_el, key_out + ".txt"))
+                if self.mapper:
+                    self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.nodedir, dir_nm_el, key_out + ".txt"))
+                else:
+                    self._output[key_out] = (state_dict, os.path.join(self.nodedir, key_out + ".txt"))
         return self._output
 
 
@@ -1320,22 +1328,25 @@ def _check_all_results(self):
             for key_out in self.output_names:
                 if not os.path.isfile(os.path.join(self.nodedir, dir_nm_el, key_out+".txt")):
                     return False
-        self._finished_all = True
+        self._is_complete = True
         return True
 
 
     def _reading_results(self):
-        """temporary: reading results from output files
+        """temporary: reading results from output files (that is now just txt)
             should be probably just reading output for self.output_names
         """
         for key_out in self.output_names:
             self._result[key_out] = []
+            #pdb.set_trace()
             if self._state_inputs:
-                for _, (st_dict, filename) in self._output[key_out].items():
+                val_l = self._dict_tuple2list(self._output[key_out])
+                for (st_dict, filename) in val_l:
                     with open(filename) as fout:
                         self._result[key_out].append((st_dict, eval(fout.readline())))
             else:
                 # st_dict should be {}
+                # not sure if this is used (not tested)
                 (st_dict, filename) = self._output[key_out][None]
                 with open(filename) as fout:
                     self._result[key_out].append(({}, eval(fout.readline())))
@@ -1351,7 +1362,7 @@ def _reading_results(self):
 
 
 class NewWorkflow(NewBase):
-    def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=None,
+    def __init__(self, name, inputs=None, wf_output_names=None, mapper=None, #join_by=None,
                  nodes=None, workingdir=None, mem_gb_node=None, *args, **kwargs):
         super(NewWorkflow, self).__init__(name=name, mapper=mapper, inputs=inputs,
                                           mem_gb_node=mem_gb_node, *args, **kwargs)
@@ -1374,7 +1385,8 @@ def __init__(self, name, inputs=None, outputs_nm=None, mapper=None, #join_by=Non
         # dj: not sure if this should be different than base_dir
         self.workingdir = os.path.join(os.getcwd(), workingdir)
         # list of (nodename, output name in the name, output name in wf) or (nodename, output name in the name)
-        self.outputs_nm = outputs_nm
+        # dj: using different name than for node, since this one it is defined by a user
+        self.wf_output_names = wf_output_names
 
         # nodes that are created when the workflow has mapper (key: node name, value: list of nodes)
         self.inner_nodes = {}
@@ -1411,8 +1423,8 @@ def graph_sorted(self):
         return list(nx.topological_sort(self.graph))
 
 
-    def map(self, mapper, node=None, inputs=None):
-        """this is setting a mapper to the wf's nodes (and not to the wf)"""
+    def map_node(self, mapper, node=None, inputs=None):
+        """this is setting a mapper to the wf's nodes (not to the wf)"""
         if not node:
             node = self._last_added
         if node.mapper:
@@ -1421,17 +1433,6 @@ def map(self, mapper, node=None, inputs=None):
         self._node_mappers[node.name] = node.mapper
 
 
-    def map_workflow(self, mapper, inputs=None):
-        # TODO:should thi be also implemented?
-        # probably this should be called map, and the other method can be called map_nodes
-        pass
-
-
-    def join(self, field, node=None):
-        # TBD
-        pass
-
-
     def get_output(self):
         # not sure, if I should collecto output of all nodes or only the ones that are used in wf.output
         self.node_outputs = {}
@@ -1440,14 +1441,14 @@ def get_output(self):
                 self.node_outputs[nn.name] = [ni.get_output() for ni in self.inner_nodes[nn.name]]
             else:
                 self.node_outputs[nn.name] = nn.get_output()
-        if self.outputs_nm:
-            for out in self.outputs_nm:
+        if self.wf_output_names:
+            for out in self.wf_output_names:
                 if len(out) == 2:
                     node_nm, out_nd_nm, out_wf_nm = out[0], out[1], out[1]
                 elif len(out) == 3:
                     node_nm, out_nd_nm, out_wf_nm = out
                 else:
-                    raise Exception("outputs_nm should have 2 or 3 elements")
+                    raise Exception("wf_output_names should have 2 or 3 elements")
                 if out_wf_nm not in self._output.keys():
                     if self.mapper:
                         self._output[out_wf_nm] = {}
@@ -1468,18 +1469,22 @@ def _check_all_results(self):
         """checking if all files that should be created are present"""
         for nn in self.graph_sorted:
             if nn.name in self.inner_nodes.keys():
-                if not all([ni.finished_all for ni in self.inner_nodes[nn.name]]):
+                if not all([ni.is_complete for ni in self.inner_nodes[nn.name]]):
                     return False
             else:
-                if not nn.finished_all:
+                if not nn.is_complete:
                     return False
-        self._finished_all = True
+        self._is_complete = True
         return True
 
+
     # TODO: should try to merge with the function from Node
     def _reading_results(self):
-        if self.outputs_nm:
-            for out in self.outputs_nm:
+        """reading all results of the workflow
+           using temporary Node._reading_results that reads txt files
+        """
+        if self.wf_output_names:
+            for out in self.wf_output_names:
                 key_out = out[2] if len(out)==3 else out[1]
                 self._result[key_out] = []
                 if self.mapper:
@@ -1487,13 +1492,15 @@ def _reading_results(self):
                         wf_inputs_dict = self.state.state_values(ind)
                         dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
                         res_l= []
-                        for key, val in self.output[key_out][dir_nm_el].items():
+                        val_l = self._dict_tuple2list(self.output[key_out][dir_nm_el])
+                        for val in val_l:
                             with open(val[1]) as fout:
                                 logger.debug('Reading Results: file={}, st_dict={}'.format(val[1], val[0]))
                                 res_l.append((val[0], eval(fout.readline())))
                         self._result[key_out].append((wf_inputs_dict, res_l))
                 else:
-                    for key, val in self.output[key_out].items():
+                    val_l = self._dict_tuple2list(self.output[key_out])
+                    for val in val_l:
                         #TODO: I think that val shouldn't be dict here...
                         # TMP solution
                         if type(val) is dict:
@@ -1587,7 +1594,7 @@ def preparing(self, wf_inputs=None):
                     nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.nodedir)
                 elif is_workflow(nn):
                     nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.name)
-                nn._finished_all = False # helps when mp is used
+                nn._is_complete = False # helps when mp is used
             else:
                 if is_node(nn):
                     #TODO: should be just nn.name?

From 2f11ad6c616b15a81f76b61e8a9a7e788e3943c6 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Thu, 4 Oct 2018 19:18:18 -0400
Subject: [PATCH 51/55] adding fixture to change directory in the tests;
 changing nodedir to workingdir for Node and simplify the code

---
 nipype/pipeline/engine/tests/test_newnode.py | 199 ++++++++++---------
 nipype/pipeline/engine/workflows.py          |  70 +++----
 2 files changed, 140 insertions(+), 129 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 713b3f1ca7..1c2f7244de 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1,3 +1,5 @@
+from ....utils.filemanip import save_json, makedirs, to_str
+
 from .. import NewNode, NewWorkflow
 from ..auxiliary import Function_Interface
 from ..submitter import Submitter
@@ -9,6 +11,19 @@
 python35_only = pytest.mark.skipif(sys.version_info < (3, 5),
                                    reason="requires Python>3.4")
 
+@pytest.fixture(scope="module")
+def change_dir(request):
+    orig_dir = os.getcwd()
+    test_dir = os.path.join(orig_dir, "test_outputs")
+    makedirs(test_dir, exist_ok=True)
+    os.chdir(test_dir)
+
+    def move2orig():
+        os.chdir(orig_dir)
+
+    request.addfinalizer(move2orig)
+
+
 Plugins = ["serial"]
 Plugins = ["serial", "mp", "cf", "dask"]
 
@@ -85,11 +100,11 @@ def test_node_4a():
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_node_5(plugin):
+def test_node_5(plugin, change_dir):
     """Node with interface and inputs, no mapper, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", inputs={"a": 3}, interface=interf_addtwo,
-                 base_dir="test_nd5_{}".format(plugin))
+                 workingdir="test_nd5_{}".format(plugin))
 
     assert (nn.inputs["NA.a"] == np.array([3])).all()
 
@@ -111,10 +126,10 @@ def test_node_5(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_node_6(plugin):
+def test_node_6(plugin, change_dir):
     """Node with interface, inputs and the simplest mapper, running interface"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    nn = NewNode(name="NA", interface=interf_addtwo, base_dir="test_nd6_{}".format(plugin))
+    nn = NewNode(name="NA", interface=interf_addtwo, workingdir="test_nd6_{}".format(plugin))
     nn.map(mapper="a", inputs={"a": [3, 5]})
 
     assert nn.mapper == "NA.a"
@@ -138,10 +153,10 @@ def test_node_6(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_node_7(plugin):
+def test_node_7(plugin, change_dir):
     """Node with interface, inputs and scalar mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test_nd7_{}".format(plugin))
+    nn = NewNode(name="NA", interface=interf_addvar, workingdir="test_nd7_{}".format(plugin))
     # scalar mapper
     nn.map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [2, 1]})
 
@@ -167,10 +182,10 @@ def test_node_7(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_node_8(plugin):
+def test_node_8(plugin, change_dir):
     """Node with interface, inputs and vector mapper, running interface"""
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nn = NewNode(name="NA", interface=interf_addvar, base_dir="test_nd8_{}".format(plugin))
+    nn = NewNode(name="NA", interface=interf_addvar, workingdir="test_nd8_{}".format(plugin))
     # [] for outer product
     nn.map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [2, 1]})
 
@@ -202,7 +217,7 @@ def test_workflow_0(plugin="serial"):
     wf = NewWorkflow(name="wf0", workingdir="test_wf0_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
     # defining a node with mapper and inputs first
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     # one of the way of adding nodes to the workflow
     wf.add_nodes([na])
@@ -212,11 +227,11 @@ def test_workflow_0(plugin="serial"):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_1(plugin):
+def test_workflow_1(plugin, change_dir):
     """workflow with one node with a mapper"""
     wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add_nodes([na])
 
@@ -235,16 +250,16 @@ def test_workflow_1(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_2(plugin):
+def test_workflow_2(plugin, change_dir):
     """workflow with two nodes, second node without mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
     # the second node does not have explicit mapper (but keeps the mapper from the NA node)
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, workingdir="nb")
 
     # adding 2 nodes and create a connection (as it is now)
     wf.add_nodes([na, nb])
@@ -276,15 +291,15 @@ def test_workflow_2(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_2a(plugin):
+def test_workflow_2a(plugin, change_dir):
     """workflow with two nodes, second node with a scalar mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # explicit scalar mapper between "a" from NA and b
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
 
@@ -322,11 +337,11 @@ def test_workflow_2b(plugin):
     """workflow with two nodes, second node with a vector mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # outer mapper
     nb.map(mapper=["NA.a", "b"], inputs={"b": [2, 1]})
 
@@ -363,11 +378,11 @@ def test_workflow_2b(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_3(plugin):
+def test_workflow_3(plugin, change_dir):
     """using add(node) method"""
     wf = NewWorkflow(name="wf3", workingdir="test_wf3_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     # using add method (as in the Satra's example) with a node
     wf.add(na)
@@ -389,13 +404,13 @@ def test_workflow_3(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_3a(plugin):
+def test_workflow_3a(plugin, change_dir):
     """using add(interface) method"""
     wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
 
     # using the add method with an interface
-    wf.add(interf_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+    wf.add(interf_addtwo, workingdir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
     assert wf.nodes[0].mapper == "NA.a"
 
@@ -414,11 +429,11 @@ def test_workflow_3a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_3b(plugin):
+def test_workflow_3b(plugin, change_dir):
     """using add (function) method"""
     wf = NewWorkflow(name="wf3b", workingdir="test_wf3b_{}".format(plugin))
     # using the add method with a function
-    wf.add(fun_addtwo, base_dir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
+    wf.add(fun_addtwo, workingdir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
 
     assert wf.nodes[0].mapper == "NA.a"
 
@@ -438,18 +453,18 @@ def test_workflow_3b(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_4(plugin):
+def test_workflow_4(plugin, change_dir):
     """ using add(node) method
         using wf.connect to connect two nodes
     """
     wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(na)
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # explicit mapper with a variable from the previous node
     # providing inputs with b
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
@@ -480,16 +495,16 @@ def test_workflow_4(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_4a(plugin):
+def test_workflow_4a(plugin, change_dir):
     """ using add(node) method with kwarg arg to connect nodes (instead of wf.connect) """
     wf = NewWorkflow(name="wf4a", workingdir="test_wf4a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(na)
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # explicit mapper with a variable from the previous node
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
     # instead of "connect", using kwrg argument in the add method as in the example
@@ -521,11 +536,11 @@ def test_workflow_4a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_5(plugin):
+def test_workflow_5(plugin, change_dir):
     """using a map method for one node"""
     wf = NewWorkflow(name="wf5", workingdir="test_wf5_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     wf.add(na)
     # using the map method after add (using mapper for the last added node as default)
@@ -546,11 +561,11 @@ def test_workflow_5(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_5a(plugin):
+def test_workflow_5a(plugin, change_dir):
     """using a map method for one node (using add and map in one chain)"""
     wf = NewWorkflow(name="wf5a", workingdir="test_wf5a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     wf.add(na).map_node(mapper="a", inputs={"a": [3, 5]})
 
@@ -569,14 +584,14 @@ def test_workflow_5a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_6(plugin):
+def test_workflow_6(plugin, change_dir):
     """using a map method for two nodes (using last added node as default)"""
     wf = NewWorkflow(name="wf6", workingdir="test_wf6_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
     wf.map_node(mapper="a", inputs={"a": [3, 5]})
@@ -607,14 +622,14 @@ def test_workflow_6(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_6a(plugin):
+def test_workflow_6a(plugin, change_dir):
     """using a map method for two nodes (specifying the node)"""
     wf = NewWorkflow(name="wf6a", workingdir="test_wf6a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map method after add (specifying the node)
     wf.add(na)
     wf.add(nb)
@@ -646,14 +661,14 @@ def test_workflow_6a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_6b(plugin):
+def test_workflow_6b(plugin, change_dir):
     """using a map method for two nodes (specifying the node), using kwarg arg instead of connect"""
     wf = NewWorkflow(name="wf6b", workingdir="test_wf6b_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
 
     wf.add(na)
     wf.add(nb, a="NA.out")
@@ -685,12 +700,12 @@ def test_workflow_6b(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_7(plugin):
+def test_workflow_7(plugin, change_dir):
     """using inputs for workflow and connect_workflow"""
     # adding inputs to the workflow directly
     wf = NewWorkflow(name="wf7", inputs={"wfa": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     wf.add(na)
     # connecting the node with inputs from the workflow
@@ -712,11 +727,11 @@ def test_workflow_7(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_7a(plugin):
+def test_workflow_7a(plugin, change_dir):
     """using inputs for workflow and kwarg arg in add (instead of connect)"""
     wf = NewWorkflow(name="wf7a", inputs={"wfa": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     # using kwrg argument in the add method (instead of connect or connect_wf_input
     wf.add(na, a="wfa")
     wf.map_node(mapper="a")
@@ -736,15 +751,15 @@ def test_workflow_7a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_8(plugin):
+def test_workflow_8(plugin, change_dir):
     """using inputs for workflow and connect_wf_input for the second node"""
     wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
 
     wf.add_nodes([na, nb])
     wf.connect("NA", "out", "NB", "a")
@@ -777,14 +792,14 @@ def test_workflow_8(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_9(plugin):
+def test_workflow_9(plugin, change_dir):
     """using add(interface) method and mapper from previous nodes"""
     wf = NewWorkflow(name="wf9", workingdir="test_wf9_{}".format(plugin))
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    wf.add(name="NA", runnable=interf_addtwo, base_dir="na").map_node(mapper="a", inputs={"a": [3, 5]})
+    wf.add(name="NA", runnable=interf_addtwo, workingdir="na").map_node(mapper="a", inputs={"a": [3, 5]})
     interf_addvar = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as ("NA.a", "b")
-    wf.add(name="NB", runnable=interf_addvar, base_dir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
+    wf.add(name="NB", runnable=interf_addvar, workingdir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -809,14 +824,14 @@ def test_workflow_9(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_10(plugin):
+def test_workflow_10(plugin, change_dir):
     """using add(interface) method and scalar mapper from previous nodes"""
     wf = NewWorkflow(name="wf10", workingdir="test_wf10_{}".format(plugin))
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
+    wf.add(name="NA", runnable=interf_addvar1, workingdir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (("NA.a", NA.b), "b")
-    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
+    wf.add(name="NB", runnable=interf_addvar2, workingdir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -841,14 +856,14 @@ def test_workflow_10(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_10a(plugin):
+def test_workflow_10a(plugin, change_dir):
     """using add(interface) method and vector mapper from previous nodes"""
     wf = NewWorkflow(name="wf10a", workingdir="test_wf10a_{}".format(plugin))
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map_node(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
+    wf.add(name="NA", runnable=interf_addvar1, workingdir="na").map_node(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (["NA.a", NA.b], "b")
-    wf.add(name="NB", runnable=interf_addvar2, base_dir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
+    wf.add(name="NB", runnable=interf_addvar2, workingdir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -875,16 +890,16 @@ def test_workflow_10a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_11(plugin):
+def test_workflow_11(plugin, change_dir):
     """using add(interface) method and vector mapper from previous two nodes"""
     wf = NewWorkflow(name="wf11", workingdir="test_wf11_{}".format(plugin))
     interf_addvar1 = Function_Interface(fun_addvar, ["out"])
-    wf.add(name="NA", runnable=interf_addvar1, base_dir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
+    wf.add(name="NA", runnable=interf_addvar1, workingdir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    wf.add(name="NB", runnable=interf_addtwo, base_dir="nb").map_node(mapper="a", inputs={"a": [2, 1]})
+    wf.add(name="NB", runnable=interf_addtwo, workingdir="nb").map_node(mapper="a", inputs={"a": [2, 1]})
     interf_addvar2 = Function_Interface(fun_addvar, ["out"])
     # _NA, _NB means that I'm using mappers from the NA/NB nodes, it's the same as [("NA.a", NA.b), "NB.a"]
-    wf.add(name="NC", runnable=interf_addvar2, base_dir="nc", a="NA.out", b="NB.out").map_node(mapper=["_NA", "_NB"]) # TODO: this should eb default?
+    wf.add(name="NC", runnable=interf_addvar2, workingdir="nc", a="NA.out", b="NB.out").map_node(mapper=["_NA", "_NB"]) # TODO: this should eb default?
 
     sub = Submitter(runnable=wf, plugin=plugin)
     sub.run()
@@ -913,15 +928,15 @@ def test_workflow_11(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_12(plugin):
+def test_workflow_12(plugin, change_dir):
     """testing if wf.result works (the same workflow as in test_workflow_6)"""
     wf = NewWorkflow(name="wf12", workingdir="test_wf12_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out"), ("NB", "out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
     wf.map_node(mapper="a", inputs={"a": [3, 5]})
@@ -959,15 +974,15 @@ def test_workflow_12(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_12a(plugin):
+def test_workflow_12a(plugin, change_dir):
     """testing if wf.result raises exceptione (the same workflow as in test_workflow_6)"""
     wf = NewWorkflow(name="wf12a", workingdir="test_wf12a_{}".format(plugin),
                      wf_output_names=[("NA", "out", "wf_out"), ("NB", "out", "wf_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
     wf.map_node(mapper="a", inputs={"a": [3, 5]})
@@ -986,12 +1001,12 @@ def test_workflow_12a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_13(plugin):
+def test_workflow_13(plugin, change_dir):
     """using inputs for workflow and connect_wf_input"""
     wf = NewWorkflow(name="wf13", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wf.add(na)
     wf.connect_wf_input("wfa", "NA", "a")
 
@@ -1010,12 +1025,12 @@ def test_workflow_13(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_13a(plugin):
+def test_workflow_13a(plugin, change_dir):
     """using inputs for workflow and connect_wf_input (the node has 2 inputs)"""
     wf = NewWorkflow(name="wf13a", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13a_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out")])
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    na = NewNode(name="NA", interface=interf_addvar, base_dir="na", mapper="b", inputs={"b": [10, 20]})
+    na = NewNode(name="NA", interface=interf_addvar, workingdir="na", mapper="b", inputs={"b": [10, 20]})
     wf.add(na)
     wf.connect_wf_input("wfa", "NA", "a")
 
@@ -1035,12 +1050,12 @@ def test_workflow_13a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_13c(plugin):
+def test_workflow_13c(plugin, change_dir):
     """using inputs for workflow and connect_wf_input, using wf.map(mapper, inputs)"""
     wf = NewWorkflow(name="wf13c", workingdir="test_wf13c_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wf.add(na).map(mapper="wfa", inputs={"wfa": [3, 5]})
     wf.connect_wf_input("wfa", "NA", "a")
 
@@ -1058,13 +1073,13 @@ def test_workflow_13c(plugin):
 
     @pytest.mark.parametrize("plugin", Plugins)
     @python35_only
-    def test_workflow_13b(plugin):
+    def test_workflow_13b(plugin, change_dir):
         """using inputs for workflow and connect_wf_input, using wf.map(mapper)"""
         wf = NewWorkflow(name="wf13b", inputs={"wfa": [3, 5]},
                          workingdir="test_wf13b_{}".format(plugin),
                          wf_output_names=[("NA", "out", "NA_out")])
         interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-        na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+        na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
         wf.add(na).map(mapper="wfa")
         wf.connect_wf_input("wfa", "NA", "a")
 
@@ -1085,10 +1100,10 @@ def test_workflow_13b(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_14(plugin):
+def test_workflow_14(plugin, change_dir):
     """workflow with a workflow as a node (no mapper)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na", inputs={"a": 3})
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na", inputs={"a": 3})
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
                       wf_output_names=[("NA", "out", "NA_out")])
     wfa.add(na)
@@ -1110,10 +1125,10 @@ def test_workflow_14(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_14a(plugin):
+def test_workflow_14a(plugin, change_dir):
     """workflow with a workflow as a node (no mapper, using connect_wf_input in wfa)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa", inputs={"a": 3},
                       wf_output_names=[("NA", "out", "NA_out")])
     wfa.add(na)
@@ -1136,10 +1151,10 @@ def test_workflow_14a(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_14b(plugin):
+def test_workflow_14b(plugin, change_dir):
     """workflow with a workflow as a node (no mapper, using connect_wf_input in wfa and wf)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
                       wf_output_names=[("NA", "out", "NA_out")])
     wfa.add(na)
@@ -1163,10 +1178,10 @@ def test_workflow_14b(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_15(plugin):
+def test_workflow_15(plugin, change_dir):
     """workflow with a workflow as a node with mapper (like 14 but with a mapper)"""
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na",
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na",
                  inputs={"a": [3, 5]}, mapper="a")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
                       wf_output_names=[("NA", "out", "NA_out")])
@@ -1189,17 +1204,17 @@ def test_workflow_15(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_16(plugin):
+def test_workflow_16(plugin, change_dir):
     """workflow with two nodes, and one is a workflow (no mapper)"""
     wf = NewWorkflow(name="wf16", workingdir="test_wf16_{}".format(plugin),
                      wf_output_names=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na", inputs={"a": 3})
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na", inputs={"a": 3})
     wf.add(na)
 
     # the second node does not have explicit mapper (but keeps the mapper from the NA node)
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
                       wf_output_names=[("NB", "out", "NB_out")])
     wfb.add(nb)
@@ -1229,18 +1244,18 @@ def test_workflow_16(plugin):
 
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
-def test_workflow_16a(plugin):
+def test_workflow_16a(plugin, change_dir):
     """workflow with two nodes, and one is a workflow (with mapper)"""
     wf = NewWorkflow(name="wf16a", workingdir="test_wf16a_{}".format(plugin),
                      wf_output_names=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
     interf_addtwo = Function_Interface(fun_addtwo, ["out"])
-    na = NewNode(name="NA", interface=interf_addtwo, base_dir="na")
+    na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(na)
 
     # the second node does not have explicit mapper (but keeps the mapper from the NA node)
     interf_addvar = Function_Interface(fun_addvar, ["out"])
-    nb = NewNode(name="NB", interface=interf_addvar, base_dir="nb")
+    nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
                       wf_output_names=[("NB", "out", "NB_out")])
     wfb.add(nb)
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 9f78d5b6d1..b6c2c62113 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1074,7 +1074,8 @@ class MapState(object):
 
 # dj ??: should I use EngineBase?
 class NewBase(object):
-    def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb_node=None, *args, **kwargs):
+    def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb=None,
+                 cache_location=None, *args, **kwargs):
         self.name = name
         #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
         if inputs:
@@ -1103,6 +1104,10 @@ def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb_no
         # flag that says if node finished all jobs
         self._is_complete = False
 
+        # TODO: don't use it yet
+        self.mem_gb = mem_gb
+        self.cache_location = cache_location
+
 
     # TBD
     def join(self, field):
@@ -1188,7 +1193,7 @@ def _collecting_input_el(self, ind):
                                        if i in list(from_node._state_inputs.keys())])
             if not from_node.mapper:
                 dir_nm_el_from = ""
-            file_from = os.path.join(from_node.nodedir, dir_nm_el_from, from_socket+".txt")
+            file_from = os.path.join(from_node.workingdir, dir_nm_el_from, from_socket+".txt")
             with open(file_from) as f:
                 inputs_dict["{}.{}".format(self.name, to_socket)] = eval(f.readline())
         return state_dict, inputs_dict
@@ -1227,13 +1232,14 @@ def _dict_tuple2list(self, container):
 
 class NewNode(NewBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
-                 base_dir=None, other_mappers=None, mem_gb_node=None, *args, **kwargs):
+                 workingdir=None, other_mappers=None, mem_gb=None, cache_location=None,
+                 *args, **kwargs):
         super(NewNode, self).__init__(name=name, mapper=mapper, inputs=inputs,
-                                      other_mappers=other_mappers, mem_gb_node=mem_gb_node,
-                                      *args, **kwargs)
+                                      other_mappers=other_mappers, mem_gb=mem_gb,
+                                      cache_location=cache_location, *args, **kwargs)
 
         # working directory for node, will be change if node is a part of a wf
-        self.nodedir = base_dir
+        self.workingdir = workingdir
         self.interface = interface
         # adding node name to the interface's name mapping
         self.interface.input_map = dict((key, "{}.{}".format(self.name, value))
@@ -1298,9 +1304,9 @@ def _writting_results_tmp(self, state_dict, dir_nm_el, output):
         """temporary method to write the results in the files (this is usually part of a interface)"""
         if not self.mapper:
             dir_nm_el = ''
-        os.makedirs(os.path.join(self.nodedir, dir_nm_el), exist_ok=True)
+        os.makedirs(os.path.join(self.workingdir, dir_nm_el), exist_ok=True)
         for key_out, val_out in output.items():
-            with open(os.path.join(self.nodedir, dir_nm_el, key_out+".txt"), "w") as fout:
+            with open(os.path.join(self.workingdir, dir_nm_el, key_out+".txt"), "w") as fout:
                 fout.write(str(val_out))
 
 
@@ -1311,9 +1317,9 @@ def get_output(self):
                 state_dict = self.state.state_values(ind)
                 dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
                 if self.mapper:
-                    self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.nodedir, dir_nm_el, key_out + ".txt"))
+                    self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.workingdir, dir_nm_el, key_out + ".txt"))
                 else:
-                    self._output[key_out] = (state_dict, os.path.join(self.nodedir, key_out + ".txt"))
+                    self._output[key_out] = (state_dict, os.path.join(self.workingdir, key_out + ".txt"))
         return self._output
 
 
@@ -1326,7 +1332,7 @@ def _check_all_results(self):
             if not self.mapper:
                 dir_nm_el = ""
             for key_out in self.output_names:
-                if not os.path.isfile(os.path.join(self.nodedir, dir_nm_el, key_out+".txt")):
+                if not os.path.isfile(os.path.join(self.workingdir, dir_nm_el, key_out+".txt")):
                     return False
         self._is_complete = True
         return True
@@ -1363,9 +1369,9 @@ def _reading_results(self):
 
 class NewWorkflow(NewBase):
     def __init__(self, name, inputs=None, wf_output_names=None, mapper=None, #join_by=None,
-                 nodes=None, workingdir=None, mem_gb_node=None, *args, **kwargs):
-        super(NewWorkflow, self).__init__(name=name, mapper=mapper, inputs=inputs,
-                                          mem_gb_node=mem_gb_node, *args, **kwargs)
+                 nodes=None, workingdir=None, mem_gb=None, cache_location=None, *args, **kwargs):
+        super(NewWorkflow, self).__init__(name=name, mapper=mapper, inputs=inputs, mem_gb=mem_gb,
+                                          cache_location=cache_location, *args, **kwargs)
 
         self.graph = nx.DiGraph()
         # all nodes in the workflow (probably will be removed)
@@ -1521,7 +1527,7 @@ def add_nodes(self, nodes):
             self._node_mappers[nn.name] = nn.mapper
 
 
-    def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, mapper=None,
+    def add(self, runnable, name=None, workingdir=None, inputs=None, output_nm=None, mapper=None,
             mem_gb=None, **kwargs):
         if is_function(runnable):
             if not output_nm:
@@ -1529,16 +1535,16 @@ def add(self, runnable, name=None, base_dir=None, inputs=None, output_nm=None, m
             interface = aux.Function_Interface(function=runnable, output_nm=output_nm)
             if not name:
                 raise Exception("you have to specify name for the node")
-            if not base_dir:
-                base_dir = name
-            node = NewNode(interface=interface, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
-                           other_mappers=self._node_mappers, mem_gb_node=mem_gb)
+            if not workingdir:
+                workingdir = name
+            node = NewNode(interface=interface, workingdir=workingdir, name=name, inputs=inputs, mapper=mapper,
+                           other_mappers=self._node_mappers, mem_gb=mem_gb)
         elif is_interface(runnable):
             if not name:
                 raise Exception("you have to specify name for the node")
-            if not base_dir:
-                base_dir = name
-            node = NewNode(interface=runnable, base_dir=base_dir, name=name, inputs=inputs, mapper=mapper,
+            if not workingdir:
+                workingdir = name
+            node = NewNode(interface=runnable, workingdir=workingdir, name=name, inputs=inputs, mapper=mapper,
                            other_mappers=self._node_mappers, mem_gb_node=mem_gb)
         elif is_node(runnable):
             node = runnable
@@ -1585,22 +1591,12 @@ def preparing(self, wf_inputs=None):
                 node.inputs.update({"{}.{}".format(node_nm, inp_nd): wf_inputs["{}.{}".format(self.name, inp_wf)]})
             else:
                 raise Exception("{}.{} not in the workflow inputs".format(self.name, inp_wf))
-        # TODO if should be later, should unify more workingdir and nodedir
         for nn in self.graph_sorted:
-            if self.mapper:
-                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
-                if is_node(nn):
-                    # TODO: should be just nn.name?
-                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.nodedir)
-                elif is_workflow(nn):
-                    nn.nodedir = os.path.join(self.workingdir, dir_nm_el, nn.name)
-                nn._is_complete = False # helps when mp is used
-            else:
-                if is_node(nn):
-                    #TODO: should be just nn.name?
-                    nn.nodedir = os.path.join(self.workingdir, nn.nodedir)
-                elif is_workflow(nn):
-                    nn.workingdir = os.path.join(self.workingdir, nn.name)
+            dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
+            if not self.mapper:
+                dir_nm_el = ""
+            nn.workingdir = os.path.join(self.workingdir, dir_nm_el, nn.name)
+            nn._is_complete = False # helps when mp is used
             try:
                 for inp, (out_node, out_var) in self.connected_var[nn].items():
                     nn.ready2run = False #it has some history (doesnt have to be in the loop)

From bf54454be511c5e83d632a758713289729f654c4 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Fri, 5 Oct 2018 19:11:51 -0400
Subject: [PATCH 52/55] adding wrapper to the current interfaces (with simple
 tests for bet); adding flag that says if values or indecies are used in
 directories names and output

---
 nipype/pipeline/engine/auxiliary.py          |  22 +++-
 nipype/pipeline/engine/state.py              |  27 +++++
 nipype/pipeline/engine/submitter.py          |  15 ++-
 nipype/pipeline/engine/tests/test_newnode.py |  91 ++++++++++++++-
 nipype/pipeline/engine/workflows.py          | 115 ++++++++++++++-----
 5 files changed, 236 insertions(+), 34 deletions(-)

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index 50b2807c4f..3148cc5cd5 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -1,7 +1,8 @@
 import pdb
-import inspect
+import inspect, os
 from ... import config, logging
 logger = logging.getLogger('nipype.workflow')
+from .nodes import Node
 
 
 # dj: might create a new class or move to State
@@ -247,3 +248,22 @@ def __getstate__(self):
     def __setstate__(self, state):
         self.update(state)
         self.__dict__ = self
+
+
+class CurrentInterface(object):
+    def __init__(self, interface, name):
+        self.nn = Node(interface=interface, name=name)
+        self.output = {}
+
+    def run(self, inputs, base_dir, set_out_nm, dir_nm_el):
+        self.nn.base_dir = os.path.join(base_dir, dir_nm_el)
+        for key, val in inputs.items():
+            key = key.split(".")[-1]
+            setattr(self.nn.inputs, key, val)
+        for key, val in set_out_nm.items():
+            key = key.split(".")[-1]
+            setattr(self.nn.inputs, key, os.path.join(self.nn.base_dir, self.nn.name, val))
+            #have to set again self._output_dir
+            self.nn._output_dir = os.path.join(self.nn.base_dir, self.nn.name)
+        res = self.nn.run()
+        return res
\ No newline at end of file
diff --git a/nipype/pipeline/engine/state.py b/nipype/pipeline/engine/state.py
index 58becdd23c..354b8d80af 100644
--- a/nipype/pipeline/engine/state.py
+++ b/nipype/pipeline/engine/state.py
@@ -67,6 +67,7 @@ def shape(self):
 
 
     def state_values(self, ind):
+        """returns state input as a dictionary (input name, value)"""
         if len(ind) > self._ndim:
             raise IndexError("too many indices")
 
@@ -85,6 +86,32 @@ def state_values(self, ind):
         for input in set(self._input_names) - set(self._input_names_mapper):
             state_dict[input] = self.state_inputs[input]
 
+        # in py3.7 we can skip OrderedDict
+        # returning a named tuple?
+        return OrderedDict(sorted(state_dict.items(), key=lambda t: t[0]))
+
+
+    def state_ind(self, ind):
+        """similar to state value but returns indices (not values)"""
+        if len(ind) > self._ndim:
+            raise IndexError("too many indices")
+
+        for ii, index in enumerate(ind):
+            if index > self._shape[ii] - 1:
+                raise IndexError("index {} is out of bounds for axis {} with size {}".format(index, ii, self._shape[ii]))
+
+        state_dict = {}
+        for input, ax in self._axis_for_input.items():
+            # checking which axes are important for the input
+            sl_ax = slice(ax[0], ax[-1]+1)
+            # taking the indexes for the axes
+            ind_inp = tuple(ind[sl_ax]) #used to be list
+            ind_inp_str = "x".join([str(el) for el in ind_inp])
+            state_dict[input] = ind_inp_str
+        # adding inputs that are not used in the mapper
+        for input in set(self._input_names) - set(self._input_names_mapper):
+            state_dict[input] = None
+
         # in py3.7 we can skip OrderedDict
         # returning a named tuple?
         return OrderedDict(sorted(state_dict.items(), key=lambda t: t[0]))
\ No newline at end of file
diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 78b4e0e6fb..9f767d4a46 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -64,7 +64,6 @@ def _submit_node(self, node):
     def _submit_node_el(self, node, i, ind):
         """submitting node's interface for one element of states"""
         logger.debug("SUBMIT WORKER, node: {}, ind: {}".format(node, ind))
-        print("SUBMIT WORK", node.inputs)
         self.worker.run_el(node.run_interface_el, (i, ind))
 
 
@@ -74,7 +73,7 @@ def run_workflow(self, workflow=None, ready=True):
             workflow = self.workflow
         workflow.prepare_state_input()
 
-        # TODO: should I havve inner_nodes for all workflow (to avoid if wf.mapper)??
+        # TODO: should I have inner_nodes for all workflow (to avoid if wf.mapper)??
         if workflow.mapper:
             for key in workflow._node_names.keys():
                 workflow.inner_nodes[key] = []
@@ -87,7 +86,11 @@ def run_workflow(self, workflow=None, ready=True):
                     self.node_line.append((new_workflow, i, ind))
         else:
             if ready:
-                workflow.preparing(wf_inputs=workflow.inputs)
+                if workflow.print_val:
+                        workflow.preparing(wf_inputs=workflow.inputs)
+                else:
+                    inputs_ind = dict((key, None) for (key, _) in workflow.inputs)
+                    workflow.preparing(wf_inputs=workflow.inputs, wf_inputs_ind=inputs_ind)
                 self._run_workflow_nd(workflow=workflow)
             else:
                 self.node_line.append((workflow, 0, ()))
@@ -115,7 +118,11 @@ def _run_workflow_el(self, workflow, i, ind, collect_inp=False):
             st_inputs, wf_inputs = workflow._collecting_input_el(ind)
         else:
             wf_inputs = workflow.state.state_values(ind)
-        workflow.preparing(wf_inputs=wf_inputs)
+        if workflow.print_val:
+            workflow.preparing(wf_inputs=wf_inputs)
+        else:
+            wf_inputs_ind = workflow.state.state_ind(ind)
+            workflow.preparing(wf_inputs=wf_inputs, wf_inputs_ind=wf_inputs_ind)
         self._run_workflow_nd(workflow=workflow)
 
 
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 1c2f7244de..0700fcb0f8 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1,7 +1,8 @@
 from ....utils.filemanip import save_json, makedirs, to_str
+from ....interfaces import fsl
 
 from .. import NewNode, NewWorkflow
-from ..auxiliary import Function_Interface
+from ..auxiliary import Function_Interface, CurrentInterface
 from ..submitter import Submitter
 
 import sys, time, os
@@ -1287,3 +1288,91 @@ def test_workflow_16a(plugin, change_dir):
     for i, res in enumerate(expected_B):
         assert wf.result["NB_out"][i][0] == res[0]
         assert wf.result["NB_out"][i][1] == res[1]
+
+
+# testing CurrentInterface that is a temporary wrapper for current interfaces
+
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_node_1(change_dir, plugin):
+    """Node with a current interface and inputs, no mapper, running interface"""
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+
+    nn = NewNode(name="NA", inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"}, interface=interf_bet,
+                 workingdir="test_cnd1_{}".format(plugin), output_names=[("brain.nii.gz", "out_file", "fsl_out")])
+
+    sub = Submitter(plugin=plugin, runnable=nn)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in nn.output.keys()
+
+
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_node_2(change_dir, plugin):
+    """Node with a current interface and mapper"""
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+
+    in_file_l = ["/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
+                 "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"]
+    nn = NewNode(name="NA", inputs={"in_file": in_file_l}, mapper="in_file", interface=interf_bet, print_val=False,
+                 workingdir="test_cnd2_{}".format(plugin), output_names=[("brain.nii.gz", "out_file", "fsl_out")])
+
+    sub = Submitter(plugin=plugin, runnable=nn)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in nn.output.keys()
+    assert "NA.in_file:0" in nn.output["fsl_out"].keys()
+    assert "NA.in_file:1" in nn.output["fsl_out"].keys()
+
+
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_wf_1(change_dir, plugin):
+    """Wf with a current interface, no mapper"""
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+
+    nn = NewNode(name="fsl", inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"}, interface=interf_bet,
+                 workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False)
+
+    wf = NewWorkflow( workingdir="test_cwf_1_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
+    wf.add_nodes([nn])
+
+    sub = Submitter(plugin=plugin, runnable=wf)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in wf.output.keys()
+
+
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_wf_2(change_dir, plugin):
+    """Wf with a current interface and mapper"""
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+
+    in_file_l = ["/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
+                 "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"]
+
+    nn = NewNode(name="fsl", interface=interf_bet, print_val=False,
+                 workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")])
+
+    wf = NewWorkflow( workingdir="test_cwf_2_{}".format(plugin), name="cw2", wf_output_names=[("fsl", "fsl_out")],
+                      inputs={"in_file": in_file_l}, mapper="in_file", print_val=False)
+    wf.add_nodes([nn])
+    wf.connect_wf_input("in_file", "fsl", "in_file")
+
+    sub = Submitter(plugin=plugin, runnable=wf)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in wf.output.keys()
+    assert 'cw2.in_file:0' in wf.output["fsl_out"].keys()
+    assert 'cw2.in_file:1' in wf.output["fsl_out"].keys()
+
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index b6c2c62113..011b69f892 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1052,7 +1052,7 @@ def _get_dot(self,
         return ('\n' + prefix).join(dotlist)
 
     def add(self, name, node_like):
-        if is_interface(node_like):
+        if is_function_interface(node_like):
             node = Node(node_like, name=name)
         elif is_node(node_like):
             node = node_like
@@ -1075,7 +1075,7 @@ class MapState(object):
 # dj ??: should I use EngineBase?
 class NewBase(object):
     def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb=None,
-                 cache_location=None, *args, **kwargs):
+                 cache_location=None, print_val=True, *args, **kwargs):
         self.name = name
         #dj TODO: I should think what is needed in the __init__ (I redefine some of rhe attributes anyway)
         if inputs:
@@ -1103,6 +1103,8 @@ def __init__(self, name, mapper=None, inputs=None, other_mappers=None, mem_gb=No
         self.needed_outputs = []
         # flag that says if node finished all jobs
         self._is_complete = False
+        # flag that says if value of state input should be printed in output and directories (otherwise indices)
+        self.print_val = print_val
 
         # TODO: don't use it yet
         self.mem_gb = mem_gb
@@ -1233,19 +1235,31 @@ def _dict_tuple2list(self, container):
 class NewNode(NewBase):
     def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
                  workingdir=None, other_mappers=None, mem_gb=None, cache_location=None,
-                 *args, **kwargs):
+                 output_names=None, print_val=True, *args, **kwargs):
         super(NewNode, self).__init__(name=name, mapper=mapper, inputs=inputs,
                                       other_mappers=other_mappers, mem_gb=mem_gb,
-                                      cache_location=cache_location, *args, **kwargs)
+                                      cache_location=cache_location, print_val=print_val,
+                                      *args, **kwargs)
 
         # working directory for node, will be change if node is a part of a wf
         self.workingdir = workingdir
         self.interface = interface
-        # adding node name to the interface's name mapping
-        self.interface.input_map = dict((key, "{}.{}".format(self.name, value))
-                                         for (key, value) in self.interface.input_map.items())
-        # output names taken from interface output name
-        self.output_names = self.interface._output_nm
+
+        # TODO: fixing mess with outputs_names etc.
+        if is_function_interface(self.interface):
+            # adding node name to the interface's name mapping
+            self.interface.input_map = dict((key, "{}.{}".format(self.name, value))
+                                             for (key, value) in self.interface.input_map.items())
+            # output names taken from interface output name
+            self.output_names = self.interface._output_nm
+        elif is_current_interface(self.interface):
+            # TODO: assuming file_name, inter_key_out, node_key_out
+            # used to define name of the output file of current interface
+            self.output_names = output_names
+
+        self.print_val = print_val
+
+
 
 
     # dj: not sure if I need it
@@ -1280,14 +1294,31 @@ def run_interface_el(self, i, ind):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
         state_dict, inputs_dict = self._collecting_input_el(ind)
+        if not self.print_val:
+            state_dict = self.state.state_ind(ind)
         dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
         print("Run interface el, dict={}".format(state_dict))
         logger.debug("Run interface el, name={}, inputs_dict={}, state_dict={}".format(
                                                             self.name, inputs_dict, state_dict))
-        res = self.interface.run(inputs_dict)
-        output = self.interface.output
-        print("Run interface el, output={}".format(output))
-        logger.debug("Run interface el, output={}".format(output))
+        if is_function_interface(self.interface):
+            res = self.interface.run(inputs_dict)
+            output = self.interface.output
+            print("Run fun interface el, output={}".format(output))
+            logger.debug("Run fun interface el, output={}".format(output))
+            self._writting_results_tmp(state_dict, dir_nm_el, output)
+        elif is_current_interface(self.interface):
+            set_nm = {}
+            for out_nm in self.output_names:
+                if len(out_nm) == 2:
+                    out_nm = (out_nm[0], out_nm[1], out_nm[1])
+                if out_nm[2] not in self._output.keys():
+                    self._output[out_nm[2]] = {}
+                set_nm[out_nm[1]] = out_nm[0]
+            if not self.mapper:
+                dir_nm_el = ""
+            res = self.interface.run(inputs=inputs_dict, base_dir=os.path.join(os.getcwd(), self.workingdir),
+                                     set_out_nm=set_nm, dir_nm_el=dir_nm_el)
+
         # TODO when join
         #if self._joinByKey:
         #    dir_join = "join_" + "_".join(["{}.{}".format(i, j) for i, j in list(state_dict.items()) if i not in self._joinByKey])
@@ -1296,7 +1327,6 @@ def run_interface_el(self, i, ind):
         #if self._joinByKey or self._join:
         #    os.makedirs(os.path.join(self.nodedir, dir_join), exist_ok=True)
         #    dir_nm_el = os.path.join(dir_join, dir_nm_el)
-        self._writting_results_tmp(state_dict, dir_nm_el, output)
         return res
 
 
@@ -1312,14 +1342,22 @@ def _writting_results_tmp(self, state_dict, dir_nm_el, output):
 
     def get_output(self):
         for key_out in self.output_names:
+            if is_current_interface(self.interface):
+                key_out, filename = key_out[-1], key_out[0]
             self._output[key_out] = {}
             for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
-                state_dict = self.state.state_values(ind)
+                if self.print_val:
+                    state_dict = self.state.state_values(ind)
+                else:
+                    state_dict = self.state.state_ind(ind)
                 dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
                 if self.mapper:
                     self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.workingdir, dir_nm_el, key_out + ".txt"))
                 else:
-                    self._output[key_out] = (state_dict, os.path.join(self.workingdir, key_out + ".txt"))
+                    if is_function_interface(self.interface):
+                        self._output[key_out] = (state_dict, os.path.join(self.workingdir, key_out + ".txt"))
+                    elif is_current_interface(self.interface):
+                        self._output[key_out] = (state_dict, os.path.join(self.workingdir, self.interface.nn.name, filename))
         return self._output
 
 
@@ -1327,13 +1365,21 @@ def get_output(self):
     def _check_all_results(self):
         """checking if all files that should be created are present"""
         for ind in itertools.product(*self.state.all_elements):
-            state_dict = self.state.state_values(ind)
+            if self.print_val:
+                state_dict = self.state.state_values(ind)
+            else:
+                state_dict = self.state.state_ind(ind)
             dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
             if not self.mapper:
                 dir_nm_el = ""
             for key_out in self.output_names:
-                if not os.path.isfile(os.path.join(self.workingdir, dir_nm_el, key_out+".txt")):
-                    return False
+                if is_function_interface(self.interface):
+                    if not os.path.isfile(os.path.join(self.workingdir, dir_nm_el, key_out+".txt")):
+                        return False
+                elif is_current_interface(self.interface):
+                    if not os.path.isfile(os.path.join(os.getcwd(), self.workingdir,
+                                                       dir_nm_el, self.interface.nn.name, key_out[0])):
+                        return False
         self._is_complete = True
         return True
 
@@ -1369,9 +1415,9 @@ def _reading_results(self):
 
 class NewWorkflow(NewBase):
     def __init__(self, name, inputs=None, wf_output_names=None, mapper=None, #join_by=None,
-                 nodes=None, workingdir=None, mem_gb=None, cache_location=None, *args, **kwargs):
+                 nodes=None, workingdir=None, mem_gb=None, cache_location=None, print_val=True, *args, **kwargs):
         super(NewWorkflow, self).__init__(name=name, mapper=mapper, inputs=inputs, mem_gb=mem_gb,
-                                          cache_location=cache_location, *args, **kwargs)
+                                          cache_location=cache_location, print_val=print_val, *args, **kwargs)
 
         self.graph = nx.DiGraph()
         # all nodes in the workflow (probably will be removed)
@@ -1459,8 +1505,12 @@ def get_output(self):
                     if self.mapper:
                         self._output[out_wf_nm] = {}
                         for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
-                            wf_inputs_dict = self.state.state_values(ind)
-                            dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
+                            if self.print_val:
+                                wf_inputs_dict = self.state.state_values(ind)
+                                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
+                            else:
+                                wf_ind_dict = self.state.state_ind(ind)
+                                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_ind_dict.items())])
                             self._output[out_wf_nm][dir_nm_el] = self.node_outputs[node_nm][i][out_nd_nm]
                     else:
                         self._output[out_wf_nm] = self.node_outputs[node_nm][out_nd_nm]
@@ -1495,7 +1545,10 @@ def _reading_results(self):
                 self._result[key_out] = []
                 if self.mapper:
                     for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
-                        wf_inputs_dict = self.state.state_values(ind)
+                        if self.print_val:
+                            wf_inputs_dict = self.state.state_values(ind)
+                        else:
+                            wf_inputs_dict = self.state.state_ind(ind)
                         dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_dict.items())])
                         res_l= []
                         val_l = self._dict_tuple2list(self.output[key_out][dir_nm_el])
@@ -1539,7 +1592,7 @@ def add(self, runnable, name=None, workingdir=None, inputs=None, output_nm=None,
                 workingdir = name
             node = NewNode(interface=interface, workingdir=workingdir, name=name, inputs=inputs, mapper=mapper,
                            other_mappers=self._node_mappers, mem_gb=mem_gb)
-        elif is_interface(runnable):
+        elif is_function_interface(runnable): # TODO: add current_dir
             if not name:
                 raise Exception("you have to specify name for the node")
             if not workingdir:
@@ -1581,7 +1634,7 @@ def connect_wf_input(self, inp_wf, node_nm, inp_nd):
         self.needed_inp_wf.append((node_nm, inp_wf, inp_nd))
 
 
-    def preparing(self, wf_inputs=None):
+    def preparing(self, wf_inputs=None, wf_inputs_ind=None):
         """preparing nodes which are connected: setting the final mapper and state_inputs"""
         #pdb.set_trace()
         for node_nm, inp_wf, inp_nd in self.needed_inp_wf:
@@ -1592,7 +1645,10 @@ def preparing(self, wf_inputs=None):
             else:
                 raise Exception("{}.{} not in the workflow inputs".format(self.name, inp_wf))
         for nn in self.graph_sorted:
-            dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
+            if self.print_val:
+                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs.items())])
+            else:
+                dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(wf_inputs_ind.items())])
             if not self.mapper:
                 dir_nm_el = ""
             nn.workingdir = os.path.join(self.workingdir, dir_nm_el, nn.name)
@@ -1628,9 +1684,12 @@ def preparing(self, wf_inputs=None):
 def is_function(obj):
     return hasattr(obj, '__call__')
 
-def is_interface(obj):
+def is_function_interface(obj):
     return type(obj) is aux.Function_Interface
 
+def is_current_interface(obj):
+    return type(obj) is aux.CurrentInterface
+
 
 def is_node(obj):
     return type(obj) is NewNode

From 0becd3aee094c1d1ec68aed6d9d270a1ec418aad Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Fri, 5 Oct 2018 21:54:36 -0400
Subject: [PATCH 53/55] updating wf.add method to include nipype interfaces

---
 nipype/pipeline/engine/tests/test_newnode.py | 56 ++++++++++++++++++++
 nipype/pipeline/engine/workflows.py          | 26 ++++++---
 2 files changed, 75 insertions(+), 7 deletions(-)

diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index 0700fcb0f8..a5c2758578 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -1350,6 +1350,61 @@ def test_current_wf_1(change_dir, plugin):
     assert "fsl_out" in wf.output.keys()
 
 
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_wf_1a(change_dir, plugin):
+    """Wf with a current interface, no mapper"""
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+
+    nn = NewNode(name="fsl", inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"}, interface=interf_bet,
+                 workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False)
+
+    wf = NewWorkflow(workingdir="test_cwf_1a_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
+    wf.add(runnable=nn)
+
+    sub = Submitter(plugin=plugin, runnable=wf)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in wf.output.keys()
+
+
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_wf_1b(change_dir, plugin):
+    """Wf with a current interface, no mapper; using wf.add(nipype CurrentInterface)"""
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+
+    wf = NewWorkflow(workingdir="test_cwf_1b_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
+    wf.add(runnable=interf_bet, name="fsl", workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False,
+           inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"})
+
+    sub = Submitter(plugin=plugin, runnable=wf)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in wf.output.keys()
+
+
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_wf_1c(change_dir, plugin):
+    """Wf with a current interface, no mapper; using wf.add(nipype interface) """
+
+    wf = NewWorkflow(workingdir="test_cwf_1c_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
+    wf.add(runnable=fsl.BET(), name="fsl", workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False,
+           inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"})
+
+    sub = Submitter(plugin=plugin, runnable=wf)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in wf.output.keys()
+
+
 @pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
@@ -1376,3 +1431,4 @@ def test_current_wf_2(change_dir, plugin):
     assert 'cw2.in_file:0' in wf.output["fsl_out"].keys()
     assert 'cw2.in_file:1' in wf.output["fsl_out"].keys()
 
+
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 011b69f892..ad077e60aa 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -1580,25 +1580,35 @@ def add_nodes(self, nodes):
             self._node_mappers[nn.name] = nn.mapper
 
 
-    def add(self, runnable, name=None, workingdir=None, inputs=None, output_nm=None, mapper=None,
-            mem_gb=None, **kwargs):
+    def add(self, runnable, name=None, workingdir=None, inputs=None, output_names=None, mapper=None,
+            mem_gb=None, print_val=True, **kwargs):
         if is_function(runnable):
-            if not output_nm:
-                output_nm = ["out"]
-            interface = aux.Function_Interface(function=runnable, output_nm=output_nm)
+            if not output_names:
+                output_names = ["out"]
+            interface = aux.Function_Interface(function=runnable, output_nm=output_names)
             if not name:
                 raise Exception("you have to specify name for the node")
             if not workingdir:
                 workingdir = name
             node = NewNode(interface=interface, workingdir=workingdir, name=name, inputs=inputs, mapper=mapper,
                            other_mappers=self._node_mappers, mem_gb=mem_gb)
-        elif is_function_interface(runnable): # TODO: add current_dir
+        elif is_function_interface(runnable) or is_current_interface(runnable):
             if not name:
                 raise Exception("you have to specify name for the node")
             if not workingdir:
                 workingdir = name
             node = NewNode(interface=runnable, workingdir=workingdir, name=name, inputs=inputs, mapper=mapper,
-                           other_mappers=self._node_mappers, mem_gb_node=mem_gb)
+                           other_mappers=self._node_mappers, mem_gb_node=mem_gb, output_names=output_names,
+                           print_val=print_val)
+        elif is_nipype_interface(runnable):
+            ci = aux.CurrentInterface(interface=runnable, name=name)
+            if not name:
+                raise Exception("you have to specify name for the node")
+            if not workingdir:
+                workingdir = name
+            node = NewNode(interface=ci, workingdir=workingdir, name=name, inputs=inputs, mapper=mapper,
+                           other_mappers=self._node_mappers, mem_gb_node=mem_gb, output_names=output_names,
+                           print_val=print_val)
         elif is_node(runnable):
             node = runnable
         elif is_workflow(runnable):
@@ -1690,6 +1700,8 @@ def is_function_interface(obj):
 def is_current_interface(obj):
     return type(obj) is aux.CurrentInterface
 
+def is_nipype_interface(obj):
+    return hasattr(obj, "_run_interface")
 
 def is_node(obj):
     return type(obj) is NewNode

From d61519c1d31d976879abb123a9bc449a666974a6 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Sun, 7 Oct 2018 22:52:46 -0400
Subject: [PATCH 54/55] test_newnode_neuro works: fixing CurrentInterface and
 adding _reading_ci_output that reads output path from results*.pklz; adding
 print_val flag to Node/Workflow that says if an input value (or index) should
 be used in directories name; adding out_read flag to FunctionInterface that
 says if the value from the file should be saved in the output (instead of
 path)

---
 nipype/pipeline/engine/auxiliary.py           |  15 +-
 nipype/pipeline/engine/submitter.py           |   8 +-
 nipype/pipeline/engine/tests/test_newnode.py  | 196 ++++++++++--------
 .../engine/tests/test_newnode_neuro.py        |  81 +++++---
 nipype/pipeline/engine/workflows.py           | 100 ++++++---
 5 files changed, 246 insertions(+), 154 deletions(-)

diff --git a/nipype/pipeline/engine/auxiliary.py b/nipype/pipeline/engine/auxiliary.py
index 3148cc5cd5..d86702bd7f 100644
--- a/nipype/pipeline/engine/auxiliary.py
+++ b/nipype/pipeline/engine/auxiliary.py
@@ -192,9 +192,9 @@ def _add_name(mlist, name):
 
 #Function interface
 
-class Function_Interface(object):
+class FunctionInterface(object):
     """ A new function interface """
-    def __init__(self, function, output_nm, input_map=None):
+    def __init__(self, function, output_nm, out_read=False, input_map=None):
         self.function = function
         if type(output_nm) is list:
             self._output_nm = output_nm
@@ -206,6 +206,8 @@ def __init__(self, function, output_nm, input_map=None):
         for key in inspect.getargspec(function)[0]:
             if key not in self.input_map.keys():
                 self.input_map[key] = key
+        # flags if we want to read the txt file to save in node.output
+        self.out_read = out_read
 
 
     def run(self, input):
@@ -255,15 +257,12 @@ def __init__(self, interface, name):
         self.nn = Node(interface=interface, name=name)
         self.output = {}
 
-    def run(self, inputs, base_dir, set_out_nm, dir_nm_el):
+    def run(self, inputs, base_dir, dir_nm_el):
         self.nn.base_dir = os.path.join(base_dir, dir_nm_el)
         for key, val in inputs.items():
             key = key.split(".")[-1]
             setattr(self.nn.inputs, key, val)
-        for key, val in set_out_nm.items():
-            key = key.split(".")[-1]
-            setattr(self.nn.inputs, key, os.path.join(self.nn.base_dir, self.nn.name, val))
-            #have to set again self._output_dir
-            self.nn._output_dir = os.path.join(self.nn.base_dir, self.nn.name)
+        #have to set again self._output_dir in case of mapper
+        self.nn._output_dir = os.path.join(self.nn.base_dir, self.nn.name)
         res = self.nn.run()
         return res
\ No newline at end of file
diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 9f767d4a46..23f8619d1f 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -89,7 +89,7 @@ def run_workflow(self, workflow=None, ready=True):
                 if workflow.print_val:
                         workflow.preparing(wf_inputs=workflow.inputs)
                 else:
-                    inputs_ind = dict((key, None) for (key, _) in workflow.inputs)
+                    inputs_ind = dict((key, None) for (key, _) in workflow.inputs.items())
                     workflow.preparing(wf_inputs=workflow.inputs, wf_inputs_ind=inputs_ind)
                 self._run_workflow_nd(workflow=workflow)
             else:
@@ -133,7 +133,7 @@ def _run_workflow_nd(self, workflow):
                 workflow.parent_wf.inner_nodes[node.name].append(node)
             node.prepare_state_input()
             self._to_finish.append(node)
-            # submitting all the nodes who are self sufficient (self.workflow.graph is already sorted)
+             # submitting all the nodes who are self sufficient (self.workflow.graph is already sorted)
             if node.ready2run:
                 if hasattr(node, 'interface'):
                     self._submit_node(node)
@@ -163,6 +163,7 @@ def _nodes_check(self):
         _to_remove = []
         for (to_node, i, ind) in self.node_line:
             if hasattr(to_node, 'interface'):
+                print("_NODES_CHECK INPUT", to_node.name, to_node.checking_input_el(ind))
                 if to_node.checking_input_el(ind):
                     self._submit_node_el(to_node, i, ind)
                     _to_remove.append((to_node, i, ind))
@@ -175,7 +176,6 @@ def _nodes_check(self):
                 else:
                     pass
 
-        # can't remove during iterating
         for rn in _to_remove:
             self.node_line.remove(rn)
         return self.node_line
@@ -186,7 +186,7 @@ def _output_check(self):
         """"checking if all nodes are done"""
         _to_remove = []
         for node in self._to_finish:
-            print("_output check node", node, node.is_complete)
+            print("_output check node", node, node.name, node.is_complete)
             if node.is_complete:
                 _to_remove.append(node)
         for rn in _to_remove:
diff --git a/nipype/pipeline/engine/tests/test_newnode.py b/nipype/pipeline/engine/tests/test_newnode.py
index a5c2758578..9bee85c5ac 100644
--- a/nipype/pipeline/engine/tests/test_newnode.py
+++ b/nipype/pipeline/engine/tests/test_newnode.py
@@ -2,7 +2,7 @@
 from ....interfaces import fsl
 
 from .. import NewNode, NewWorkflow
-from ..auxiliary import Function_Interface, CurrentInterface
+from ..auxiliary import FunctionInterface, CurrentInterface
 from ..submitter import Submitter
 
 import sys, time, os
@@ -40,7 +40,7 @@ def fun_addvar(a, b):
 
 def test_node_1():
     """Node with mandatory arguments only"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo)
     assert nn.mapper is None
     assert nn.inputs == {}
@@ -49,7 +49,7 @@ def test_node_1():
 
 def test_node_2():
     """Node with interface and inputs"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": 3})
     assert nn.mapper is None
     # adding NA to the name of the variable
@@ -59,11 +59,11 @@ def test_node_2():
 
 def test_node_3():
     """Node with interface, inputs and mapper"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]}, mapper="a")
     assert nn.mapper == "NA.a"
     assert (nn.inputs["NA.a"] == np.array([3, 5])).all()
-    
+
     assert nn.state._mapper == "NA.a"
 
     nn.prepare_state_input()
@@ -73,7 +73,7 @@ def test_node_3():
 
 def test_node_4():
     """Node with interface and inputs. mapper set using map method"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, inputs={"a": [3, 5]})
     nn.map(mapper="a")
     assert nn.mapper == "NA.a"
@@ -87,7 +87,7 @@ def test_node_4():
 
 def test_node_4a():
     """Node with interface, mapper and inputs set with the map method"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo)
     nn.map(mapper="a", inputs={"a": [3, 5]})
     assert nn.mapper == "NA.a"
@@ -103,7 +103,7 @@ def test_node_4a():
 @python35_only
 def test_node_5(plugin, change_dir):
     """Node with interface and inputs, no mapper, running interface"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", inputs={"a": 3}, interface=interf_addtwo,
                  workingdir="test_nd5_{}".format(plugin))
 
@@ -129,7 +129,7 @@ def test_node_5(plugin, change_dir):
 @python35_only
 def test_node_6(plugin, change_dir):
     """Node with interface, inputs and the simplest mapper, running interface"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     nn = NewNode(name="NA", interface=interf_addtwo, workingdir="test_nd6_{}".format(plugin))
     nn.map(mapper="a", inputs={"a": [3, 5]})
 
@@ -156,7 +156,7 @@ def test_node_6(plugin, change_dir):
 @python35_only
 def test_node_7(plugin, change_dir):
     """Node with interface, inputs and scalar mapper, running interface"""
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nn = NewNode(name="NA", interface=interf_addvar, workingdir="test_nd7_{}".format(plugin))
     # scalar mapper
     nn.map(mapper=("a", "b"), inputs={"a": [3, 5], "b": [2, 1]})
@@ -185,7 +185,7 @@ def test_node_7(plugin, change_dir):
 @python35_only
 def test_node_8(plugin, change_dir):
     """Node with interface, inputs and vector mapper, running interface"""
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nn = NewNode(name="NA", interface=interf_addvar, workingdir="test_nd8_{}".format(plugin))
     # [] for outer product
     nn.map(mapper=["a", "b"], inputs={"a": [3, 5], "b": [2, 1]})
@@ -216,7 +216,7 @@ def test_node_8(plugin, change_dir):
 def test_workflow_0(plugin="serial"):
     """workflow (without run) with one node with a mapper"""
     wf = NewWorkflow(name="wf0", workingdir="test_wf0_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     # defining a node with mapper and inputs first
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
@@ -231,7 +231,7 @@ def test_workflow_0(plugin="serial"):
 def test_workflow_1(plugin, change_dir):
     """workflow with one node with a mapper"""
     wf = NewWorkflow(name="wf1", workingdir="test_wf1_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add_nodes([na])
@@ -254,12 +254,12 @@ def test_workflow_1(plugin, change_dir):
 def test_workflow_2(plugin, change_dir):
     """workflow with two nodes, second node without mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
     # the second node does not have explicit mapper (but keeps the mapper from the NA node)
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, inputs={"b": 10}, workingdir="nb")
 
     # adding 2 nodes and create a connection (as it is now)
@@ -295,11 +295,11 @@ def test_workflow_2(plugin, change_dir):
 def test_workflow_2a(plugin, change_dir):
     """workflow with two nodes, second node with a scalar mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # explicit scalar mapper between "a" from NA and b
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
@@ -337,11 +337,11 @@ def test_workflow_2a(plugin, change_dir):
 def test_workflow_2b(plugin):
     """workflow with two nodes, second node with a vector mapper"""
     wf = NewWorkflow(name="wf2", workingdir="test_wf2b_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # outer mapper
     nb.map(mapper=["NA.a", "b"], inputs={"b": [2, 1]})
@@ -382,7 +382,7 @@ def test_workflow_2b(plugin):
 def test_workflow_3(plugin, change_dir):
     """using add(node) method"""
     wf = NewWorkflow(name="wf3", workingdir="test_wf3_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     # using add method (as in the Satra's example) with a node
@@ -408,7 +408,7 @@ def test_workflow_3(plugin, change_dir):
 def test_workflow_3a(plugin, change_dir):
     """using add(interface) method"""
     wf = NewWorkflow(name="wf3a", workingdir="test_wf3a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
 
     # using the add method with an interface
     wf.add(interf_addtwo, workingdir="na", mapper="a", inputs={"a": [3, 5]}, name="NA")
@@ -459,12 +459,12 @@ def test_workflow_4(plugin, change_dir):
         using wf.connect to connect two nodes
     """
     wf = NewWorkflow(name="wf4", workingdir="test_wf4_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(na)
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # explicit mapper with a variable from the previous node
     # providing inputs with b
@@ -499,12 +499,12 @@ def test_workflow_4(plugin, change_dir):
 def test_workflow_4a(plugin, change_dir):
     """ using add(node) method with kwarg arg to connect nodes (instead of wf.connect) """
     wf = NewWorkflow(name="wf4a", workingdir="test_wf4a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(na)
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # explicit mapper with a variable from the previous node
     nb.map(mapper=("NA.a", "b"), inputs={"b": [2, 1]})
@@ -540,7 +540,7 @@ def test_workflow_4a(plugin, change_dir):
 def test_workflow_5(plugin, change_dir):
     """using a map method for one node"""
     wf = NewWorkflow(name="wf5", workingdir="test_wf5_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     wf.add(na)
@@ -565,7 +565,7 @@ def test_workflow_5(plugin, change_dir):
 def test_workflow_5a(plugin, change_dir):
     """using a map method for one node (using add and map in one chain)"""
     wf = NewWorkflow(name="wf5a", workingdir="test_wf5a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     wf.add(na).map_node(mapper="a", inputs={"a": [3, 5]})
@@ -588,10 +588,10 @@ def test_workflow_5a(plugin, change_dir):
 def test_workflow_6(plugin, change_dir):
     """using a map method for two nodes (using last added node as default)"""
     wf = NewWorkflow(name="wf6", workingdir="test_wf6_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
@@ -626,10 +626,10 @@ def test_workflow_6(plugin, change_dir):
 def test_workflow_6a(plugin, change_dir):
     """using a map method for two nodes (specifying the node)"""
     wf = NewWorkflow(name="wf6a", workingdir="test_wf6a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map method after add (specifying the node)
     wf.add(na)
@@ -665,10 +665,10 @@ def test_workflow_6a(plugin, change_dir):
 def test_workflow_6b(plugin, change_dir):
     """using a map method for two nodes (specifying the node), using kwarg arg instead of connect"""
     wf = NewWorkflow(name="wf6b", workingdir="test_wf6b_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
 
     wf.add(na)
@@ -705,7 +705,7 @@ def test_workflow_7(plugin, change_dir):
     """using inputs for workflow and connect_workflow"""
     # adding inputs to the workflow directly
     wf = NewWorkflow(name="wf7", inputs={"wfa": [3, 5]}, workingdir="test_wf7_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
     wf.add(na)
@@ -731,7 +731,7 @@ def test_workflow_7(plugin, change_dir):
 def test_workflow_7a(plugin, change_dir):
     """using inputs for workflow and kwarg arg in add (instead of connect)"""
     wf = NewWorkflow(name="wf7a", inputs={"wfa": [3, 5]}, workingdir="test_wf7a_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     # using kwrg argument in the add method (instead of connect or connect_wf_input
     wf.add(na, a="wfa")
@@ -755,11 +755,11 @@ def test_workflow_7a(plugin, change_dir):
 def test_workflow_8(plugin, change_dir):
     """using inputs for workflow and connect_wf_input for the second node"""
     wf = NewWorkflow(name="wf8", workingdir="test_wf8_{}".format(plugin), inputs={"b": 10})
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
 
     wf.add_nodes([na, nb])
@@ -796,9 +796,9 @@ def test_workflow_8(plugin, change_dir):
 def test_workflow_9(plugin, change_dir):
     """using add(interface) method and mapper from previous nodes"""
     wf = NewWorkflow(name="wf9", workingdir="test_wf9_{}".format(plugin))
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     wf.add(name="NA", runnable=interf_addtwo, workingdir="na").map_node(mapper="a", inputs={"a": [3, 5]})
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as ("NA.a", "b")
     wf.add(name="NB", runnable=interf_addvar, workingdir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
@@ -828,9 +828,9 @@ def test_workflow_9(plugin, change_dir):
 def test_workflow_10(plugin, change_dir):
     """using add(interface) method and scalar mapper from previous nodes"""
     wf = NewWorkflow(name="wf10", workingdir="test_wf10_{}".format(plugin))
-    interf_addvar1 = Function_Interface(fun_addvar, ["out"])
+    interf_addvar1 = FunctionInterface(fun_addvar, ["out"])
     wf.add(name="NA", runnable=interf_addvar1, workingdir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
-    interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    interf_addvar2 = FunctionInterface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (("NA.a", NA.b), "b")
     wf.add(name="NB", runnable=interf_addvar2, workingdir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [2, 1]})
 
@@ -860,9 +860,9 @@ def test_workflow_10(plugin, change_dir):
 def test_workflow_10a(plugin, change_dir):
     """using add(interface) method and vector mapper from previous nodes"""
     wf = NewWorkflow(name="wf10a", workingdir="test_wf10a_{}".format(plugin))
-    interf_addvar1 = Function_Interface(fun_addvar, ["out"])
+    interf_addvar1 = FunctionInterface(fun_addvar, ["out"])
     wf.add(name="NA", runnable=interf_addvar1, workingdir="na").map_node(mapper=["a", "b"], inputs={"a": [3, 5], "b": [0, 10]})
-    interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    interf_addvar2 = FunctionInterface(fun_addvar, ["out"])
     # _NA means that I'm using mapper from the NA node, it's the same as (["NA.a", NA.b], "b")
     wf.add(name="NB", runnable=interf_addvar2, workingdir="nb", a="NA.out").map_node(mapper=("_NA", "b"), inputs={"b": [[2, 1], [0, 0]]})
 
@@ -894,11 +894,11 @@ def test_workflow_10a(plugin, change_dir):
 def test_workflow_11(plugin, change_dir):
     """using add(interface) method and vector mapper from previous two nodes"""
     wf = NewWorkflow(name="wf11", workingdir="test_wf11_{}".format(plugin))
-    interf_addvar1 = Function_Interface(fun_addvar, ["out"])
+    interf_addvar1 = FunctionInterface(fun_addvar, ["out"])
     wf.add(name="NA", runnable=interf_addvar1, workingdir="na").map_node(mapper=("a", "b"), inputs={"a": [3, 5], "b": [0, 10]})
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     wf.add(name="NB", runnable=interf_addtwo, workingdir="nb").map_node(mapper="a", inputs={"a": [2, 1]})
-    interf_addvar2 = Function_Interface(fun_addvar, ["out"])
+    interf_addvar2 = FunctionInterface(fun_addvar, ["out"])
     # _NA, _NB means that I'm using mappers from the NA/NB nodes, it's the same as [("NA.a", NA.b), "NB.a"]
     wf.add(name="NC", runnable=interf_addvar2, workingdir="nc", a="NA.out", b="NB.out").map_node(mapper=["_NA", "_NB"]) # TODO: this should eb default?
 
@@ -933,10 +933,10 @@ def test_workflow_12(plugin, change_dir):
     """testing if wf.result works (the same workflow as in test_workflow_6)"""
     wf = NewWorkflow(name="wf12", workingdir="test_wf12_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out"), ("NB", "out")])
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
@@ -979,10 +979,10 @@ def test_workflow_12a(plugin, change_dir):
     """testing if wf.result raises exceptione (the same workflow as in test_workflow_6)"""
     wf = NewWorkflow(name="wf12a", workingdir="test_wf12a_{}".format(plugin),
                      wf_output_names=[("NA", "out", "wf_out"), ("NB", "out", "wf_out")])
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
 
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     # using the map methods after add (using mapper for the last added nodes as default)
     wf.add(na)
@@ -1006,7 +1006,7 @@ def test_workflow_13(plugin, change_dir):
     """using inputs for workflow and connect_wf_input"""
     wf = NewWorkflow(name="wf13", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out")])
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wf.add(na)
     wf.connect_wf_input("wfa", "NA", "a")
@@ -1030,7 +1030,7 @@ def test_workflow_13a(plugin, change_dir):
     """using inputs for workflow and connect_wf_input (the node has 2 inputs)"""
     wf = NewWorkflow(name="wf13a", inputs={"wfa": [3, 5]}, mapper="wfa", workingdir="test_wf13a_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out")])
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     na = NewNode(name="NA", interface=interf_addvar, workingdir="na", mapper="b", inputs={"b": [10, 20]})
     wf.add(na)
     wf.connect_wf_input("wfa", "NA", "a")
@@ -1055,7 +1055,7 @@ def test_workflow_13c(plugin, change_dir):
     """using inputs for workflow and connect_wf_input, using wf.map(mapper, inputs)"""
     wf = NewWorkflow(name="wf13c", workingdir="test_wf13c_{}".format(plugin),
                      wf_output_names=[("NA", "out", "NA_out")])
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wf.add(na).map(mapper="wfa", inputs={"wfa": [3, 5]})
     wf.connect_wf_input("wfa", "NA", "a")
@@ -1079,7 +1079,7 @@ def test_workflow_13b(plugin, change_dir):
         wf = NewWorkflow(name="wf13b", inputs={"wfa": [3, 5]},
                          workingdir="test_wf13b_{}".format(plugin),
                          wf_output_names=[("NA", "out", "NA_out")])
-        interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+        interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
         na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
         wf.add(na).map(mapper="wfa")
         wf.connect_wf_input("wfa", "NA", "a")
@@ -1103,7 +1103,7 @@ def test_workflow_13b(plugin, change_dir):
 @python35_only
 def test_workflow_14(plugin, change_dir):
     """workflow with a workflow as a node (no mapper)"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na", inputs={"a": 3})
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
                       wf_output_names=[("NA", "out", "NA_out")])
@@ -1128,7 +1128,7 @@ def test_workflow_14(plugin, change_dir):
 @python35_only
 def test_workflow_14a(plugin, change_dir):
     """workflow with a workflow as a node (no mapper, using connect_wf_input in wfa)"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa", inputs={"a": 3},
                       wf_output_names=[("NA", "out", "NA_out")])
@@ -1154,7 +1154,7 @@ def test_workflow_14a(plugin, change_dir):
 @python35_only
 def test_workflow_14b(plugin, change_dir):
     """workflow with a workflow as a node (no mapper, using connect_wf_input in wfa and wf)"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
                       wf_output_names=[("NA", "out", "NA_out")])
@@ -1181,7 +1181,7 @@ def test_workflow_14b(plugin, change_dir):
 @python35_only
 def test_workflow_15(plugin, change_dir):
     """workflow with a workflow as a node with mapper (like 14 but with a mapper)"""
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na",
                  inputs={"a": [3, 5]}, mapper="a")
     wfa = NewWorkflow(name="wfa", workingdir="test_wfa",
@@ -1209,12 +1209,12 @@ def test_workflow_16(plugin, change_dir):
     """workflow with two nodes, and one is a workflow (no mapper)"""
     wf = NewWorkflow(name="wf16", workingdir="test_wf16_{}".format(plugin),
                      wf_output_names=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na", inputs={"a": 3})
     wf.add(na)
 
     # the second node does not have explicit mapper (but keeps the mapper from the NA node)
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
                       wf_output_names=[("NB", "out", "NB_out")])
@@ -1249,13 +1249,13 @@ def test_workflow_16a(plugin, change_dir):
     """workflow with two nodes, and one is a workflow (with mapper)"""
     wf = NewWorkflow(name="wf16a", workingdir="test_wf16a_{}".format(plugin),
                      wf_output_names=[("wfb", "NB_out"), ("NA", "out", "NA_out")])
-    interf_addtwo = Function_Interface(fun_addtwo, ["out"])
+    interf_addtwo = FunctionInterface(fun_addtwo, ["out"])
     na = NewNode(name="NA", interface=interf_addtwo, workingdir="na")
     na.map(mapper="a", inputs={"a": [3, 5]})
     wf.add(na)
 
     # the second node does not have explicit mapper (but keeps the mapper from the NA node)
-    interf_addvar = Function_Interface(fun_addvar, ["out"])
+    interf_addvar = FunctionInterface(fun_addvar, ["out"])
     nb = NewNode(name="NB", interface=interf_addvar, workingdir="nb")
     wfb = NewWorkflow(name="wfb", workingdir="test_wfb", inputs={"b": 10},
                       wf_output_names=[("NB", "out", "NB_out")])
@@ -1297,16 +1297,16 @@ def test_workflow_16a(plugin, change_dir):
 @python35_only
 def test_current_node_1(change_dir, plugin):
     """Node with a current interface and inputs, no mapper, running interface"""
-    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     nn = NewNode(name="NA", inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"}, interface=interf_bet,
-                 workingdir="test_cnd1_{}".format(plugin), output_names=[("brain.nii.gz", "out_file", "fsl_out")])
+                 workingdir="test_cnd1_{}".format(plugin), output_names=["out_file"])
 
     sub = Submitter(plugin=plugin, runnable=nn)
     sub.run()
     sub.close()
-
-    assert "fsl_out" in nn.output.keys()
+    # TODO: nodes only returns relative path
+    assert "out_file" in nn.output.keys()
 
 
 @pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
@@ -1314,20 +1314,20 @@ def test_current_node_1(change_dir, plugin):
 @python35_only
 def test_current_node_2(change_dir, plugin):
     """Node with a current interface and mapper"""
-    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     in_file_l = ["/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
                  "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"]
     nn = NewNode(name="NA", inputs={"in_file": in_file_l}, mapper="in_file", interface=interf_bet, print_val=False,
-                 workingdir="test_cnd2_{}".format(plugin), output_names=[("brain.nii.gz", "out_file", "fsl_out")])
+                 workingdir="test_cnd2_{}".format(plugin), output_names=["out_file"])
 
     sub = Submitter(plugin=plugin, runnable=nn)
     sub.run()
     sub.close()
 
-    assert "fsl_out" in nn.output.keys()
-    assert "NA.in_file:0" in nn.output["fsl_out"].keys()
-    assert "NA.in_file:1" in nn.output["fsl_out"].keys()
+    assert "out_file" in nn.output.keys()
+    assert "NA.in_file:0" in nn.output["out_file"].keys()
+    assert "NA.in_file:1" in nn.output["out_file"].keys()
 
 
 @pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
@@ -1335,12 +1335,12 @@ def test_current_node_2(change_dir, plugin):
 @python35_only
 def test_current_wf_1(change_dir, plugin):
     """Wf with a current interface, no mapper"""
-    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     nn = NewNode(name="fsl", inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"}, interface=interf_bet,
-                 workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False)
+                 workingdir="nn", output_names=["out_file"], print_val=False)
 
-    wf = NewWorkflow( workingdir="test_cwf_1_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
+    wf = NewWorkflow( workingdir="test_cwf_1_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "out_file", "fsl_out")], print_val=False)
     wf.add_nodes([nn])
 
     sub = Submitter(plugin=plugin, runnable=wf)
@@ -1355,12 +1355,12 @@ def test_current_wf_1(change_dir, plugin):
 @python35_only
 def test_current_wf_1a(change_dir, plugin):
     """Wf with a current interface, no mapper"""
-    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     nn = NewNode(name="fsl", inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"}, interface=interf_bet,
-                 workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False)
+                 workingdir="nn", output_names=["out_file"], print_val=False)
 
-    wf = NewWorkflow(workingdir="test_cwf_1a_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
+    wf = NewWorkflow(workingdir="test_cwf_1a_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "out_file", "fsl_out")], print_val=False)
     wf.add(runnable=nn)
 
     sub = Submitter(plugin=plugin, runnable=wf)
@@ -1375,10 +1375,10 @@ def test_current_wf_1a(change_dir, plugin):
 @python35_only
 def test_current_wf_1b(change_dir, plugin):
     """Wf with a current interface, no mapper; using wf.add(nipype CurrentInterface)"""
-    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
-    wf = NewWorkflow(workingdir="test_cwf_1b_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
-    wf.add(runnable=interf_bet, name="fsl", workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False,
+    wf = NewWorkflow(workingdir="test_cwf_1b_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "out_file", "fsl_out")], print_val=False)
+    wf.add(runnable=interf_bet, name="fsl", workingdir="nn", output_names=["out_file"], print_val=False,
            inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"})
 
     sub = Submitter(plugin=plugin, runnable=wf)
@@ -1394,8 +1394,8 @@ def test_current_wf_1b(change_dir, plugin):
 def test_current_wf_1c(change_dir, plugin):
     """Wf with a current interface, no mapper; using wf.add(nipype interface) """
 
-    wf = NewWorkflow(workingdir="test_cwf_1c_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "fsl_out")], print_val=False)
-    wf.add(runnable=fsl.BET(), name="fsl", workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")], print_val=False,
+    wf = NewWorkflow(workingdir="test_cwf_1c_{}".format(plugin), name="cw1", wf_output_names=[("fsl", "out_file", "fsl_out")], print_val=False)
+    wf.add(runnable=fsl.BET(), name="fsl", workingdir="nn", output_names=["out_file"], print_val=False,
            inputs={"in_file": "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"})
 
     sub = Submitter(plugin=plugin, runnable=wf)
@@ -1410,15 +1410,15 @@ def test_current_wf_1c(change_dir, plugin):
 @python35_only
 def test_current_wf_2(change_dir, plugin):
     """Wf with a current interface and mapper"""
-    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl")
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     in_file_l = ["/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
                  "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"]
 
     nn = NewNode(name="fsl", interface=interf_bet, print_val=False,
-                 workingdir="nn", output_names=[("brain.nii.gz", "out_file", "fsl_out")])
+                 workingdir="nn", output_names=["out_file"])
 
-    wf = NewWorkflow( workingdir="test_cwf_2_{}".format(plugin), name="cw2", wf_output_names=[("fsl", "fsl_out")],
+    wf = NewWorkflow( workingdir="test_cwf_2_{}".format(plugin), name="cw2", wf_output_names=[("fsl", "out_file", "fsl_out")],
                       inputs={"in_file": in_file_l}, mapper="in_file", print_val=False)
     wf.add_nodes([nn])
     wf.connect_wf_input("in_file", "fsl", "in_file")
@@ -1432,3 +1432,29 @@ def test_current_wf_2(change_dir, plugin):
     assert 'cw2.in_file:1' in wf.output["fsl_out"].keys()
 
 
+@pytest.mark.skipif(not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.parametrize("plugin", Plugins)
+@python35_only
+def test_current_wf_2a(change_dir, plugin):
+    """Wf with a current interface and mapper"""
+    interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
+
+    in_file_l = ["/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
+                 "/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"]
+
+    nn = NewNode(name="fsl", interface=interf_bet, print_val=False,
+                 workingdir="nn", output_names=["out_file"],
+                 inputs={"in_file": in_file_l}, mapper="in_file")
+
+    wf = NewWorkflow( workingdir="test_cwf_2a_{}".format(plugin), name="cw2a", wf_output_names=[("fsl", "out_file", "fsl_out")],
+                      print_val=False)
+    wf.add_nodes([nn])
+   # wf.connect_wf_input("in_file", "fsl", "in_file")
+
+    sub = Submitter(plugin=plugin, runnable=wf)
+    sub.run()
+    sub.close()
+
+    assert "fsl_out" in wf.output.keys()
+    assert 'fsl.in_file:0' in wf.output["fsl_out"].keys()
+    assert 'fsl.in_file:1' in wf.output["fsl_out"].keys()
diff --git a/nipype/pipeline/engine/tests/test_newnode_neuro.py b/nipype/pipeline/engine/tests/test_newnode_neuro.py
index ab70a35c17..eef8841f4f 100644
--- a/nipype/pipeline/engine/tests/test_newnode_neuro.py
+++ b/nipype/pipeline/engine/tests/test_newnode_neuro.py
@@ -1,5 +1,8 @@
+import os
+import pytest
+
 from nipype.pipeline.engine import NewNode, NewWorkflow
-from ..auxiliary import Function_Interface
+from ..submitter import Submitter
 
 #dj niworkflows vs ...??
 from nipype.interfaces.utility import Rename
@@ -7,6 +10,18 @@
 
 from fmriprep.interfaces.freesurfer import PatchedConcatenateLTA as ConcatenateLTA
 
+@pytest.fixture()
+def change_dir(request):
+    orig_dir = os.getcwd()
+    test_dir = os.path.join(orig_dir, "/nipype/nipype/pipeline/engine/test_neuro")
+    os.makedirs(test_dir, exist_ok=True)
+    os.chdir(test_dir)
+
+    def move2orig():
+        os.chdir(orig_dir)
+
+    request.addfinalizer(move2orig)
+
 import pdb
 
 Name = "example"
@@ -14,13 +29,20 @@
 # TODO, adding fields to Inputs (subject_id)
 Inputs = {"subject_id": "sub-01",
           "output_spaces": ["fsaverage", "fsaverage5"],
-          "source_file": "/Users/dorota/fmriprep_test/workdir1/fmriprep_wf/single_subject_01_wf/func_preproc_ses_test_task_fingerfootlips_wf/bold_t1_trans_wf/merge/vol0000_xform-00000_merged.nii",
-          "t1_preproc": "/Users/dorota/fmriprep_test/output1/fmriprep/sub-01/anat/sub-01_T1w_preproc.nii.gz",
-          "t1_2_fsnative_forward_transform": "/Users/dorota/fmriprep_test/workdir1/fmriprep_wf/single_subject_01_wf/anat_preproc_wf/surface_recon_wf/t1_2_fsnative_xfm/out.lta",
-          "subjects_dir": "/Users/dorota/fmriprep_test/fmriprep_test/output1/freesurfer/"
+          "source_file": "/fmriprep_test/workdir1/fmriprep_wf/single_subject_01_wf/func_preproc_ses_test_task_fingerfootlips_wf/bold_t1_trans_wf/merge/vol0000_xform-00000_merged.nii",
+          "t1_preproc": "/fmriprep_test/output1/fmriprep/sub-01/anat/sub-01_T1w_preproc.nii.gz",
+          "t1_2_fsnative_forward_transform": "/fmriprep_test/workdir1/fmriprep_wf/single_subject_01_wf/anat_preproc_wf/surface_recon_wf/t1_2_fsnative_xfm/out.lta",
+          "subjects_dir": "/fmriprep_test/output1/freesurfer/"
 }
 
-def test_neuro():
+Plugins = ["serial", "mp", "cf", "dask"]
+
+def select_target(subject_id, space):
+    """ Given a source subject ID and a target space, get the target subject ID """
+    return subject_id if space == 'fsnative' else space
+
+@pytest.mark.parametrize("plugin", Plugins)
+def test_neuro(change_dir, plugin):
 
     # wf = Workflow(name, mem_gb_node=DEFAULT_MEMORY_MIN_GB,
     #               inputs=['source_file', 't1_preproc', 'subject_id',
@@ -30,22 +52,15 @@ def test_neuro():
     #
     #dj: why do I need outputs?
 
-    pdb.set_trace()
-    wf = NewWorkflow(name=Name, inputs=Inputs, workingdir="test_neuro")
-    pdb.set_trace()
 
+    wf = NewWorkflow(name=Name, inputs=Inputs, workingdir="test_neuro_{}".format(plugin), print_val=False,
+                     wf_output_names=[("sampler", "out_file", "sampler_out"), ("targets", "out", "target_out")])
 
     # @interface
     # def select_target(subject_id, space):
     #     """ Given a source subject ID and a target space, get the target subject ID """
     #     return subject_id if space == 'fsnative' else space
 
-    # TODO: shouldn't map with subject?
-    def select_target(subject_id, space):
-        """ Given a source subject ID and a target space, get the target subject ID """
-        return subject_id if space == 'fsnative' else space
-
-    #select_target_interface = Function_Interface(select_target, ["out"])
 
 
     # wf.add('targets', select_target(subject_id=wf.inputs.subject_id))
@@ -54,22 +69,23 @@ def select_target(subject_id, space):
 
     #dj: don't have option in map to connect with wf input
 
-    wf.add(runnable=select_target, name="targets", subject_id="subject_id")\
+    wf.add(runnable=select_target, name="targets", subject_id="subject_id", output_names=["out"],
+           out_read=True, print_val=False)\
         .map_node(mapper="space", inputs={"space": [space for space in Inputs["output_spaces"]
                                                if space.startswith("fs")]})
 
-
     # wf.add('rename_src', Rename(format_string='%(subject)s',
     #                             keep_ext=True,
     #                             in_file=wf.inputs.source_file))
     #   .map('subject')
 
-    pdb.set_trace()
     wf.add(name='rename_src',
            runnable=Rename(format_string='%(subject)s', keep_ext=True),
-                                in_file="source_file", subject="subject_id")\
-        .map_node('subject')
-    pdb.set_trace()
+                                in_file="source_file", subject="subject_id",
+                                output_names=["out_file"],
+           print_val=False)\
+    #    .map_node('subject') #TODO: now it's only one subject
+
 
     # wf.add('resampling_xfm',
     #        fs.utils.LTAConvert(in_lta='identity.nofile',
@@ -83,9 +99,13 @@ def select_target(subject_id, space):
 
     wf.add(name='resampling_xfm',
            runnable=fs.utils.LTAConvert(in_lta='identity.nofile', out_lta=True),
-           source_file="source_file", target_file="t1_preproc")\
+           source_file="source_file", target_file="t1_preproc",
+           output_names=["out_lta"],
+           print_val=False)\
         .add(name='set_xfm_source', runnable=ConcatenateLTA(out_type='RAS2RAS'),
-             in_lta2="t1_2_fsnative_forward_transform", in_lta1="resampling_xfm.out_lta")
+            in_lta2="t1_2_fsnative_forward_transform", in_lta1="resampling_xfm.out_lta",
+            output_names=["out_file"], print_val=False)
+
 
 
     # wf.add('sampler',
@@ -106,12 +126,21 @@ def select_target(subject_id, space):
            runnable=fs.SampleToSurface(sampling_method='average', sampling_range=(0, 1, 0.2),
                                   sampling_units='frac', interp_method='trilinear',
                                   cortex_mask=True, override_reg_subj=True,
-                                  out_type='gii'),
+                                  out_type='gii'), print_val=False,
            subjects_dir="subjects_dir", subject_id="subject_id", reg_file="set_xfm_source.out_file",
-           target_subject="targets.out", source_file="rename_src.out_file")\
-        .map_node(mapper=[('source_file', 'target_subject'), 'hemi'], inputs={"hemi": ['lh', 'rh']})
+           target_subject="targets.out", source_file="rename_src.out_file", output_names=["out_file"])\
+        .map_node(mapper=['_targets', 'hemi'], inputs={"hemi": ['lh', 'rh']})
+
+
+    sub = Submitter(plugin=plugin, runnable=wf)
+    sub.run()
+    sub.close()
 
+    assert "target_out" in wf.output.keys()
+    assert len(list(wf.output["target_out"].keys())) == 2
 
+    assert "sampler_out" in wf.output.keys()
+    assert len(list(wf.output["sampler_out"].keys())) == 4
     # dj: no conditions
     # dj: no join for now
 
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index ad077e60aa..6670814b56 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -29,7 +29,7 @@
 
 from ...interfaces.base import (traits, TraitedSpec, TraitDictObject,
                                 TraitListObject)
-from ...utils.filemanip import save_json, makedirs, to_str
+from ...utils.filemanip import save_json, makedirs, to_str, loadpkl
 from .utils import (generate_expanded_graph, export_graph, write_workflow_prov,
                     write_workflow_resources, format_dot, topological_sort,
                     get_print_name, merge_dict, format_node)
@@ -1189,19 +1189,49 @@ def _collecting_input_el(self, ind):
         """collecting all inputs required to run the node (for specific state element)"""
         state_dict = self.state.state_values(ind)
         inputs_dict = {k: state_dict[k] for k in self._inputs.keys()}
+        if not self.print_val:
+            state_dict = self.state.state_ind(ind)
         # reading extra inputs that come from previous nodes
         for (from_node, from_socket, to_socket) in self.needed_outputs:
             dir_nm_el_from = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())
                                        if i in list(from_node._state_inputs.keys())])
             if not from_node.mapper:
                 dir_nm_el_from = ""
-            file_from = os.path.join(from_node.workingdir, dir_nm_el_from, from_socket+".txt")
-            with open(file_from) as f:
-                inputs_dict["{}.{}".format(self.name, to_socket)] = eval(f.readline())
+
+            if is_node(from_node) and is_current_interface(from_node.interface):
+                file_from = self._reading_ci_output(node=from_node, dir_nm_el=dir_nm_el_from, out_nm=from_socket)
+                if file_from and os.path.exists(file_from):
+                    inputs_dict["{}.{}".format(self.name, to_socket)] = file_from
+                else:
+                    raise Exception("{} doesnt exist".format(file_from))
+            else: # assuming here that I want to read the file (will not be used with the current interfaces)
+                file_from = os.path.join(from_node.workingdir, dir_nm_el_from, from_socket+".txt")
+                with open(file_from) as f:
+                    content = f.readline()
+                    try:
+                        inputs_dict["{}.{}".format(self.name, to_socket)] = eval(content)
+                    except NameError:
+                        inputs_dict["{}.{}".format(self.name, to_socket)] = content
+
         return state_dict, inputs_dict
 
+    def _reading_ci_output(self, dir_nm_el, out_nm, node=None):
+        """used for current interfaces: checking if the output exists and returns the path if it does"""
+        if not node:
+            node = self
+        result_pklfile = os.path.join(os.getcwd(), node.workingdir, dir_nm_el,
+                                      node.interface.nn.name, "result_{}.pklz".format(node.interface.nn.name))
+        if os.path.exists(result_pklfile):
+            out_file = getattr(loadpkl(result_pklfile).outputs, out_nm)
+            if os.path.exists(out_file):
+                return out_file
+            else:
+                return False
+        else:
+            return False
+
 
-   # checking if all outputs are saved
+    # checking if all outputs are saved
     @property
     def is_complete(self):
         # once _is_complete os True, this should not change
@@ -1245,20 +1275,17 @@ def __init__(self, name, interface, inputs=None, mapper=None, join_by=None,
         self.workingdir = workingdir
         self.interface = interface
 
-        # TODO: fixing mess with outputs_names etc.
         if is_function_interface(self.interface):
             # adding node name to the interface's name mapping
             self.interface.input_map = dict((key, "{}.{}".format(self.name, value))
                                              for (key, value) in self.interface.input_map.items())
-            # output names taken from interface output name
+            # list of output names taken from interface output name
             self.output_names = self.interface._output_nm
         elif is_current_interface(self.interface):
-            # TODO: assuming file_name, inter_key_out, node_key_out
-            # used to define name of the output file of current interface
+            # list of  interf_key_out
             self.output_names = output_names
-
-        self.print_val = print_val
-
+        if not self.output_names:
+            self.output_names = []
 
 
 
@@ -1307,17 +1334,10 @@ def run_interface_el(self, i, ind):
             logger.debug("Run fun interface el, output={}".format(output))
             self._writting_results_tmp(state_dict, dir_nm_el, output)
         elif is_current_interface(self.interface):
-            set_nm = {}
-            for out_nm in self.output_names:
-                if len(out_nm) == 2:
-                    out_nm = (out_nm[0], out_nm[1], out_nm[1])
-                if out_nm[2] not in self._output.keys():
-                    self._output[out_nm[2]] = {}
-                set_nm[out_nm[1]] = out_nm[0]
             if not self.mapper:
                 dir_nm_el = ""
             res = self.interface.run(inputs=inputs_dict, base_dir=os.path.join(os.getcwd(), self.workingdir),
-                                     set_out_nm=set_nm, dir_nm_el=dir_nm_el)
+                                     dir_nm_el=dir_nm_el)
 
         # TODO when join
         #if self._joinByKey:
@@ -1342,8 +1362,6 @@ def _writting_results_tmp(self, state_dict, dir_nm_el, output):
 
     def get_output(self):
         for key_out in self.output_names:
-            if is_current_interface(self.interface):
-                key_out, filename = key_out[-1], key_out[0]
             self._output[key_out] = {}
             for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):
                 if self.print_val:
@@ -1352,12 +1370,32 @@ def get_output(self):
                     state_dict = self.state.state_ind(ind)
                 dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
                 if self.mapper:
-                    self._output[key_out][dir_nm_el] = (state_dict, os.path.join(self.workingdir, dir_nm_el, key_out + ".txt"))
+                    if is_function_interface(self.interface):
+                        output = os.path.join(self.workingdir, dir_nm_el, key_out + ".txt")
+                        if self.interface.out_read:
+                            with open(output) as fout:
+                                content = fout.readline()
+                                try:
+                                    output = eval(content)
+                                except NameError:
+                                    output = content
+                        self._output[key_out][dir_nm_el] = (state_dict, output)
+                    elif is_current_interface(self.interface):
+                        self._output[key_out][dir_nm_el] = \
+                            (state_dict, (state_dict, self._reading_ci_output(dir_nm_el=dir_nm_el, out_nm=key_out)))
                 else:
                     if is_function_interface(self.interface):
-                        self._output[key_out] = (state_dict, os.path.join(self.workingdir, key_out + ".txt"))
+                        output = os.path.join(self.workingdir, key_out + ".txt")
+                        if self.interface.out_read:
+                            with open(output) as fout:
+                                try:
+                                    output = eval(fout.readline())
+                                except NewWorkflow:
+                                    output = fout.readline()
+                        self._output[key_out] = (state_dict, output)
                     elif is_current_interface(self.interface):
-                        self._output[key_out] = (state_dict, os.path.join(self.workingdir, self.interface.nn.name, filename))
+                        self._output[key_out] = \
+                            (state_dict, self._reading_ci_output(dir_nm_el="", out_nm=key_out))
         return self._output
 
 
@@ -1372,13 +1410,13 @@ def _check_all_results(self):
             dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
             if not self.mapper:
                 dir_nm_el = ""
+
             for key_out in self.output_names:
                 if is_function_interface(self.interface):
                     if not os.path.isfile(os.path.join(self.workingdir, dir_nm_el, key_out+".txt")):
                         return False
                 elif is_current_interface(self.interface):
-                    if not os.path.isfile(os.path.join(os.getcwd(), self.workingdir,
-                                                       dir_nm_el, self.interface.nn.name, key_out[0])):
+                    if not self._reading_ci_output(dir_nm_el, key_out):
                         return False
         self._is_complete = True
         return True
@@ -1581,17 +1619,17 @@ def add_nodes(self, nodes):
 
 
     def add(self, runnable, name=None, workingdir=None, inputs=None, output_names=None, mapper=None,
-            mem_gb=None, print_val=True, **kwargs):
+            mem_gb=None, print_val=True, out_read=False, **kwargs):
         if is_function(runnable):
             if not output_names:
                 output_names = ["out"]
-            interface = aux.Function_Interface(function=runnable, output_nm=output_names)
+            interface = aux.FunctionInterface(function=runnable, output_nm=output_names, out_read=out_read)
             if not name:
                 raise Exception("you have to specify name for the node")
             if not workingdir:
                 workingdir = name
             node = NewNode(interface=interface, workingdir=workingdir, name=name, inputs=inputs, mapper=mapper,
-                           other_mappers=self._node_mappers, mem_gb=mem_gb)
+                           other_mappers=self._node_mappers, mem_gb=mem_gb, print_val=print_val)
         elif is_function_interface(runnable) or is_current_interface(runnable):
             if not name:
                 raise Exception("you have to specify name for the node")
@@ -1695,7 +1733,7 @@ def is_function(obj):
     return hasattr(obj, '__call__')
 
 def is_function_interface(obj):
-    return type(obj) is aux.Function_Interface
+    return type(obj) is aux.FunctionInterface
 
 def is_current_interface(obj):
     return type(obj) is aux.CurrentInterface

From 2e3ea2a84f47013d3c85dd750a59e5d039803d33 Mon Sep 17 00:00:00 2001
From: Dorota Jarecka <djarecka@gmail.com>
Date: Mon, 8 Oct 2018 09:59:20 -0400
Subject: [PATCH 55/55] [skip ci] small naming change

---
 nipype/pipeline/engine/submitter.py |  2 +-
 nipype/pipeline/engine/workflows.py | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/nipype/pipeline/engine/submitter.py b/nipype/pipeline/engine/submitter.py
index 23f8619d1f..f24ab5a63b 100644
--- a/nipype/pipeline/engine/submitter.py
+++ b/nipype/pipeline/engine/submitter.py
@@ -115,7 +115,7 @@ def _run_workflow_el(self, workflow, i, ind, collect_inp=False):
         """running one internal workflow (if workflow has a mapper)"""
         # TODO: can I simplify and remove collect inp? where should it be?
         if collect_inp:
-            st_inputs, wf_inputs = workflow._collecting_input_el(ind)
+            st_inputs, wf_inputs = workflow.get_input_el(ind)
         else:
             wf_inputs = workflow.state.state_values(ind)
         if workflow.print_val:
diff --git a/nipype/pipeline/engine/workflows.py b/nipype/pipeline/engine/workflows.py
index 6670814b56..41b8414364 100644
--- a/nipype/pipeline/engine/workflows.py
+++ b/nipype/pipeline/engine/workflows.py
@@ -347,7 +347,7 @@ def remove_nodes(self, nodes):
     # Input-Output access
     @property
     def inputs(self):
-        return self._get_inputs()
+        return self.get_inputs()
 
     @property
     def outputs(self):
@@ -770,7 +770,7 @@ def _check_outputs(self, parameter):
     def _check_inputs(self, parameter):
         return self._has_attr(parameter, subtype='in')
 
-    def _get_inputs(self):
+    def get_inputs(self):
         """Returns the inputs of a workflow
 
         This function does not return any input ports that are already
@@ -1178,14 +1178,14 @@ def join(self, field, node=None):
     def checking_input_el(self, ind):
         """checking if all inputs are available (for specific state element)"""
         try:
-            self._collecting_input_el(ind)
+            self.get_input_el(ind)
             return True
         except: #TODO specify
             return False
 
 
     # dj: this is not used for a single node
-    def _collecting_input_el(self, ind):
+    def get_input_el(self, ind):
         """collecting all inputs required to run the node (for specific state element)"""
         state_dict = self.state.state_values(ind)
         inputs_dict = {k: state_dict[k] for k in self._inputs.keys()}
@@ -1320,7 +1320,7 @@ def inputs(self, inputs):
     def run_interface_el(self, i, ind):
         """ running interface one element generated from node_state."""
         logger.debug("Run interface el, name={}, i={}, ind={}".format(self.name, i, ind))
-        state_dict, inputs_dict = self._collecting_input_el(ind)
+        state_dict, inputs_dict = self.get_input_el(ind)
         if not self.print_val:
             state_dict = self.state.state_ind(ind)
         dir_nm_el = "_".join(["{}:{}".format(i, j) for i, j in list(state_dict.items())])
@@ -1361,6 +1361,7 @@ def _writting_results_tmp(self, state_dict, dir_nm_el, output):
 
 
     def get_output(self):
+        """collecting all outputs and updating self._output"""
         for key_out in self.output_names:
             self._output[key_out] = {}
             for (i, ind) in enumerate(itertools.product(*self.state.all_elements)):