From 7c98066cbf1b5cd9f23eb1dd585d29ae315bd8bd Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 10 Feb 2015 10:56:31 +0100 Subject: [PATCH 1/5] Add defaults input to JSONFileGrabber --- nipype/interfaces/io.py | 37 ++++++++++++------- .../tests/test_auto_JSONFileGrabber.py | 6 +-- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 4872310b42..247d9cd321 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -1804,8 +1804,9 @@ def _get_ssh_client(self): class JSONFileGrabberInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): - in_file = File(exists=True, mandatory=True, - desc='JSON source file') + in_file = File(exists=True, desc='JSON source file') + defaults = traits.Dict(desc=('JSON dictionary that sets default output' + 'values, overridden by values found in in_file')) class JSONFileGrabber(IOBase): @@ -1819,12 +1820,15 @@ class JSONFileGrabber(IOBase): >>> from nipype.interfaces.io import JSONFileGrabber >>> jsonSource = JSONFileGrabber() + >>> jsonSource.inputs.defaults = {'param1': 'overrideMe', 'param3': 1.0} + >>> res = jsonSource.run() + >>> res.outputs.get() + {'param1': 'overrideMe', 'param3': 1.0 } >>> jsonSource.inputs.in_file = 'jsongrabber.txt' >>> res = jsonSource.run() - >>> print res.outputs.param1 - exampleStr - >>> print res.outputs.param2 - 4 + >>> res.outputs.get() + {'param1': 'exampleStr', 'param2': 4, 'param3': 1.0} + """ input_spec = JSONFileGrabberInputSpec @@ -1834,15 +1838,22 @@ class JSONFileGrabber(IOBase): def _list_outputs(self): import json - with open(self.inputs.in_file, 'r') as f: - data = json.load(f) + outputs = {} + if isdefined(self.inputs.in_file): + with open(self.inputs.in_file, 'r') as f: + data = json.load(f) - if not isinstance(data, dict): - raise RuntimeError('JSON input has no dictionary structure') + if not isinstance(data, dict): + raise RuntimeError('JSON input has no dictionary structure') - outputs = {} - for key, value in data.iteritems(): - outputs[key] = value + for key, value in data.iteritems(): + outputs[key] = value + + if isdefined(self.inputs.defaults): + defaults = self.inputs.defaults + for key, value in defaults.iteritems(): + if key not in outputs.keys(): + outputs[key] = value return outputs diff --git a/nipype/interfaces/tests/test_auto_JSONFileGrabber.py b/nipype/interfaces/tests/test_auto_JSONFileGrabber.py index 64d6057f5a..f1872e791a 100644 --- a/nipype/interfaces/tests/test_auto_JSONFileGrabber.py +++ b/nipype/interfaces/tests/test_auto_JSONFileGrabber.py @@ -3,11 +3,11 @@ from nipype.interfaces.io import JSONFileGrabber def test_JSONFileGrabber_inputs(): - input_map = dict(ignore_exception=dict(nohash=True, + input_map = dict(defaults=dict(), + ignore_exception=dict(nohash=True, usedefault=True, ), - in_file=dict(mandatory=True, - ), + in_file=dict(), ) inputs = JSONFileGrabber.input_spec() From fd9728479ba8c2485162e000d2d490c2aea1dd6c Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 10 Feb 2015 11:02:57 +0100 Subject: [PATCH 2/5] Update CHANGES --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index 6888e32207..93c8865190 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ Next release ============ +* ENH: Improve JSON interfaces: default settings when reading and consistent output creation + when writing (https://github.com/nipy/nipype/pull/1047) * ENH: New io interfaces for JSON files reading/writing (https://github.com/nipy/nipype/pull/1020) * ENH: Enhanced openfmri script to support freesurfer linkage (https://github.com/nipy/nipype/pull/1037) * BUG: matplotlib is supposed to be optional (https://github.com/nipy/nipype/pull/1003) From 098f96f7f324d7aa99821fe6a6f02fcc9c90715c Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 10 Feb 2015 11:36:24 +0100 Subject: [PATCH 3/5] fix doctests output --- nipype/interfaces/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 247d9cd321..1cad68298b 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -1820,14 +1820,14 @@ class JSONFileGrabber(IOBase): >>> from nipype.interfaces.io import JSONFileGrabber >>> jsonSource = JSONFileGrabber() - >>> jsonSource.inputs.defaults = {'param1': 'overrideMe', 'param3': 1.0} + >>> jsonSource.inputs.defaults = {'param1': u'overrideMe', 'param3': 1.0} >>> res = jsonSource.run() >>> res.outputs.get() - {'param1': 'overrideMe', 'param3': 1.0 } + {'param3': 1.0, 'param1': u'overrideMe'} >>> jsonSource.inputs.in_file = 'jsongrabber.txt' >>> res = jsonSource.run() >>> res.outputs.get() - {'param1': 'exampleStr', 'param2': 4, 'param3': 1.0} + {'param3': 1.0, 'param2': 4, 'param1': u'exampleStr'} """ From 02a8de6c2fc07ca79475ddc4616270b73f80550a Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 11 Feb 2015 14:09:35 +0100 Subject: [PATCH 4/5] add dot nesting for inputs in JSONSink --- nipype/interfaces/io.py | 65 ++++++++++++++++++++++-------- nipype/interfaces/tests/test_io.py | 38 +++++++++++++++++ 2 files changed, 87 insertions(+), 16 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 1cad68298b..002582018c 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -1860,7 +1860,19 @@ def _list_outputs(self): class JSONFileSinkInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): out_file = File(desc='JSON sink file') - in_dict = traits.Dict(desc='input JSON dictionary') + in_dict = traits.Dict(value={}, usedefault=True, + desc='input JSON dictionary') + _outputs = traits.Dict(value={}, usedefault=True) + + def __setattr__(self, key, value): + if key not in self.copyable_trait_names(): + if not isdefined(value): + super(JSONFileSinkInputSpec, self).__setattr__(key, value) + self._outputs[key] = value + else: + if key in self._outputs: + self._outputs[key] = value + super(JSONFileSinkInputSpec, self).__setattr__(key, value) class JSONFileSinkOutputSpec(TraitedSpec): @@ -1869,7 +1881,10 @@ class JSONFileSinkOutputSpec(TraitedSpec): class JSONFileSink(IOBase): - """ Very simple frontend for storing values into a JSON file. + """ + Very simple frontend for storing values into a JSON file. + Entries already existing in in_dict will be overridden by matching + entries dynamically added as inputs. .. warning:: @@ -1896,34 +1911,52 @@ class JSONFileSink(IOBase): input_spec = JSONFileSinkInputSpec output_spec = JSONFileSinkOutputSpec - def __init__(self, input_names=[], **inputs): + def __init__(self, infields=[], force_run=True, **inputs): super(JSONFileSink, self).__init__(**inputs) - self._input_names = filename_to_list(input_names) - add_traits(self.inputs, [name for name in self._input_names]) + self._input_names = infields + + undefined_traits = {} + for key in infields: + self.inputs.add_trait(key, traits.Any) + self.inputs._outputs[key] = Undefined + undefined_traits[key] = Undefined + self.inputs.trait_set(trait_change_notify=False, **undefined_traits) + + if force_run: + self._always_run = True + + def _process_name(self, name, val): + if '.' in name: + newkeys = name.split('.') + name = newkeys.pop(0) + nested_dict = {newkeys.pop(): val} + + for nk in reversed(newkeys): + nested_dict = {nk: nested_dict} + val = nested_dict + + return name, val def _list_outputs(self): import json import os.path as op + if not isdefined(self.inputs.out_file): out_file = op.abspath('datasink.json') else: out_file = self.inputs.out_file - out_dict = dict() + out_dict = self.inputs.in_dict - if isdefined(self.inputs.in_dict): - if isinstance(self.inputs.in_dict, dict): - out_dict = self.inputs.in_dict - else: - for name in self._input_names: - val = getattr(self.inputs, name) - val = val if isdefined(val) else 'undefined' - out_dict[name] = val + # Overwrite in_dict entries automatically + for key, val in self.inputs._outputs.items(): + if not isdefined(val) or key == 'trait_added': + continue + key, val = self._process_name(key, val) + out_dict[key] = val with open(out_file, 'w') as f: json.dump(out_dict, f) outputs = self.output_spec().get() outputs['out_file'] = out_file return outputs - - diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 07fcb89040..65932492d3 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -238,3 +238,41 @@ def test_freesurfersource(): yield assert_equal, fss.inputs.hemi, 'both' yield assert_equal, fss.inputs.subject_id, Undefined yield assert_equal, fss.inputs.subjects_dir, Undefined + + +def test_jsonsink(): + import json + import os + + ds = nio.JSONFileSink() + yield assert_equal, ds.inputs._outputs, {} + ds = nio.JSONFileSink(in_dict={'foo': 'var'}) + yield assert_equal, ds.inputs.in_dict, {'foo': 'var'} + ds = nio.JSONFileSink(infields=['test']) + yield assert_true, 'test' in ds.inputs.copyable_trait_names() + + curdir = os.getcwd() + outdir = mkdtemp() + os.chdir(outdir) + js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'}) + js.inputs.new_entry = 'someValue' + setattr(js.inputs, 'contrasts.alt', 'someNestedValue') + res = js.run() + + with open(res.outputs['out_file'], 'r') as f: + data = json.load(f) + yield assert_true, data == {"contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"} + + js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'}) + js.inputs.new_entry = 'someValue' + js.inputs.test = 'testInfields' + setattr(js.inputs, 'contrasts.alt', 'someNestedValue') + res = js.run() + + with open(res.outputs['out_file'], 'r') as f: + data = json.load(f) + yield assert_true, data == {"test": "testInfields", "contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"} + + os.chdir(curdir) + shutil.rmtree(outdir) + From a094ce4fbce017e43e3e4da00e6d6a1a8c9debf9 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 11 Feb 2015 14:52:43 +0100 Subject: [PATCH 5/5] add regression test --- nipype/interfaces/tests/test_auto_JSONFileSink.py | 7 +++++-- nipype/interfaces/tests/test_io.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nipype/interfaces/tests/test_auto_JSONFileSink.py b/nipype/interfaces/tests/test_auto_JSONFileSink.py index fb95f03017..7c8cd80f98 100644 --- a/nipype/interfaces/tests/test_auto_JSONFileSink.py +++ b/nipype/interfaces/tests/test_auto_JSONFileSink.py @@ -3,10 +3,13 @@ from nipype.interfaces.io import JSONFileSink def test_JSONFileSink_inputs(): - input_map = dict(ignore_exception=dict(nohash=True, + input_map = dict(_outputs=dict(usedefault=True, + ), + ignore_exception=dict(nohash=True, usedefault=True, ), - in_dict=dict(), + in_dict=dict(usedefault=True, + ), out_file=dict(), ) inputs = JSONFileSink.input_spec() diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 65932492d3..072f304496 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -259,7 +259,7 @@ def test_jsonsink(): setattr(js.inputs, 'contrasts.alt', 'someNestedValue') res = js.run() - with open(res.outputs['out_file'], 'r') as f: + with open(res.outputs.out_file, 'r') as f: data = json.load(f) yield assert_true, data == {"contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"} @@ -269,7 +269,7 @@ def test_jsonsink(): setattr(js.inputs, 'contrasts.alt', 'someNestedValue') res = js.run() - with open(res.outputs['out_file'], 'r') as f: + with open(res.outputs.out_file, 'r') as f: data = json.load(f) yield assert_true, data == {"test": "testInfields", "contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"}