From a1b903c6009d07a6c8e922f86745f2f536e52df5 Mon Sep 17 00:00:00 2001 From: adelavega Date: Fri, 1 Sep 2017 11:15:49 -0700 Subject: [PATCH 01/19] Added BIDSDataGrabber, test data and tests --- nipype/interfaces/bids.py | 147 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 148 insertions(+) create mode 100644 nipype/interfaces/bids.py diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py new file mode 100644 index 0000000000..343542aa09 --- /dev/null +++ b/nipype/interfaces/bids.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +""" Set of interfaces that allow interaction with BIDS data. Currently + available interfaces are: + + BIDSDataGrabber: Query data from BIDS dataset using pybids grabbids. + + Change directory to provide relative paths for doctests + >>> import os + >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) + >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) + >>> os.chdir(datadir) + +""" + +from .base import (traits, + DynamicTraitedSpec, + BaseInterface, + isdefined, + Str, + Undefined) + +from bids.grabbids import BIDSLayout + + +class BIDSDataGrabberInputSpec(DynamicTraitedSpec): + base_dir = traits.Directory(exists=True, + desc='Path to BIDS Directory.', + mandatory=True) + output_query = traits.Dict(key_trait=Str, + value_trait=traits.Dict, + desc='Queries for outfield outputs') + return_type = traits.Enum('filename', 'namedtuple', usedefault=True) + + +class BIDSDataGrabber(BaseInterface): + + """ BIDS datagrabber module that wraps around pybids to allow arbitrary + querying of BIDS datasets. + + Examples + -------- + + >>> from nipype.interfaces.bids import BIDSDataGrabber + >>> from os.path import basename + >>> import pprint + + Select all files from a BIDS project + + >>> bg = BIDSDataGrabber() + >>> bg.inputs.base_dir = 'ds005/' + >>> results = bg.run() + >>> pprint.pprint(len(results.outputs.outfield)) # doctest: +ALLOW_UNICODE + 116 + + Using dynamically created, user-defined input fields, + filter files based on BIDS entities. + + >>> bg = BIDSDataGrabber(infields = ['subject', 'run']) + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> bg.inputs.run = '01' + >>> results = bg.run() + >>> basename(results.outputs.outfield[0]) # doctest: +ALLOW_UNICODE + 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' + + Using user-defined output fields, return different types of outputs, + filtered on common entities + filter files based on BIDS entities. + + >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['func', 'anat']) + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> bg.inputs.output_query['func'] = dict(modality='func') + >>> bg.inputs.output_query['anat'] = dict(modality='anat') + >>> results = bg.run() + >>> basename(results.outputs.func[0]) # doctest: +ALLOW_UNICODE + 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' + + >>> basename(results.outputs.anat[0]) # doctest: +ALLOW_UNICODE + 'sub-01_T1w.nii.gz' + """ + input_spec = BIDSDataGrabberInputSpec + output_spec = DynamicTraitedSpec + _always_run = True + + def __init__(self, infields=None, outfields=None, **kwargs): + """ + Parameters + ---------- + infields : list of str + Indicates the input fields to be dynamically created + + outfields: list of str + Indicates output fields to be dynamically created + + """ + if not outfields: + outfields = [] + if not infields: + infields = [] + + super(BIDSDataGrabber, self).__init__(**kwargs) + undefined_traits = {} + # used for mandatory inputs check + self._infields = infields + self._outfields = outfields + for key in infields: + self.inputs.add_trait(key, traits.Any) + undefined_traits[key] = Undefined + + if not isdefined(self.inputs.output_query): + self.inputs.output_query = {} + + self.inputs.trait_set(trait_change_notify=False, **undefined_traits) + + def _run_interface(self, runtime): + return runtime + + def _list_outputs(self): + if not self._outfields: + self._outfields = ['outfield'] + self.inputs.output_query = {'outfield' : {}} + else: + for key in self._outfields: + if key not in self.inputs.output_query: + raise ValueError("Define query for all outputs") + + for key in self._infields: + value = getattr(self.inputs, key) + if not isdefined(value): + msg = "%s requires a value for input '%s' because" \ + " it was listed in 'infields'" % \ + (self.__class__.__name__, key) + raise ValueError(msg) + + layout = BIDSLayout(self.inputs.base_dir) + + filters = {i: getattr(self.inputs, i) for i in self._infields} + + outputs = {} + for key, query in self.inputs.output_query.items(): + outputs[key] = layout.get( + **dict(query.items() | filters.items()), + return_type='file') + return outputs diff --git a/requirements.txt b/requirements.txt index bcd3ab2fef..3e9009a21a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ configparser pytest>=3.0 mock pydotplus +pybids==0.3 packaging From da44acf74365dfe49668abb39406860a1d4805b1 Mon Sep 17 00:00:00 2001 From: adelavega Date: Fri, 1 Sep 2017 11:50:38 -0700 Subject: [PATCH 02/19] Edited contributors --- .zenodo.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.zenodo.json b/.zenodo.json index b76c0e6313..ad164e2795 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -523,7 +523,12 @@ "affiliation": "University of Amsterdam", "name": "Lukas Snoek", "orcid": "0000-0001-8972-204X" - } + }, + { + "affiliation": "University of Texas at Austin", + "name": "De La Vega, Alejandro", + "orcid": "0000-0001-9062-3778" + }, ], "keywords": [ "neuroimaging", From 8ed1c44ea23884b6d2a597d30b87c31a10452402 Mon Sep 17 00:00:00 2001 From: adelavega Date: Mon, 4 Sep 2017 15:42:06 -0700 Subject: [PATCH 03/19] Removed data, using test data from pybids package --- nipype/interfaces/bids.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py index 343542aa09..317c871468 100644 --- a/nipype/interfaces/bids.py +++ b/nipype/interfaces/bids.py @@ -8,8 +8,9 @@ Change directory to provide relative paths for doctests >>> import os - >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) - >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) + >>> import bids + >>> filepath = os.path.realpath(os.path.dirname(bids.__file__)) + >>> datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/')) >>> os.chdir(datadir) """ @@ -51,7 +52,7 @@ class BIDSDataGrabber(BaseInterface): >>> bg = BIDSDataGrabber() >>> bg.inputs.base_dir = 'ds005/' >>> results = bg.run() - >>> pprint.pprint(len(results.outputs.outfield)) # doctest: +ALLOW_UNICODE + >>> len(results.outputs.outfield) # doctest: +ALLOW_UNICODE 116 Using dynamically created, user-defined input fields, From 52cf9e4977d3b4371d26222562b2ae0c03233caf Mon Sep 17 00:00:00 2001 From: adelavega Date: Mon, 4 Sep 2017 16:14:03 -0700 Subject: [PATCH 04/19] Check if pybids is installed when importing --- nipype/interfaces/bids.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py index 317c871468..5e772569b2 100644 --- a/nipype/interfaces/bids.py +++ b/nipype/interfaces/bids.py @@ -22,7 +22,12 @@ Str, Undefined) -from bids.grabbids import BIDSLayout +try: + from bids.grabbids import BIDSLayout +except ImportError: + have_pybids = False +else: + have_pybids = True class BIDSDataGrabberInputSpec(DynamicTraitedSpec): @@ -117,6 +122,9 @@ def __init__(self, infields=None, outfields=None, **kwargs): self.inputs.trait_set(trait_change_notify=False, **undefined_traits) def _run_interface(self, runtime): + if not have_pybids: + raise ImportError("The BIDSEventsGrabber interface requires pybids." + " Please make sure it is installed.") return runtime def _list_outputs(self): From 6a4ce640570c2fdecfd87dce14998d4744fb0fba Mon Sep 17 00:00:00 2001 From: adelavega Date: Mon, 4 Sep 2017 18:56:03 -0700 Subject: [PATCH 05/19] Tried to fix reqs and travis --- .travis.yml | 6 +++--- nipype/info.py | 1 + requirements.txt | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f97f48dddb..4cd6d7bb7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,9 @@ python: - 3.5 - 3.6 env: -- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" -- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" +- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,pybids" +- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler,pybids" +- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit,pybids" before_install: - function apt_inst { if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && diff --git a/nipype/info.py b/nipype/info.py index 9db9a02abd..d1a6501252 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -160,6 +160,7 @@ def get_nipype_gitversion(): 'profiler': ['psutil'], 'duecredit': ['duecredit'], 'xvfbwrapper': ['xvfbwrapper'], + 'pybids' : ['pybids'] # 'mesh': ['mayavi'] # Enable when it works } diff --git a/requirements.txt b/requirements.txt index 3e9009a21a..bcd3ab2fef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,4 @@ configparser pytest>=3.0 mock pydotplus -pybids==0.3 packaging From 72bd8e267e838cc91693c6eb39eb9ab01208c54f Mon Sep 17 00:00:00 2001 From: adelavega Date: Fri, 8 Sep 2017 11:56:26 -0700 Subject: [PATCH 06/19] Added mandatory option for outfieds --- nipype/interfaces/bids.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py index 5e772569b2..db2e7c6f44 100644 --- a/nipype/interfaces/bids.py +++ b/nipype/interfaces/bids.py @@ -29,6 +29,7 @@ else: have_pybids = True +from warnings import warn class BIDSDataGrabberInputSpec(DynamicTraitedSpec): base_dir = traits.Directory(exists=True, @@ -37,6 +38,9 @@ class BIDSDataGrabberInputSpec(DynamicTraitedSpec): output_query = traits.Dict(key_trait=Str, value_trait=traits.Dict, desc='Queries for outfield outputs') + raise_on_empty = traits.Bool(True, usedefault=True, + desc='Generate exception if list is empty ' + 'for a given field') return_type = traits.Enum('filename', 'namedtuple', usedefault=True) @@ -58,7 +62,7 @@ class BIDSDataGrabber(BaseInterface): >>> bg.inputs.base_dir = 'ds005/' >>> results = bg.run() >>> len(results.outputs.outfield) # doctest: +ALLOW_UNICODE - 116 + 135 Using dynamically created, user-defined input fields, filter files based on BIDS entities. @@ -99,8 +103,8 @@ def __init__(self, infields=None, outfields=None, **kwargs): Indicates the input fields to be dynamically created outfields: list of str - Indicates output fields to be dynamically created - + Indicates output fields to be dynamically created. + If no matching items, returns Undefined. """ if not outfields: outfields = [] @@ -150,7 +154,18 @@ def _list_outputs(self): outputs = {} for key, query in self.inputs.output_query.items(): - outputs[key] = layout.get( - **dict(query.items() | filters.items()), - return_type='file') + args = query.copy() + args.update(filters) + filelist = layout.get(return_type='file', + **args) + if len(filelist) == 0: + msg = 'Output key: %s returned no files' % ( + key) + if self.inputs.raise_on_empty: + raise IOError(msg) + else: + warn(msg) + filelist = Undefined + else: + outputs[key] = filelist return outputs From dc64c7284cb972dd288883515cd11999cb22e4fe Mon Sep 17 00:00:00 2001 From: adelavega Date: Wed, 13 Sep 2017 19:24:58 -0500 Subject: [PATCH 07/19] Install pybids from github in travis --- .travis.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4cd6d7bb7f..73d42a15f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,9 @@ python: - 3.5 - 3.6 env: -- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,pybids" -- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler,pybids" -- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit,pybids" +- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" +- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" before_install: - function apt_inst { if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi && @@ -36,7 +36,12 @@ before_install: conda install python=${TRAVIS_PYTHON_VERSION} && conda config --add channels conda-forge && conda install -y nipype icu && - rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; } + rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; + pushd $HOME; + git clone https://github.com/INCF/pybids.git; + cd pybids; + pip install -e .; + popd; } # Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi - travis_retry apt_inst - travis_retry conda_inst From 406c72914e67e147ea484b6684d9be5fce463eb9 Mon Sep 17 00:00:00 2001 From: adelavega Date: Wed, 13 Sep 2017 19:26:05 -0500 Subject: [PATCH 08/19] Fix docstring, and Directory import --- nipype/interfaces/bids.py | 109 +++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py index db2e7c6f44..cf2e63d545 100644 --- a/nipype/interfaces/bids.py +++ b/nipype/interfaces/bids.py @@ -2,21 +2,22 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """ Set of interfaces that allow interaction with BIDS data. Currently - available interfaces are: +available interfaces are: - BIDSDataGrabber: Query data from BIDS dataset using pybids grabbids. +BIDSDataGrabber: Query data from BIDS dataset using pybids grabbids. - Change directory to provide relative paths for doctests - >>> import os - >>> import bids - >>> filepath = os.path.realpath(os.path.dirname(bids.__file__)) - >>> datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/')) - >>> os.chdir(datadir) +Change directory to provide relative paths for doctests +>>> import os +>>> import bids +>>> filepath = os.path.realpath(os.path.dirname(bids.__file__)) +>>> datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/')) +>>> os.chdir(datadir) """ from .base import (traits, DynamicTraitedSpec, + Directory, BaseInterface, isdefined, Str, @@ -32,9 +33,9 @@ from warnings import warn class BIDSDataGrabberInputSpec(DynamicTraitedSpec): - base_dir = traits.Directory(exists=True, - desc='Path to BIDS Directory.', - mandatory=True) + base_dir = Directory(exists=True, + desc='Path to BIDS Directory.', + mandatory=True) output_query = traits.Dict(key_trait=Str, value_trait=traits.Dict, desc='Queries for outfield outputs') @@ -47,49 +48,49 @@ class BIDSDataGrabberInputSpec(DynamicTraitedSpec): class BIDSDataGrabber(BaseInterface): """ BIDS datagrabber module that wraps around pybids to allow arbitrary - querying of BIDS datasets. - - Examples - -------- - - >>> from nipype.interfaces.bids import BIDSDataGrabber - >>> from os.path import basename - >>> import pprint - - Select all files from a BIDS project - - >>> bg = BIDSDataGrabber() - >>> bg.inputs.base_dir = 'ds005/' - >>> results = bg.run() - >>> len(results.outputs.outfield) # doctest: +ALLOW_UNICODE - 135 - - Using dynamically created, user-defined input fields, - filter files based on BIDS entities. - - >>> bg = BIDSDataGrabber(infields = ['subject', 'run']) - >>> bg.inputs.base_dir = 'ds005/' - >>> bg.inputs.subject = '01' - >>> bg.inputs.run = '01' - >>> results = bg.run() - >>> basename(results.outputs.outfield[0]) # doctest: +ALLOW_UNICODE - 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' - - Using user-defined output fields, return different types of outputs, - filtered on common entities - filter files based on BIDS entities. - - >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['func', 'anat']) - >>> bg.inputs.base_dir = 'ds005/' - >>> bg.inputs.subject = '01' - >>> bg.inputs.output_query['func'] = dict(modality='func') - >>> bg.inputs.output_query['anat'] = dict(modality='anat') - >>> results = bg.run() - >>> basename(results.outputs.func[0]) # doctest: +ALLOW_UNICODE - 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' - - >>> basename(results.outputs.anat[0]) # doctest: +ALLOW_UNICODE - 'sub-01_T1w.nii.gz' + querying of BIDS datasets. + + Examples + -------- + + >>> from nipype.interfaces.bids import BIDSDataGrabber + >>> from os.path import basename + >>> import pprint + + Select all files from a BIDS project + + >>> bg = BIDSDataGrabber() + >>> bg.inputs.base_dir = 'ds005/' + >>> results = bg.run() + >>> len(results.outputs.outfield) # doctest: +ALLOW_UNICODE + 135 + + Using dynamically created, user-defined input fields, + filter files based on BIDS entities. + + >>> bg = BIDSDataGrabber(infields = ['subject', 'run']) + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> bg.inputs.run = '01' + >>> results = bg.run() + >>> basename(results.outputs.outfield[0]) # doctest: +ALLOW_UNICODE + 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' + + Using user-defined output fields, return different types of outputs, + filtered on common entities + filter files based on BIDS entities. + + >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['func', 'anat']) + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> bg.inputs.output_query['func'] = dict(modality='func') + >>> bg.inputs.output_query['anat'] = dict(modality='anat') + >>> results = bg.run() + >>> basename(results.outputs.func[0]) # doctest: +ALLOW_UNICODE + 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' + + >>> basename(results.outputs.anat[0]) # doctest: +ALLOW_UNICODE + 'sub-01_T1w.nii.gz' """ input_spec = BIDSDataGrabberInputSpec output_spec = DynamicTraitedSpec From 0915a9d0ec055ebb0115f631baa78d8d15563918 Mon Sep 17 00:00:00 2001 From: adelavega Date: Wed, 13 Sep 2017 19:27:19 -0500 Subject: [PATCH 09/19] Use logger --- nipype/interfaces/bids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py index cf2e63d545..190608fb2a 100644 --- a/nipype/interfaces/bids.py +++ b/nipype/interfaces/bids.py @@ -14,7 +14,7 @@ >>> os.chdir(datadir) """ - +from .. import logging from .base import (traits, DynamicTraitedSpec, Directory, @@ -30,7 +30,7 @@ else: have_pybids = True -from warnings import warn +LOGGER = logging.getLogger('workflows') class BIDSDataGrabberInputSpec(DynamicTraitedSpec): base_dir = Directory(exists=True, @@ -165,7 +165,7 @@ def _list_outputs(self): if self.inputs.raise_on_empty: raise IOError(msg) else: - warn(msg) + LOGGER.warning(msg) filelist = Undefined else: outputs[key] = filelist From 651bc85c831e32bbc5e4f1ccfb2c08275f208cae Mon Sep 17 00:00:00 2001 From: adelavega Date: Wed, 13 Sep 2017 20:37:12 -0500 Subject: [PATCH 10/19] Smart defaults --- nipype/interfaces/bids.py | 102 +++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py index 190608fb2a..1bb15bc6bf 100644 --- a/nipype/interfaces/bids.py +++ b/nipype/interfaces/bids.py @@ -14,6 +14,7 @@ >>> os.chdir(datadir) """ +from os.path import join, dirname from .. import logging from .base import (traits, DynamicTraitedSpec, @@ -24,7 +25,8 @@ Undefined) try: - from bids.grabbids import BIDSLayout + from bids import grabbids as gb + import json except ImportError: have_pybids = False else: @@ -57,40 +59,34 @@ class BIDSDataGrabber(BaseInterface): >>> from os.path import basename >>> import pprint - Select all files from a BIDS project + By default, the BIDSDataGrabber fetches anatomical and functional images + from a project, and makes BIDS entities (e.g. subject) available for + filtering outputs. >>> bg = BIDSDataGrabber() >>> bg.inputs.base_dir = 'ds005/' - >>> results = bg.run() - >>> len(results.outputs.outfield) # doctest: +ALLOW_UNICODE - 135 - - Using dynamically created, user-defined input fields, - filter files based on BIDS entities. - - >>> bg = BIDSDataGrabber(infields = ['subject', 'run']) - >>> bg.inputs.base_dir = 'ds005/' >>> bg.inputs.subject = '01' - >>> bg.inputs.run = '01' >>> results = bg.run() - >>> basename(results.outputs.outfield[0]) # doctest: +ALLOW_UNICODE + >>> basename(results.outputs.anat[0]) # doctest: +ALLOW_UNICODE + 'sub-01_T1w.nii.gz' + + >>> basename(results.outputs.func[0]) # doctest: +ALLOW_UNICODE 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' - Using user-defined output fields, return different types of outputs, - filtered on common entities - filter files based on BIDS entities. - >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['func', 'anat']) + Dynamically created, user-defined output fields can also be defined to + return different types of outputs from the same project. All outputs + are filtered on common entities, which can be explicitly defined as + infields. + + >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['dwi']) >>> bg.inputs.base_dir = 'ds005/' >>> bg.inputs.subject = '01' - >>> bg.inputs.output_query['func'] = dict(modality='func') - >>> bg.inputs.output_query['anat'] = dict(modality='anat') + >>> bg.inputs.output_query['dwi'] = dict(modality='dwi') >>> results = bg.run() - >>> basename(results.outputs.func[0]) # doctest: +ALLOW_UNICODE - 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' + >>> basename(results.outputs.dwi[0]) # doctest: +ALLOW_UNICODE + 'sub-01_dwi.nii.gz' - >>> basename(results.outputs.anat[0]) # doctest: +ALLOW_UNICODE - 'sub-01_T1w.nii.gz' """ input_spec = BIDSDataGrabberInputSpec output_spec = DynamicTraitedSpec @@ -107,51 +103,53 @@ def __init__(self, infields=None, outfields=None, **kwargs): Indicates output fields to be dynamically created. If no matching items, returns Undefined. """ - if not outfields: - outfields = [] - if not infields: - infields = [] - super(BIDSDataGrabber, self).__init__(**kwargs) - undefined_traits = {} - # used for mandatory inputs check + if not have_pybids: + raise ImportError("The BIDSEventsGrabber interface requires pybids." + " Please make sure it is installed.") + + # If outfields is None use anat and func as default + if outfields is None: + outfields = ['func', 'anat'] + self.inputs.output_query = { + "func": {"modality": "func"}, + "anat": {"modality": "anat"}} + else: + self.inputs.output_query = {} + + # If infields is None, use all BIDS entities + if infields is None: + bids_config = join(dirname(gb.__file__), 'config', 'bids.json') + bids_config = json.load(open(bids_config, 'r')) + infields = [i['name'] for i in bids_config['entities']] + self._infields = infields self._outfields = outfields + + # used for mandatory inputs check + undefined_traits = {} for key in infields: self.inputs.add_trait(key, traits.Any) undefined_traits[key] = Undefined - if not isdefined(self.inputs.output_query): - self.inputs.output_query = {} - self.inputs.trait_set(trait_change_notify=False, **undefined_traits) def _run_interface(self, runtime): - if not have_pybids: - raise ImportError("The BIDSEventsGrabber interface requires pybids." - " Please make sure it is installed.") return runtime def _list_outputs(self): - if not self._outfields: - self._outfields = ['outfield'] - self.inputs.output_query = {'outfield' : {}} - else: - for key in self._outfields: - if key not in self.inputs.output_query: - raise ValueError("Define query for all outputs") + layout = gb.BIDSLayout(self.inputs.base_dir) + + for key in self._outfields: + if key not in self.inputs.output_query: + raise ValueError("Define query for all outputs") + # If infield is not given nm input value, silently ignore + filters = {} for key in self._infields: value = getattr(self.inputs, key) - if not isdefined(value): - msg = "%s requires a value for input '%s' because" \ - " it was listed in 'infields'" % \ - (self.__class__.__name__, key) - raise ValueError(msg) - - layout = BIDSLayout(self.inputs.base_dir) - - filters = {i: getattr(self.inputs, i) for i in self._infields} + if isdefined(value): + filters[key] = value outputs = {} for key, query in self.inputs.output_query.items(): From dd03863d9df998f8e2c06c066c7ff80f5f23a51d Mon Sep 17 00:00:00 2001 From: adelavega Date: Wed, 13 Sep 2017 21:04:30 -0500 Subject: [PATCH 11/19] Try fix CI --- docker/base.Dockerfile | 5 +- nipype/interfaces/bids.py | 170 -------------------------------------- 2 files changed, 4 insertions(+), 171 deletions(-) delete mode 100644 nipype/interfaces/bids.py diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index a5b5134c2c..3de42492ce 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -148,5 +148,8 @@ ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \ SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \ FORCE_SPMMCR=1 -WORKDIR /work +# Install pybids +RUN git clone https://github.com/INCF/pybids.git +RUN pip install -e . +WORKDIR /work diff --git a/nipype/interfaces/bids.py b/nipype/interfaces/bids.py deleted file mode 100644 index 1bb15bc6bf..0000000000 --- a/nipype/interfaces/bids.py +++ /dev/null @@ -1,170 +0,0 @@ -# -*- coding: utf-8 -*- -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -""" Set of interfaces that allow interaction with BIDS data. Currently -available interfaces are: - -BIDSDataGrabber: Query data from BIDS dataset using pybids grabbids. - -Change directory to provide relative paths for doctests ->>> import os ->>> import bids ->>> filepath = os.path.realpath(os.path.dirname(bids.__file__)) ->>> datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/')) ->>> os.chdir(datadir) - -""" -from os.path import join, dirname -from .. import logging -from .base import (traits, - DynamicTraitedSpec, - Directory, - BaseInterface, - isdefined, - Str, - Undefined) - -try: - from bids import grabbids as gb - import json -except ImportError: - have_pybids = False -else: - have_pybids = True - -LOGGER = logging.getLogger('workflows') - -class BIDSDataGrabberInputSpec(DynamicTraitedSpec): - base_dir = Directory(exists=True, - desc='Path to BIDS Directory.', - mandatory=True) - output_query = traits.Dict(key_trait=Str, - value_trait=traits.Dict, - desc='Queries for outfield outputs') - raise_on_empty = traits.Bool(True, usedefault=True, - desc='Generate exception if list is empty ' - 'for a given field') - return_type = traits.Enum('filename', 'namedtuple', usedefault=True) - - -class BIDSDataGrabber(BaseInterface): - - """ BIDS datagrabber module that wraps around pybids to allow arbitrary - querying of BIDS datasets. - - Examples - -------- - - >>> from nipype.interfaces.bids import BIDSDataGrabber - >>> from os.path import basename - >>> import pprint - - By default, the BIDSDataGrabber fetches anatomical and functional images - from a project, and makes BIDS entities (e.g. subject) available for - filtering outputs. - - >>> bg = BIDSDataGrabber() - >>> bg.inputs.base_dir = 'ds005/' - >>> bg.inputs.subject = '01' - >>> results = bg.run() - >>> basename(results.outputs.anat[0]) # doctest: +ALLOW_UNICODE - 'sub-01_T1w.nii.gz' - - >>> basename(results.outputs.func[0]) # doctest: +ALLOW_UNICODE - 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' - - - Dynamically created, user-defined output fields can also be defined to - return different types of outputs from the same project. All outputs - are filtered on common entities, which can be explicitly defined as - infields. - - >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['dwi']) - >>> bg.inputs.base_dir = 'ds005/' - >>> bg.inputs.subject = '01' - >>> bg.inputs.output_query['dwi'] = dict(modality='dwi') - >>> results = bg.run() - >>> basename(results.outputs.dwi[0]) # doctest: +ALLOW_UNICODE - 'sub-01_dwi.nii.gz' - - """ - input_spec = BIDSDataGrabberInputSpec - output_spec = DynamicTraitedSpec - _always_run = True - - def __init__(self, infields=None, outfields=None, **kwargs): - """ - Parameters - ---------- - infields : list of str - Indicates the input fields to be dynamically created - - outfields: list of str - Indicates output fields to be dynamically created. - If no matching items, returns Undefined. - """ - super(BIDSDataGrabber, self).__init__(**kwargs) - if not have_pybids: - raise ImportError("The BIDSEventsGrabber interface requires pybids." - " Please make sure it is installed.") - - # If outfields is None use anat and func as default - if outfields is None: - outfields = ['func', 'anat'] - self.inputs.output_query = { - "func": {"modality": "func"}, - "anat": {"modality": "anat"}} - else: - self.inputs.output_query = {} - - # If infields is None, use all BIDS entities - if infields is None: - bids_config = join(dirname(gb.__file__), 'config', 'bids.json') - bids_config = json.load(open(bids_config, 'r')) - infields = [i['name'] for i in bids_config['entities']] - - self._infields = infields - self._outfields = outfields - - # used for mandatory inputs check - undefined_traits = {} - for key in infields: - self.inputs.add_trait(key, traits.Any) - undefined_traits[key] = Undefined - - self.inputs.trait_set(trait_change_notify=False, **undefined_traits) - - def _run_interface(self, runtime): - return runtime - - def _list_outputs(self): - layout = gb.BIDSLayout(self.inputs.base_dir) - - for key in self._outfields: - if key not in self.inputs.output_query: - raise ValueError("Define query for all outputs") - - # If infield is not given nm input value, silently ignore - filters = {} - for key in self._infields: - value = getattr(self.inputs, key) - if isdefined(value): - filters[key] = value - - outputs = {} - for key, query in self.inputs.output_query.items(): - args = query.copy() - args.update(filters) - filelist = layout.get(return_type='file', - **args) - if len(filelist) == 0: - msg = 'Output key: %s returned no files' % ( - key) - if self.inputs.raise_on_empty: - raise IOError(msg) - else: - LOGGER.warning(msg) - filelist = Undefined - else: - outputs[key] = filelist - return outputs From f460eaa9b42f3fdaebbadcd54672f4d7a2e16910 Mon Sep 17 00:00:00 2001 From: adelavega Date: Wed, 13 Sep 2017 21:20:30 -0500 Subject: [PATCH 12/19] Moved to bids utils --- nipype/interfaces/bids_utils.py | 169 ++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 nipype/interfaces/bids_utils.py diff --git a/nipype/interfaces/bids_utils.py b/nipype/interfaces/bids_utils.py new file mode 100644 index 0000000000..fda56f79b2 --- /dev/null +++ b/nipype/interfaces/bids_utils.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +""" Set of interfaces that allow interaction with BIDS data. Currently +available interfaces are: + +BIDSDataGrabber: Query data from BIDS dataset using pybids grabbids. + +Change directory to provide relative paths for doctests +>>> import os +>>> import bids +>>> filepath = os.path.realpath(os.path.dirname(bids.__file__)) +>>> datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/')) +>>> os.chdir(datadir) + +""" +from os.path import join, dirname +from .. import logging +from .base import (traits, + DynamicTraitedSpec, + Directory, + BaseInterface, + isdefined, + Str, + Undefined) + +try: + from bids import grabbids as gb + import json +except ImportError: + have_pybids = False +else: + have_pybids = True + +LOGGER = logging.getLogger('workflows') + +class BIDSDataGrabberInputSpec(DynamicTraitedSpec): + base_dir = Directory(exists=True, + desc='Path to BIDS Directory.', + mandatory=True) + output_query = traits.Dict(key_trait=Str, + value_trait=traits.Dict, + desc='Queries for outfield outputs') + raise_on_empty = traits.Bool(True, usedefault=True, + desc='Generate exception if list is empty ' + 'for a given field') + return_type = traits.Enum('filename', 'namedtuple', usedefault=True) + + +class BIDSDataGrabber(BaseInterface): + + """ BIDS datagrabber module that wraps around pybids to allow arbitrary + querying of BIDS datasets. + + Examples + -------- + + >>> from nipype.interfaces.bids_utils import BIDSDataGrabber + >>> from os.path import basename + + By default, the BIDSDataGrabber fetches anatomical and functional images + from a project, and makes BIDS entities (e.g. subject) available for + filtering outputs. + + >>> bg = BIDSDataGrabber() + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> results = bg.run() + >>> basename(results.outputs.anat[0]) # doctest: +ALLOW_UNICODE + 'sub-01_T1w.nii.gz' + + >>> basename(results.outputs.func[0]) # doctest: +ALLOW_UNICODE + 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' + + + Dynamically created, user-defined output fields can also be defined to + return different types of outputs from the same project. All outputs + are filtered on common entities, which can be explicitly defined as + infields. + + >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['dwi']) + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> bg.inputs.output_query['dwi'] = dict(modality='dwi') + >>> results = bg.run() + >>> basename(results.outputs.dwi[0]) # doctest: +ALLOW_UNICODE + 'sub-01_dwi.nii.gz' + + """ + input_spec = BIDSDataGrabberInputSpec + output_spec = DynamicTraitedSpec + _always_run = True + + def __init__(self, infields=None, outfields=None, **kwargs): + """ + Parameters + ---------- + infields : list of str + Indicates the input fields to be dynamically created + + outfields: list of str + Indicates output fields to be dynamically created. + If no matching items, returns Undefined. + """ + super(BIDSDataGrabber, self).__init__(**kwargs) + if not have_pybids: + raise ImportError("The BIDSEventsGrabber interface requires pybids." + " Please make sure it is installed.") + + # If outfields is None use anat and func as default + if outfields is None: + outfields = ['func', 'anat'] + self.inputs.output_query = { + "func": {"modality": "func"}, + "anat": {"modality": "anat"}} + else: + self.inputs.output_query = {} + + # If infields is None, use all BIDS entities + if infields is None: + bids_config = join(dirname(gb.__file__), 'config', 'bids.json') + bids_config = json.load(open(bids_config, 'r')) + infields = [i['name'] for i in bids_config['entities']] + + self._infields = infields + self._outfields = outfields + + # used for mandatory inputs check + undefined_traits = {} + for key in infields: + self.inputs.add_trait(key, traits.Any) + undefined_traits[key] = Undefined + + self.inputs.trait_set(trait_change_notify=False, **undefined_traits) + + def _run_interface(self, runtime): + return runtime + + def _list_outputs(self): + layout = gb.BIDSLayout(self.inputs.base_dir) + + for key in self._outfields: + if key not in self.inputs.output_query: + raise ValueError("Define query for all outputs") + + # If infield is not given nm input value, silently ignore + filters = {} + for key in self._infields: + value = getattr(self.inputs, key) + if isdefined(value): + filters[key] = value + + outputs = {} + for key, query in self.inputs.output_query.items(): + args = query.copy() + args.update(filters) + filelist = layout.get(return_type='file', + **args) + if len(filelist) == 0: + msg = 'Output key: %s returned no files' % ( + key) + if self.inputs.raise_on_empty: + raise IOError(msg) + else: + LOGGER.warning(msg) + filelist = Undefined + else: + outputs[key] = filelist + return outputs From 3cc3a236384ac235df1dbe1feea7d15f1d5decba Mon Sep 17 00:00:00 2001 From: adelavega Date: Wed, 13 Sep 2017 21:41:22 -0500 Subject: [PATCH 13/19] Try fix Circle --- circle.yml | 2 ++ docker/base.Dockerfile | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 5624dbb7f8..f414972934 100644 --- a/circle.yml +++ b/circle.yml @@ -27,6 +27,8 @@ dependencies: - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi - (cd $HOME/docker && gzip -d cache.tar.gz && docker load --input $HOME/docker/cache.tar) || true : timeout: 6000 + - git clone https://github.com/INCF/pybids.git && cd pybids && python setup.py develop + override: # Get data - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 3de42492ce..5b39951f64 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -148,8 +148,4 @@ ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \ SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \ FORCE_SPMMCR=1 -# Install pybids -RUN git clone https://github.com/INCF/pybids.git -RUN pip install -e . - WORKDIR /work From e2e78c4b9fcc4f730f40e64267b3162b7a347ad8 Mon Sep 17 00:00:00 2001 From: adelavega Date: Thu, 14 Sep 2017 00:19:15 -0500 Subject: [PATCH 14/19] Return type argument --- nipype/interfaces/bids_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/bids_utils.py b/nipype/interfaces/bids_utils.py index fda56f79b2..11d342baf6 100644 --- a/nipype/interfaces/bids_utils.py +++ b/nipype/interfaces/bids_utils.py @@ -154,7 +154,7 @@ def _list_outputs(self): for key, query in self.inputs.output_query.items(): args = query.copy() args.update(filters) - filelist = layout.get(return_type='file', + filelist = layout.get(return_type=self.inputs.return_type, **args) if len(filelist) == 0: msg = 'Output key: %s returned no files' % ( From 60dce3ff855ef8a988c9ed36fb9bb1157921e701 Mon Sep 17 00:00:00 2001 From: adelavega Date: Thu, 14 Sep 2017 00:22:32 -0500 Subject: [PATCH 15/19] Undefined output if no match --- nipype/interfaces/bids_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nipype/interfaces/bids_utils.py b/nipype/interfaces/bids_utils.py index 11d342baf6..ebbf2a1149 100644 --- a/nipype/interfaces/bids_utils.py +++ b/nipype/interfaces/bids_utils.py @@ -164,6 +164,6 @@ def _list_outputs(self): else: LOGGER.warning(msg) filelist = Undefined - else: - outputs[key] = filelist + + outputs[key] = filelist return outputs From 97d832ba6668d74c3711ac6f1b2a51bc8e1c8fd5 Mon Sep 17 00:00:00 2001 From: adelavega Date: Thu, 14 Sep 2017 01:04:50 -0500 Subject: [PATCH 16/19] Fix return type --- nipype/interfaces/bids_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nipype/interfaces/bids_utils.py b/nipype/interfaces/bids_utils.py index ebbf2a1149..e1606e7d92 100644 --- a/nipype/interfaces/bids_utils.py +++ b/nipype/interfaces/bids_utils.py @@ -44,7 +44,7 @@ class BIDSDataGrabberInputSpec(DynamicTraitedSpec): raise_on_empty = traits.Bool(True, usedefault=True, desc='Generate exception if list is empty ' 'for a given field') - return_type = traits.Enum('filename', 'namedtuple', usedefault=True) + return_type = traits.Enum('file', 'namedtuple', usedefault=True) class BIDSDataGrabber(BaseInterface): @@ -164,6 +164,6 @@ def _list_outputs(self): else: LOGGER.warning(msg) filelist = Undefined - + outputs[key] = filelist return outputs From ce148eb573c570baac04ce7e7f39795125e660e8 Mon Sep 17 00:00:00 2001 From: adelavega Date: Tue, 19 Sep 2017 17:49:13 -0500 Subject: [PATCH 17/19] Install pybids for tesing in Dockerfile --- Dockerfile | 3 +++ circle.yml | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 502216cf8d..14a6dec135 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,6 +87,9 @@ COPY requirements.txt requirements.txt RUN pip install -r requirements.txt && \ rm -rf ~/.cache/pip +RUN git clone https://github.com/INCF/pybids.git && \ + cd pybids && python setup.py develop + # Installing nipype COPY . /src/nipype RUN cd /src/nipype && \ diff --git a/circle.yml b/circle.yml index f414972934..82d67432d6 100644 --- a/circle.yml +++ b/circle.yml @@ -27,7 +27,6 @@ dependencies: - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi - (cd $HOME/docker && gzip -d cache.tar.gz && docker load --input $HOME/docker/cache.tar) || true : timeout: 6000 - - git clone https://github.com/INCF/pybids.git && cd pybids && python setup.py develop override: # Get data From fc6dd01cf8c8b88ebcfbb79cb33b552ffab451c4 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Wed, 20 Sep 2017 00:12:28 -0500 Subject: [PATCH 18/19] Update circle.yml --- circle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/circle.yml b/circle.yml index 82d67432d6..5624dbb7f8 100644 --- a/circle.yml +++ b/circle.yml @@ -27,7 +27,6 @@ dependencies: - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi - (cd $HOME/docker && gzip -d cache.tar.gz && docker load --input $HOME/docker/cache.tar) || true : timeout: 6000 - override: # Get data - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi From 03243d881e341e399ac55e9092ec1d0fa39005ef Mon Sep 17 00:00:00 2001 From: adelavega Date: Fri, 22 Sep 2017 11:11:57 -0500 Subject: [PATCH 19/19] Added reference to io --- nipype/interfaces/io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 516c92c804..f2e3fcd943 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -40,6 +40,7 @@ from .base import ( TraitedSpec, traits, Str, File, Directory, BaseInterface, InputMultiPath, isdefined, OutputMultiPath, DynamicTraitedSpec, Undefined, BaseInterfaceInputSpec) +from .bids_utils import BIDSDataGrabber try: import pyxnat