diff --git a/CHANGES b/CHANGES index 3fdacfd639..29e34a1854 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ Upcoming release (0.14.1) ========================= +* REF+FIX: Move BIDSDataGrabber to `interfaces.io` + fix correct default behavior (https://github.com/nipy/nipype/pull/2336) * ENH: Add AFNI interface for 3dConvertDset (https://github.com/nipy/nipype/pull/2337) * MAINT: Cleanup EngineBase (https://github.com/nipy/nipype/pull/2376) * FIX: Robustly handled outputs of 3dFWHMx across different versions of AFNI (https://github.com/nipy/nipype/pull/2373) diff --git a/nipype/interfaces/__init__.py b/nipype/interfaces/__init__.py index afae87d483..a19efa64e5 100644 --- a/nipype/interfaces/__init__.py +++ b/nipype/interfaces/__init__.py @@ -10,5 +10,5 @@ absolute_import) __docformat__ = 'restructuredtext' -from .io import DataGrabber, DataSink, SelectFiles +from .io import DataGrabber, DataSink, SelectFiles, BIDSDataGrabber from .utility import IdentityInterface, Rename, Function, Select, Merge diff --git a/nipype/interfaces/bids_utils.py b/nipype/interfaces/bids_utils.py deleted file mode 100644 index 0a74410936..0000000000 --- a/nipype/interfaces/bids_utils.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -""" Set of interfaces that allow interaction with BIDS data. Currently -available interfaces are: - -BIDSDataGrabber: Query data from BIDS dataset using pybids grabbids. - - - Change directory to provide relative paths for doctests - >>> import os - >>> filepath = os.path.dirname( os.path.realpath( __file__ ) ) - >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data')) - >>> os.chdir(datadir) -""" -from os.path import join, dirname -import json -from .. import logging -from .base import (traits, DynamicTraitedSpec, Directory, BaseInterface, - isdefined, Str, Undefined) - -have_pybids = True -try: - from bids import grabbids as gb -except ImportError: - have_pybids = False - -LOGGER = logging.getLogger('workflows') - - -class BIDSDataGrabberInputSpec(DynamicTraitedSpec): - base_dir = Directory( - exists=True, desc='Path to BIDS Directory.', mandatory=True) - output_query = traits.Dict( - key_trait=Str, - value_trait=traits.Dict, - desc='Queries for outfield outputs') - raise_on_empty = traits.Bool( - True, - usedefault=True, - desc='Generate exception if list is empty ' - 'for a given field') - return_type = traits.Enum('file', 'namedtuple', usedefault=True) - - -class BIDSDataGrabber(BaseInterface): - """ BIDS datagrabber module that wraps around pybids to allow arbitrary - querying of BIDS datasets. - - Examples - -------- - - By default, the BIDSDataGrabber fetches anatomical and functional images - from a project, and makes BIDS entities (e.g. subject) available for - filtering outputs. - - >>> bg = BIDSDataGrabber() - >>> bg.inputs.base_dir = 'ds005/' - >>> bg.inputs.subject = '01' - >>> results = bg.run() # doctest: +SKIP - - - Dynamically created, user-defined output fields can also be defined to - return different types of outputs from the same project. All outputs - are filtered on common entities, which can be explicitly defined as - infields. - - >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['dwi']) - >>> bg.inputs.base_dir = 'ds005/' - >>> bg.inputs.subject = '01' - >>> bg.inputs.output_query['dwi'] = dict(modality='dwi') - >>> results = bg.run() # doctest: +SKIP - - """ - input_spec = BIDSDataGrabberInputSpec - output_spec = DynamicTraitedSpec - _always_run = True - - def __init__(self, infields=None, **kwargs): - """ - Parameters - ---------- - infields : list of str - Indicates the input fields to be dynamically created - - outfields: list of str - Indicates output fields to be dynamically created. - If no matching items, returns Undefined. - """ - super(BIDSDataGrabber, self).__init__(**kwargs) - - if not isdefined(self.inputs.output_query): - self.inputs.output_query = { - "func": { - "modality": "func" - }, - "anat": { - "modality": "anat" - } - } - - # If infields is empty, use all BIDS entities - if infields is not None and have_pybids: - bids_config = join(dirname(gb.__file__), 'config', 'bids.json') - bids_config = json.load(open(bids_config, 'r')) - infields = [i['name'] for i in bids_config['entities']] - - self._infields = infields or [] - - # used for mandatory inputs check - undefined_traits = {} - for key in self._infields: - self.inputs.add_trait(key, traits.Any) - undefined_traits[key] = kwargs[key] if key in kwargs else Undefined - - self.inputs.trait_set(trait_change_notify=False, **undefined_traits) - - def _run_interface(self, runtime): - if not have_pybids: - raise ImportError( - "The BIDSEventsGrabber interface requires pybids." - " Please make sure it is installed.") - return runtime - - def _list_outputs(self): - layout = gb.BIDSLayout(self.inputs.base_dir) - - # If infield is not given nm input value, silently ignore - filters = {} - for key in self._infields: - value = getattr(self.inputs, key) - if isdefined(value): - filters[key] = value - - outputs = {} - for key, query in self.inputs.output_query.items(): - args = query.copy() - args.update(filters) - filelist = layout.get(return_type=self.inputs.return_type, **args) - if len(filelist) == 0: - msg = 'Output key: %s returned no files' % key - if self.inputs.raise_on_empty: - raise IOError(msg) - else: - LOGGER.warning(msg) - filelist = Undefined - - outputs[key] = filelist - return outputs diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 6bb9a943f0..5687b3f77e 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -25,12 +25,14 @@ import glob import fnmatch import string +import json import os import os.path as op import shutil import subprocess import re import tempfile +from os.path import join, dirname from warnings import warn import sqlite3 @@ -38,10 +40,15 @@ from .. import config, logging from ..utils.filemanip import copyfile, list_to_filename, filename_to_list from ..utils.misc import human_order_sorted, str2bool -from .base import (TraitedSpec, traits, Str, File, Directory, BaseInterface, - InputMultiPath, isdefined, OutputMultiPath, - DynamicTraitedSpec, Undefined, BaseInterfaceInputSpec) -from .bids_utils import BIDSDataGrabber +from .base import ( + TraitedSpec, traits, Str, File, Directory, BaseInterface, InputMultiPath, + isdefined, OutputMultiPath, DynamicTraitedSpec, Undefined, BaseInterfaceInputSpec) + +have_pybids = True +try: + from bids import grabbids as gb +except ImportError: + have_pybids = False try: import pyxnat @@ -2717,3 +2724,116 @@ def _list_outputs(self): outputs = self.output_spec().get() outputs['out_file'] = out_file return outputs + + +class BIDSDataGrabberInputSpec(DynamicTraitedSpec): + base_dir = Directory(exists=True, + desc='Path to BIDS Directory.', + mandatory=True) + output_query = traits.Dict(key_trait=Str, + value_trait=traits.Dict, + desc='Queries for outfield outputs') + raise_on_empty = traits.Bool(True, usedefault=True, + desc='Generate exception if list is empty ' + 'for a given field') + return_type = traits.Enum('file', 'namedtuple', usedefault=True) + + +class BIDSDataGrabber(IOBase): + + """ BIDS datagrabber module that wraps around pybids to allow arbitrary + querying of BIDS datasets. + + Examples + -------- + + By default, the BIDSDataGrabber fetches anatomical and functional images + from a project, and makes BIDS entities (e.g. subject) available for + filtering outputs. + + >>> bg = BIDSDataGrabber() + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> results = bg.run() # doctest: +SKIP + + + Dynamically created, user-defined output fields can also be defined to + return different types of outputs from the same project. All outputs + are filtered on common entities, which can be explicitly defined as + infields. + + >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['dwi']) + >>> bg.inputs.base_dir = 'ds005/' + >>> bg.inputs.subject = '01' + >>> bg.inputs.output_query['dwi'] = dict(modality='dwi') + >>> results = bg.run() # doctest: +SKIP + + """ + input_spec = BIDSDataGrabberInputSpec + output_spec = DynamicTraitedSpec + _always_run = True + + def __init__(self, infields=None, **kwargs): + """ + Parameters + ---------- + infields : list of str + Indicates the input fields to be dynamically created + """ + super(BIDSDataGrabber, self).__init__(**kwargs) + + if not isdefined(self.inputs.output_query): + self.inputs.output_query = {"func": {"modality": "func"}, + "anat": {"modality": "anat"}} + + # If infields is empty, use all BIDS entities + if infields is None and have_pybids: + bids_config = join(dirname(gb.__file__), 'config', 'bids.json') + bids_config = json.load(open(bids_config, 'r')) + infields = [i['name'] for i in bids_config['entities']] + + self._infields = infields or [] + + # used for mandatory inputs check + undefined_traits = {} + for key in self._infields: + self.inputs.add_trait(key, traits.Any) + undefined_traits[key] = kwargs[key] if key in kwargs else Undefined + + self.inputs.trait_set(trait_change_notify=False, **undefined_traits) + + def _run_interface(self, runtime): + if not have_pybids: + raise ImportError( + "The BIDSEventsGrabber interface requires pybids." + " Please make sure it is installed.") + return runtime + + def _list_outputs(self): + layout = gb.BIDSLayout(self.inputs.base_dir) + + # If infield is not given nm input value, silently ignore + filters = {} + for key in self._infields: + value = getattr(self.inputs, key) + if isdefined(value): + filters[key] = value + + outputs = {} + for key, query in self.inputs.output_query.items(): + args = query.copy() + args.update(filters) + filelist = layout.get(return_type=self.inputs.return_type, **args) + if len(filelist) == 0: + msg = 'Output key: %s returned no files' % key + if self.inputs.raise_on_empty: + raise IOError(msg) + else: + iflogger.warning(msg) + filelist = Undefined + + outputs[key] = filelist + return outputs + + def _add_output_traits(self, base): + return add_traits(base, list(self.inputs.output_query.keys())) diff --git a/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py b/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py index da594a7365..d77e9d7509 100644 --- a/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py +++ b/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py @@ -1,6 +1,6 @@ # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT from __future__ import unicode_literals -from ..bids_utils import BIDSDataGrabber +from ..io import BIDSDataGrabber def test_BIDSDataGrabber_inputs(): diff --git a/nipype/interfaces/tests/test_bids.py b/nipype/interfaces/tests/test_bids.py deleted file mode 100644 index 33ef861e11..0000000000 --- a/nipype/interfaces/tests/test_bids.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import json -import sys - -import pytest -from nipype.interfaces.bids_utils import BIDSDataGrabber -from nipype.utils.filemanip import dist_is_editable - -have_pybids = True -try: - import bids - from bids import grabbids as gb - filepath = os.path.realpath(os.path.dirname(bids.__file__)) - datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/')) -except ImportError: - have_pybids = False - - -# There are three reasons these tests will be skipped: -@pytest.mark.skipif(not have_pybids, reason="Pybids is not installed") -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Pybids no longer supports Python 2") -@pytest.mark.skipif( - not dist_is_editable('pybids'), - reason="Pybids is not installed in editable mode") -def test_bids_grabber(tmpdir): - tmpdir.chdir() - bg = BIDSDataGrabber() - bg.inputs.base_dir = os.path.join(datadir, 'ds005') - bg.inputs.subject = '01' - results = bg.run() - assert os.path.basename(results.outputs.anat[0]) == 'sub-01_T1w.nii.gz' - assert os.path.basename(results.outputs.func[0]) == ( - 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz') - - -@pytest.mark.skipif(not have_pybids, reason="Pybids is not installed") -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Pybids no longer supports Python 2") -@pytest.mark.skipif( - not dist_is_editable('pybids'), - reason="Pybids is not installed in editable mode") -def test_bids_fields(tmpdir): - tmpdir.chdir() - bg = BIDSDataGrabber(infields=['subject'], outfields=['dwi']) - bg.inputs.base_dir = os.path.join(datadir, 'ds005') - bg.inputs.subject = '01' - bg.inputs.output_query['dwi'] = dict(modality='dwi') - results = bg.run() - assert os.path.basename(results.outputs.dwi[0]) == 'sub-01_dwi.nii.gz' diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 4e28f2ee4d..eeae322050 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -9,6 +9,7 @@ import glob import shutil import os.path as op +import sys from subprocess import Popen import hashlib from collections import namedtuple @@ -16,7 +17,9 @@ import pytest import nipype import nipype.interfaces.io as nio +from nipype.interfaces.base.traits_extension import isdefined from nipype.interfaces.base import Undefined, TraitError +from nipype.utils.filemanip import dist_is_editable # Check for boto noboto = False @@ -43,6 +46,16 @@ except CalledProcessError: fakes3 = False +# check for bids +have_pybids = True +try: + import bids + from bids import grabbids as gb + filepath = os.path.realpath(os.path.dirname(bids.__file__)) + datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/')) +except ImportError: + have_pybids = False + def test_datagrabber(): dg = nio.DataGrabber() @@ -537,3 +550,64 @@ def test_jsonsink(tmpdir, inputs_attributes): data = simplejson.load(f) assert data == expected_data + + +# There are three reasons these tests will be skipped: +@pytest.mark.skipif(not have_pybids, + reason="Pybids is not installed") +@pytest.mark.skipif(sys.version_info < (3, 0), + reason="Pybids no longer supports Python 2") +@pytest.mark.skipif(not dist_is_editable('pybids'), + reason="Pybids is not installed in editable mode") +def test_bids_grabber(tmpdir): + tmpdir.chdir() + bg = nio.BIDSDataGrabber() + bg.inputs.base_dir = os.path.join(datadir, 'ds005') + bg.inputs.subject = '01' + results = bg.run() + assert os.path.basename(results.outputs.anat[0]) == 'sub-01_T1w.nii.gz' + assert os.path.basename(results.outputs.func[0]) == ( + 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz') + + +@pytest.mark.skipif(not have_pybids, + reason="Pybids is not installed") +@pytest.mark.skipif(sys.version_info < (3, 0), + reason="Pybids no longer supports Python 2") +@pytest.mark.skipif(not dist_is_editable('pybids'), + reason="Pybids is not installed in editable mode") +def test_bids_fields(tmpdir): + tmpdir.chdir() + bg = nio.BIDSDataGrabber(infields = ['subject'], outfields = ['dwi']) + bg.inputs.base_dir = os.path.join(datadir, 'ds005') + bg.inputs.subject = '01' + bg.inputs.output_query['dwi'] = dict(modality='dwi') + results = bg.run() + assert os.path.basename(results.outputs.dwi[0]) == 'sub-01_dwi.nii.gz' + + +@pytest.mark.skipif(not have_pybids, + reason="Pybids is not installed") +@pytest.mark.skipif(sys.version_info < (3, 0), + reason="Pybids no longer supports Python 2") +@pytest.mark.skipif(not dist_is_editable('pybids'), + reason="Pybids is not installed in editable mode") +def test_bids_infields_outfields(tmpdir): + tmpdir.chdir() + infields = ['infield1', 'infield2'] + outfields = ['outfield1', 'outfield2'] + bg = nio.BIDSDataGrabber(infields=infields) + for outfield in outfields: + bg.inputs.output_query[outfield] = {'key': 'value'} + + for infield in infields: + assert(infield in bg.inputs.traits()) + assert(not(isdefined(bg.inputs.get()[infield]))) + + for outfield in outfields: + assert(outfield in bg._outputs().traits()) + + # now try without defining outfields, we should get anat and func for free + bg = nio.BIDSDataGrabber() + for outfield in ['anat', 'func']: + assert outfield in bg._outputs().traits()