diff --git a/CHANGES b/CHANGES index fc37efb061..8720078b98 100644 --- a/CHANGES +++ b/CHANGES @@ -29,6 +29,7 @@ Upcoming release 0.13 * ENH: Added support for custom job submission check in SLURM (https://github.com/nipy/nipype/pull/1582) * ENH: Added ANTs interface CreateJacobianDeterminantImage; replaces deprecated JacobianDeterminant (https://github.com/nipy/nipype/pull/1654) +* ENH: Update ReconAll interface for FreeSurfer v6.0.0 (https://github.com/nipy/nipype/pull/1790) Release 0.12.1 (August 3, 2016) =============================== diff --git a/nipype/interfaces/freesurfer/preprocess.py b/nipype/interfaces/freesurfer/preprocess.py index ecced01676..2953963149 100644 --- a/nipype/interfaces/freesurfer/preprocess.py +++ b/nipype/interfaces/freesurfer/preprocess.py @@ -21,8 +21,8 @@ import numpy as np from nibabel import load -from ... import logging -from ...utils.filemanip import fname_presuffix +from ... import logging, LooseVersion +from ...utils.filemanip import fname_presuffix, check_depends from ..io import FreeSurferSource from ..base import (TraitedSpec, File, traits, Directory, InputMultiPath, @@ -30,12 +30,20 @@ CommandLineInputSpec, isdefined) from .base import (FSCommand, FSTraitedSpec, FSTraitedSpecOpenMP, - FSCommandOpenMP) + FSCommandOpenMP, Info) from .utils import copy2subjdir __docformat__ = 'restructuredtext' iflogger = logging.getLogger('interface') +FSVersion = "0" +_ver = Info.version() +if _ver: + if 'dev' in _ver: + FSVersion = _ver.rstrip().split('-')[-1] + '.dev' + else: + FSVersion = _ver.rstrip().split('-v')[-1] + class ParseDICOMDirInputSpec(FSTraitedSpec): dicom_dir = Directory(exists=True, argstr='--d %s', mandatory=True, @@ -622,6 +630,8 @@ class ReconAllInputSpec(CommandLineInputSpec): desc='Use converted T2 to refine the cortical surface') openmp = traits.Int(argstr="-openmp %d", desc="Number of processors to use in parallel") + parallel = traits.Bool(argstr="-parallel", + desc="Enable parallel execution") subjects_dir = Directory(exists=True, argstr='-sd %s', hash_files=False, desc='path to subjects directory', genfile=True) flags = traits.Str(argstr='%s', desc='additional parameters') @@ -656,85 +666,173 @@ class ReconAll(CommandLine): output_spec = ReconAllOutputSpec _can_resume = True - _steps = [ - # autorecon1 - ('motioncor', ['mri/rawavg.mgz', 'mri/orig.mgz']), - ('talairach', ['mri/transforms/talairach.auto.xfm', - 'mri/transforms/talairach.xfm']), - ('nuintensitycor', ['mri/nu.mgz']), - ('normalization', ['mri/T1.mgz']), - ('skullstrip', - ['mri/brainmask.auto.mgz', - 'mri/brainmask.mgz']), - # autorecon2 - ('gcareg', ['mri/transforms/talairach.lta']), - ('canorm', ['mri/norm.mgz']), - ('careg', ['mri/transforms/talairach.m3z']), - ('careginv', ['mri/transforms/talairach.m3z.inv.x.mgz', - 'mri/transforms/talairach.m3z.inv.y.mgz', - 'mri/transforms/talairach.m3z.inv.z.mgz']), - ('rmneck', ['mri/nu_noneck.mgz']), - ('skull-lta', ['mri/transforms/talairach_with_skull_2.lta']), - ('calabel', - ['mri/aseg.auto_noCCseg.mgz', 'mri/aseg.auto.mgz', 'mri/aseg.mgz']), - ('normalization2', ['mri/brain.mgz']), - ('maskbfs', ['mri/brain.finalsurfs.mgz']), - ('segmentation', ['mri/wm.asegedit.mgz', 'mri/wm.mgz']), - ('fill', ['mri/filled.mgz']), - ('tessellate', ['surf/lh.orig.nofix', 'surf/rh.orig.nofix']), - ('smooth1', ['surf/lh.smoothwm.nofix', 'surf/rh.smoothwm.nofix']), - ('inflate1', ['surf/lh.inflated.nofix', 'surf/rh.inflated.nofix']), - ('qsphere', ['surf/lh.qsphere.nofix', 'surf/rh.qsphere.nofix']), - ('fix', ['surf/lh.orig', 'surf/rh.orig']), - ('white', - ['surf/lh.white', - 'surf/rh.white', - 'surf/lh.curv', - 'surf/rh.curv', - 'surf/lh.area', - 'surf/rh.area', - 'label/lh.cortex.label', - 'label/rh.cortex.label']), - ('smooth2', ['surf/lh.smoothwm', 'surf/rh.smoothwm']), - ('inflate2', - ['surf/lh.inflated', - 'surf/rh.inflated', - 'surf/lh.sulc', - 'surf/rh.sulc', - 'surf/lh.inflated.H', - 'surf/rh.inflated.H', - 'surf/lh.inflated.K', - 'surf/rh.inflated.K']), - # autorecon3 - ('sphere', ['surf/lh.sphere', 'surf/rh.sphere']), - ('surfreg', ['surf/lh.sphere.reg', 'surf/rh.sphere.reg']), - ('jacobian_white', ['surf/lh.jacobian_white', - 'surf/rh.jacobian_white']), - ('avgcurv', ['surf/lh.avg_curv', 'surf/rh.avg_curv']), - ('cortparc', ['label/lh.aparc.annot', 'label/rh.aparc.annot']), - ('pial', - ['surf/lh.pial', - 'surf/rh.pial', - 'surf/lh.curv.pial', - 'surf/rh.curv.pial', - 'surf/lh.area.pial', - 'surf/rh.area.pial', - 'surf/lh.thickness', - 'surf/rh.thickness']), - ('cortparc2', ['label/lh.aparc.a2009s.annot', - 'label/rh.aparc.a2009s.annot']), - ('parcstats2', - ['stats/lh.aparc.a2009s.stats', - 'stats/rh.aparc.a2009s.stats', - 'stats/aparc.annot.a2009s.ctab']), - ('cortribbon', ['mri/lh.ribbon.mgz', 'mri/rh.ribbon.mgz', - 'mri/ribbon.mgz']), - ('segstats', ['stats/aseg.stats']), - ('aparc2aseg', ['mri/aparc+aseg.mgz', 'mri/aparc.a2009s+aseg.mgz']), - ('wmparc', ['mri/wmparc.mgz', 'stats/wmparc.stats']), - ('balabels', ['BA.ctab', 'BA.thresh.ctab']), - ('label-exvivo-ec', ['label/lh.entorhinal_exvivo.label', - 'label/rh.entorhinal_exvivo.label'])] + # Steps are based off of the recon-all tables [0,1] describing, inputs, + # commands, and outputs of each step of the recon-all process, + # controlled by flags. + # + # Each step is a 3-tuple containing (flag, [outputs], [inputs]) + # A step is considered complete if all of its outputs exist and are newer + # than the inputs. An empty input list indicates input mtimes will not + # be checked. This may need updating, if users are working with manually + # edited files. + # + # [0] https://surfer.nmr.mgh.harvard.edu/fswiki/ReconAllTableStableV5.3 + # [1] https://surfer.nmr.mgh.harvard.edu/fswiki/ReconAllTableStableV6.0 + _autorecon1_steps = [ + ('motioncor', ['mri/rawavg.mgz', 'mri/orig.mgz'], []), + ('talairach', ['mri/orig_nu.mgz', + 'mri/transforms/talairach.auto.xfm', + 'mri/transforms/talairach.xfm', + # 'mri/transforms/talairach_avi.log', + ], []), + ('nuintensitycor', ['mri/nu.mgz'], []), + ('normalization', ['mri/T1.mgz'], []), + ('skullstrip', ['mri/talairach_with_skull.lta', + 'mri/brainmask.auto.mgz', + 'mri/brainmask.mgz'], []), + ] + if LooseVersion(FSVersion) < LooseVersion("6.0.0"): + _autorecon2_steps = [ + ('gcareg', ['mri/transforms/talairach.lta'], []), + ('canorm', ['mri/norm.mgz'], []), + ('careg', ['mri/transforms/talairach.m3z'], []), + ('careginv', ['mri/transforms/talairach.m3z.inv.x.mgz', + 'mri/transforms/talairach.m3z.inv.y.mgz', + 'mri/transforms/talairach.m3z.inv.z.mgz', + ], []), + ('rmneck', ['mri/nu_noneck.mgz'], []), + ('skull-lta', ['mri/transforms/talairach_with_skull_2.lta'], []), + ('calabel', ['mri/aseg.auto_noCCseg.mgz', + 'mri/aseg.auto.mgz', + 'mri/aseg.mgz'], []), + ('normalization2', ['mri/brain.mgz'], []), + ('maskbfs', ['mri/brain.finalsurfs.mgz'], []), + ('segmentation', ['mri/wm.seg.mgz', + 'mri/wm.asegedit.mgz', + 'mri/wm.mgz'], []), + ('fill', ['mri/filled.mgz', + # 'scripts/ponscc.cut.log', + ], []), + ('tessellate', ['surf/lh.orig.nofix', 'surf/rh.orig.nofix'], []), + ('smooth1', ['surf/lh.smoothwm.nofix', 'surf/rh.smoothwm.nofix'], + []), + ('inflate1', ['surf/lh.inflated.nofix', 'surf/rh.inflated.nofix'], + []), + ('qsphere', ['surf/lh.qsphere.nofix', 'surf/rh.qsphere.nofix'], + []), + ('fix', ['surf/lh.orig', 'surf/rh.orig'], []), + ('white', ['surf/lh.white', 'surf/rh.white', + 'surf/lh.curv', 'surf/rh.curv', + 'surf/lh.area', 'surf/rh.area', + 'label/lh.cortex.label', 'label/rh.cortex.label'], []), + ('smooth2', ['surf/lh.smoothwm', 'surf/rh.smoothwm'], []), + ('inflate2', ['surf/lh.inflated', 'surf/rh.inflated', + 'surf/lh.sulc', 'surf/rh.sulc', + 'surf/lh.inflated.H', 'surf/rh.inflated.H', + 'surf/lh.inflated.K', 'surf/rh.inflated.K'], []), + ] + _autorecon3_steps = [ + ('sphere', ['surf/lh.sphere', 'surf/rh.sphere'], []), + ('surfreg', ['surf/lh.sphere.reg', 'surf/rh.sphere.reg'], []), + ('jacobian_white', ['surf/lh.jacobian_white', + 'surf/rh.jacobian_white'], []), + ('avgcurv', ['surf/lh.avg_curv', 'surf/rh.avg_curv'], []), + ('cortparc', ['label/lh.aparc.annot', 'label/rh.aparc.annot'], []), + ('pial', ['surf/lh.pial', 'surf/rh.pial', + 'surf/lh.curv.pial', 'surf/rh.curv.pial', + 'surf/lh.area.pial', 'surf/rh.area.pial', + 'surf/lh.thickness', 'surf/rh.thickness'], []), + ('cortparc2', ['label/lh.aparc.a2009s.annot', + 'label/rh.aparc.a2009s.annot'], []), + ('parcstats2', ['stats/lh.aparc.a2009s.stats', + 'stats/rh.aparc.a2009s.stats', + 'stats/aparc.annot.a2009s.ctab'], []), + ('cortribbon', ['mri/lh.ribbon.mgz', 'mri/rh.ribbon.mgz', + 'mri/ribbon.mgz'], []), + ('segstats', ['stats/aseg.stats'], []), + ('aparc2aseg', ['mri/aparc+aseg.mgz', + 'mri/aparc.a2009s+aseg.mgz'], []), + ('wmparc', ['mri/wmparc.mgz', 'stats/wmparc.stats'], []), + ('balabels', ['BA.ctab', 'BA.thresh.ctab'], []), + ('label-exvivo-ec', ['label/lh.entorhinal_exvivo.label', + 'label/rh.entorhinal_exvivo.label'], []), + ] + else: + _autorecon2_steps = [ + ('gcareg', ['mri/transforms/talairach.lta'], []), + ('canorm', ['mri/norm.mgz'], []), + ('careg', ['mri/transforms/talairach.m3z'], []), + ('calabel', ['mri/aseg.auto_noCCseg.mgz', + 'mri/aseg.auto.mgz', + 'mri/aseg.mgz'], []), + ('normalization2', ['mri/brain.mgz'], []), + ('maskbfs', ['mri/brain.finalsurfs.mgz'], []), + ('segmentation', ['mri/wm.seg.mgz', + 'mri/wm.asegedit.mgz', + 'mri/wm.mgz'], []), + ('fill', ['mri/filled.mgz', + # 'scripts/ponscc.cut.log', + ], []), + ('tessellate', ['surf/lh.orig.nofix', 'surf/rh.orig.nofix'], []), + ('smooth1', ['surf/lh.smoothwm.nofix', 'surf/rh.smoothwm.nofix'], + []), + ('inflate1', ['surf/lh.inflated.nofix', 'surf/rh.inflated.nofix'], + []), + ('qsphere', ['surf/lh.qsphere.nofix', 'surf/rh.qsphere.nofix'], + []), + ('fix', ['surf/lh.orig', 'surf/rh.orig'], []), + ('white', ['surf/lh.white.preaparc', 'surf/rh.white.preaparc', + 'surf/lh.curv', 'surf/rh.curv', + 'surf/lh.area', 'surf/rh.area', + 'label/lh.cortex.label', 'label/rh.cortex.label'], []), + ('smooth2', ['surf/lh.smoothwm', 'surf/rh.smoothwm'], []), + ('inflate2', ['surf/lh.inflated', 'surf/rh.inflated', + 'surf/lh.sulc', 'surf/rh.sulc'], []), + ('curvHK', ['surf/lh.white.H', 'surf/rh.white.H', + 'surf/lh.white.K', 'surf/rh.white.K', + 'surf/lh.inflated.H', 'surf/rh.inflated.H', + 'surf/lh.inflated.K', 'surf/rh.inflated.K'], []), + ('curvstats', ['stats/lh.curv.stats', 'stats/rh.curv.stats'], []), + ] + _autorecon3_steps = [ + ('sphere', ['surf/lh.sphere', 'surf/rh.sphere'], []), + ('surfreg', ['surf/lh.sphere.reg', 'surf/rh.sphere.reg'], []), + ('jacobian_white', ['surf/lh.jacobian_white', + 'surf/rh.jacobian_white'], []), + ('avgcurv', ['surf/lh.avg_curv', 'surf/rh.avg_curv'], []), + ('cortparc', ['label/lh.aparc.annot', 'label/rh.aparc.annot'], []), + ('pial', ['surf/lh.pial', 'surf/rh.pial', + 'surf/lh.curv.pial', 'surf/rh.curv.pial', + 'surf/lh.area.pial', 'surf/rh.area.pial', + 'surf/lh.thickness', 'surf/rh.thickness'], []), + ('cortribbon', ['mri/lh.ribbon.mgz', 'mri/rh.ribbon.mgz', + 'mri/ribbon.mgz'], []), + ('parcstats', ['stats/lh.aparc.astats', 'stats/rh.aparc.stats', + 'stats/aparc.annot.ctab'], []), + ('cortparc2', ['label/lh.aparc.a2009s.annot', + 'label/rh.aparc.a2009s.annot'], []), + ('parcstats2', ['stats/lh.aparc.a2009s.stats', + 'stats/rh.aparc.a2009s.stats', + 'stats/aparc.annot.a2009s.ctab'], []), + ('cortparc3', ['label/lh.aparc.DKTatlas.annot', + 'label/rh.aparc.DKTatlas.annot'], []), + ('parcstats3', ['stats/lh.aparc.DKTatlas.stats', + 'stats/rh.aparc.DKTatlas.stats', + 'stats/aparc.annot.DKTatlas.ctab'], []), + ('pctsurfcon', ['surf/lh.w-g.pct.mgh', 'surf/rh.w-g.pct.mgh'], []), + ('hyporelabel', ['mri/aseg.presurf.hypos.mgz'], []), + ('aparc2aseg', ['mri/aparc+aseg.mgz', + 'mri/aparc.a2009s+aseg.mgz', + 'mri/aparc.DKTatlas+aseg.mgz'], []), + ('apas2aseg', ['mri/aseg.mgz'], ['mri/aparc+aseg.mgz']), + ('segstats', ['stats/aseg.stats'], []), + ('wmparc', ['mri/wmparc.mgz', 'stats/wmparc.stats'], []), + ('balabels', ['BA.ctab', 'BA.thresh.ctab', + 'label/lh.entorhinal_exvivo.label', + 'label/rh.entorhinal_exvivo.label'], []), + ] + + _steps = _autorecon1_steps + _autorecon2_steps + _autorecon3_steps def _gen_subjects_dir(self): return os.getcwd() @@ -790,24 +888,21 @@ def cmdline(self): subjects_dir = self.inputs.subjects_dir if not isdefined(subjects_dir): subjects_dir = self._gen_subjects_dir() - # cmd = cmd.replace(' -all ', ' -make all ') - iflogger.info('Overriding recon-all directive') + flags = [] - directive = 'all' for idx, step in enumerate(self._steps): - step, outfiles = step - if all([os.path.exists(os.path.join(subjects_dir, - self.inputs.subject_id, f)) for - f in outfiles]): - flags.append('-no%s' % step) - if idx > 4: - directive = 'autorecon2' - elif idx > 23: - directive = 'autorecon3' - else: - flags.append('-%s' % step) - cmd = cmd.replace(' -%s ' % self.inputs.directive, ' -%s ' % directive) + step, outfiles, infiles = step + flag = '-{}'.format(step) + noflag = '-no{}'.format(step) + if flag in cmd or noflag in cmd: + continue + + subj_dir = os.path.join(subjects_dir, self.inputs.subject_id) + if check_depends([os.path.join(subj_dir, f) for f in outfiles], + [os.path.join(subj_dir, f) for f in infiles]): + flags.append(noflag) cmd += ' ' + ' '.join(flags) + iflogger.info('resume recon-all : %s' % cmd) return cmd diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 3f7f7462f9..ca5cd66f07 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -447,6 +447,19 @@ def list_to_filename(filelist): else: return filelist[0] + +def check_depends(targets, dependencies): + """Return true if all targets exist and are newer than all dependencies. + + An OSError will be raised if there are missing dependencies. + """ + tgts = filename_to_list(targets) + deps = filename_to_list(dependencies) + return all(map(os.path.exists, tgts)) and \ + min(map(os.path.getmtime, tgts)) > \ + max(list(map(os.path.getmtime, deps)) + [0]) + + def save_json(filename, data): """Save data to a json file diff --git a/nipype/utils/tests/test_filemanip.py b/nipype/utils/tests/test_filemanip.py index f0dee52870..644a055f4b 100644 --- a/nipype/utils/tests/test_filemanip.py +++ b/nipype/utils/tests/test_filemanip.py @@ -5,7 +5,9 @@ from builtins import open import os -from tempfile import mkstemp +import time +from tempfile import mkstemp, mkdtemp +import shutil import warnings import pytest @@ -15,6 +17,7 @@ hash_rename, check_forhash, copyfile, copyfiles, filename_to_list, list_to_filename, + check_depends, split_filename, get_related_files) import numpy as np @@ -271,6 +274,41 @@ def test_list_to_filename(list, expected): assert x == expected +def test_check_depends(): + def touch(fname): + with open(fname, 'a'): + os.utime(fname, None) + + tmpdir = mkdtemp() + + dependencies = [os.path.join(tmpdir, str(i)) for i in range(3)] + targets = [os.path.join(tmpdir, str(i)) for i in range(3, 6)] + + # Targets newer than dependencies + for dep in dependencies: + touch(dep) + time.sleep(1) + for tgt in targets: + touch(tgt) + assert check_depends(targets, dependencies) + + # Targets older than newest dependency + time.sleep(1) + touch(dependencies[0]) + assert not check_depends(targets, dependencies) + + # Missing dependency + os.unlink(dependencies[0]) + try: + check_depends(targets, dependencies) + except OSError as e: + pass + else: + assert False, "Should raise OSError on missing dependency" + + shutil.rmtree(tmpdir) + + def test_json(): # Simple roundtrip test of json files, just a sanity check. adict = dict(a='one', c='three', b='two')