Skip to content

Commit a746633

Browse files
authored
Merge pull request #1599 from shoshber/compcor1594
ENH CompCor
2 parents aae371d + bb45785 commit a746633

File tree

10 files changed

+569
-108
lines changed

10 files changed

+569
-108
lines changed

examples/README

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
A dataset for use with these scripts can be downloaded from the nipype
22
website. At the time of writing, it's at:
33

4-
http://nipy.org/nipype/users/pipeline_tutorial.html
4+
http://nipype.readthedocs.io/en/0.12.0/users/pipeline_tutorial.html

examples/rsfmri_vol_surface_preprocessing_nipy.py

Lines changed: 8 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@
5252
from nipype.interfaces.base import CommandLine
5353
CommandLine.set_default_terminal_output('allatonce')
5454

55+
# https://github.com/moloney/dcmstack
5556
from dcmstack.extract import default_extractor
57+
# pip install pydicom
5658
from dicom import read_file
5759

5860
from nipype.interfaces import (fsl, Function, ants, freesurfer, nipy)
@@ -64,6 +66,7 @@
6466

6567
from nipype.algorithms.rapidart import ArtifactDetect
6668
from nipype.algorithms.misc import TSNR
69+
from nipype.algorithms.compcor import ACompCor
6770
from nipype.interfaces.utility import Rename, Merge, IdentityInterface
6871
from nipype.utils.filemanip import filename_to_list
6972
from nipype.interfaces.io import DataSink, FreeSurferSource
@@ -228,51 +231,6 @@ def build_filter1(motion_params, comp_norm, outliers, detrend_poly=None):
228231
out_files.append(filename)
229232
return out_files
230233

231-
232-
def extract_noise_components(realigned_file, mask_file, num_components=5,
233-
extra_regressors=None):
234-
"""Derive components most reflective of physiological noise
235-
236-
Parameters
237-
----------
238-
realigned_file: a 4D Nifti file containing realigned volumes
239-
mask_file: a 3D Nifti file containing white matter + ventricular masks
240-
num_components: number of components to use for noise decomposition
241-
extra_regressors: additional regressors to add
242-
243-
Returns
244-
-------
245-
components_file: a text file containing the noise components
246-
"""
247-
imgseries = nb.load(realigned_file)
248-
components = None
249-
for filename in filename_to_list(mask_file):
250-
mask = nb.load(filename).get_data()
251-
if len(np.nonzero(mask > 0)[0]) == 0:
252-
continue
253-
voxel_timecourses = imgseries.get_data()[mask > 0]
254-
voxel_timecourses[np.isnan(np.sum(voxel_timecourses, axis=1)), :] = 0
255-
# remove mean and normalize by variance
256-
# voxel_timecourses.shape == [nvoxels, time]
257-
X = voxel_timecourses.T
258-
stdX = np.std(X, axis=0)
259-
stdX[stdX == 0] = 1.
260-
stdX[np.isnan(stdX)] = 1.
261-
stdX[np.isinf(stdX)] = 1.
262-
X = (X - np.mean(X, axis=0)) / stdX
263-
u, _, _ = sp.linalg.svd(X, full_matrices=False)
264-
if components is None:
265-
components = u[:, :num_components]
266-
else:
267-
components = np.hstack((components, u[:, :num_components]))
268-
if extra_regressors:
269-
regressors = np.genfromtxt(extra_regressors)
270-
components = np.hstack((components, regressors))
271-
components_file = os.path.join(os.getcwd(), 'noise_components.txt')
272-
np.savetxt(components_file, components, fmt=b"%.10f")
273-
return components_file
274-
275-
276234
def rename(in_files, suffix=None):
277235
from nipype.utils.filemanip import (filename_to_list, split_filename,
278236
list_to_filename)
@@ -592,7 +550,7 @@ def create_workflow(files,
592550
realign.inputs.slice_info = 2
593551
realign.plugin_args = {'sbatch_args': '-c%d' % 4}
594552

595-
# Comute TSNR on realigned data regressing polynomials upto order 2
553+
# Compute TSNR on realigned data regressing polynomials up to order 2
596554
tsnr = MapNode(TSNR(regress_poly=2), iterfield=['in_file'], name='tsnr')
597555
wf.connect(realign, "out_file", tsnr, "in_file")
598556

@@ -694,14 +652,10 @@ def merge_files(in1, in2):
694652
filter1, 'out_res_name')
695653
wf.connect(createfilter1, 'out_files', filter1, 'design')
696654

697-
createfilter2 = MapNode(Function(input_names=['realigned_file', 'mask_file',
698-
'num_components',
699-
'extra_regressors'],
700-
output_names=['out_files'],
701-
function=extract_noise_components,
702-
imports=imports),
655+
createfilter2 = MapNode(ACompCor(),
703656
iterfield=['realigned_file', 'extra_regressors'],
704657
name='makecompcorrfilter')
658+
createfilter2.inputs.components_file = 'noise_components.txt'
705659
createfilter2.inputs.num_components = num_components
706660

707661
wf.connect(createfilter1, 'out_files', createfilter2, 'extra_regressors')
@@ -717,7 +671,7 @@ def merge_files(in1, in2):
717671
wf.connect(filter1, 'out_res', filter2, 'in_file')
718672
wf.connect(filter1, ('out_res', rename, '_cleaned'),
719673
filter2, 'out_res_name')
720-
wf.connect(createfilter2, 'out_files', filter2, 'design')
674+
wf.connect(createfilter2, 'components_file', filter2, 'design')
721675
wf.connect(mask, 'mask_file', filter2, 'mask')
722676

723677
bandpass = Node(Function(input_names=['files', 'lowpass_freq',
@@ -923,7 +877,7 @@ def get_names(files, suffix):
923877
wf.connect(smooth, 'out_file', datasink, 'resting.timeseries.@smoothed')
924878
wf.connect(createfilter1, 'out_files',
925879
datasink, 'resting.regress.@regressors')
926-
wf.connect(createfilter2, 'out_files',
880+
wf.connect(createfilter2, 'components_file',
927881
datasink, 'resting.regress.@compcorr')
928882
wf.connect(maskts, 'out_file', datasink, 'resting.timeseries.target')
929883
wf.connect(sampleaparc, 'summary_file',

nipype/algorithms/compcor.py

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
# -*- coding: utf-8 -*-
2+
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
3+
# vi: set ft=python sts=4 ts=4 sw=4 et:
4+
'''
5+
Miscellaneous algorithms
6+
7+
Change directory to provide relative paths for doctests
8+
>>> import os
9+
>>> filepath = os.path.dirname(os.path.realpath(__file__))
10+
>>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data'))
11+
>>> os.chdir(datadir)
12+
13+
'''
14+
from ..interfaces.base import (BaseInterfaceInputSpec, TraitedSpec,
15+
BaseInterface, traits, File)
16+
from ..pipeline import engine as pe
17+
from ..interfaces.utility import IdentityInterface
18+
from .misc import regress_poly
19+
20+
import nibabel as nb
21+
import numpy as np
22+
from scipy import linalg, stats
23+
import os
24+
25+
class CompCorInputSpec(BaseInterfaceInputSpec):
26+
realigned_file = File(exists=True, mandatory=True,
27+
desc='already realigned brain image (4D)')
28+
mask_file = File(exists=True, mandatory=False,
29+
desc='mask file that determines ROI (3D)')
30+
components_file = File('components_file.txt', exists=False,
31+
mandatory=False, usedefault=True,
32+
desc='filename to store physiological components')
33+
num_components = traits.Int(6, usedefault=True) # 6 for BOLD, 4 for ASL
34+
use_regress_poly = traits.Bool(True, usedefault=True,
35+
desc='use polynomial regression'
36+
'pre-component extraction')
37+
regress_poly_degree = traits.Range(low=1, default=1, usedefault=True,
38+
desc='the degree polynomial to use')
39+
40+
class CompCorOutputSpec(TraitedSpec):
41+
components_file = File(exists=True,
42+
desc='text file containing the noise components')
43+
44+
class CompCor(BaseInterface):
45+
'''
46+
Interface with core CompCor computation, used in aCompCor and tCompCor
47+
48+
Example
49+
-------
50+
51+
>>> ccinterface = CompCor()
52+
>>> ccinterface.inputs.realigned_file = 'functional.nii'
53+
>>> ccinterface.inputs.mask_file = 'mask.nii'
54+
>>> ccinterface.inputs.num_components = 1
55+
>>> ccinterface.inputs.use_regress_poly = True
56+
>>> ccinterface.inputs.regress_poly_degree = 2
57+
'''
58+
input_spec = CompCorInputSpec
59+
output_spec = CompCorOutputSpec
60+
61+
def _run_interface(self, runtime):
62+
imgseries = nb.load(self.inputs.realigned_file).get_data()
63+
mask = nb.load(self.inputs.mask_file).get_data()
64+
voxel_timecourses = imgseries[mask > 0]
65+
# Zero-out any bad values
66+
voxel_timecourses[np.isnan(np.sum(voxel_timecourses, axis=1)), :] = 0
67+
68+
# from paper:
69+
# "The constant and linear trends of the columns in the matrix M were
70+
# removed [prior to ...]"
71+
if self.inputs.use_regress_poly:
72+
voxel_timecourses = regress_poly(self.inputs.regress_poly_degree,
73+
voxel_timecourses)
74+
voxel_timecourses = voxel_timecourses - np.mean(voxel_timecourses,
75+
axis=1)[:, np.newaxis]
76+
77+
# "Voxel time series from the noise ROI (either anatomical or tSTD) were
78+
# placed in a matrix M of size Nxm, with time along the row dimension
79+
# and voxels along the column dimension."
80+
M = voxel_timecourses.T
81+
numvols = M.shape[0]
82+
numvoxels = M.shape[1]
83+
84+
# "[... were removed] prior to column-wise variance normalization."
85+
M = M / self._compute_tSTD(M, 1.)
86+
87+
# "The covariance matrix C = MMT was constructed and decomposed into its
88+
# principal components using a singular value decomposition."
89+
u, _, _ = linalg.svd(M, full_matrices=False)
90+
components = u[:, :self.inputs.num_components]
91+
components_file = os.path.join(os.getcwd(), self.inputs.components_file)
92+
np.savetxt(components_file, components, fmt="%.10f")
93+
return runtime
94+
95+
def _list_outputs(self):
96+
outputs = self._outputs().get()
97+
outputs['components_file'] = os.path.abspath(self.inputs.components_file)
98+
return outputs
99+
100+
def _compute_tSTD(self, M, x):
101+
stdM = np.std(M, axis=0)
102+
# set bad values to x
103+
stdM[stdM == 0] = x
104+
stdM[np.isnan(stdM)] = x
105+
return stdM
106+
107+
class TCompCorInputSpec(CompCorInputSpec):
108+
# and all the fields in CompCorInputSpec
109+
percentile_threshold = traits.Range(low=0., high=1., value=.02,
110+
exclude_low=True, exclude_high=True,
111+
usedefault=True, desc='the percentile '
112+
'used to select highest-variance '
113+
'voxels, represented by a number '
114+
'between 0 and 1, exclusive. By '
115+
'default, this value is set to .02. '
116+
'That is, the 2% of voxels '
117+
'with the highest variance are used.')
118+
119+
class TCompCor(CompCor):
120+
'''
121+
Interface for tCompCor. Computes a ROI mask based on variance of voxels.
122+
123+
Example
124+
-------
125+
126+
>>> ccinterface = TCompCor()
127+
>>> ccinterface.inputs.realigned_file = 'functional.nii'
128+
>>> ccinterface.inputs.mask_file = 'mask.nii'
129+
>>> ccinterface.inputs.num_components = 1
130+
>>> ccinterface.inputs.use_regress_poly = True
131+
>>> ccinterface.inputs.regress_poly_degree = 2
132+
>>> ccinterface.inputs.percentile_threshold = .03
133+
'''
134+
135+
input_spec = TCompCorInputSpec
136+
output_spec = CompCorOutputSpec
137+
138+
def _run_interface(self, runtime):
139+
imgseries = nb.load(self.inputs.realigned_file).get_data()
140+
141+
# From the paper:
142+
# "For each voxel time series, the temporal standard deviation is
143+
# defined as the standard deviation of the time series after the removal
144+
# of low-frequency nuisance terms (e.g., linear and quadratic drift)."
145+
imgseries = regress_poly(2, imgseries)
146+
imgseries = imgseries - np.mean(imgseries, axis=1)[:, np.newaxis]
147+
148+
time_voxels = imgseries.T
149+
num_voxels = np.prod(time_voxels.shape[1:])
150+
151+
# "To construct the tSTD noise ROI, we sorted the voxels by their
152+
# temporal standard deviation ..."
153+
tSTD = self._compute_tSTD(time_voxels, 0)
154+
sortSTD = np.sort(tSTD, axis=None) # flattened sorted matrix
155+
156+
# use percentile_threshold to pick voxels
157+
threshold_index = int(num_voxels * (1. - self.inputs.percentile_threshold))
158+
threshold_std = sortSTD[threshold_index]
159+
mask = tSTD >= threshold_std
160+
mask = mask.astype(int)
161+
162+
# save mask
163+
mask_file = 'mask.nii'
164+
nb.nifti1.save(nb.Nifti1Image(mask, np.eye(4)), mask_file)
165+
self.inputs.mask_file = mask_file
166+
167+
super(TCompCor, self)._run_interface(runtime)
168+
return runtime
169+
170+
ACompCor = CompCor

nipype/algorithms/misc.py

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
BaseInterfaceInputSpec, isdefined,
3636
DynamicTraitedSpec, Undefined)
3737
from ..utils.filemanip import fname_presuffix, split_filename
38+
3839
iflogger = logging.getLogger('interface')
3940

4041

@@ -257,7 +258,6 @@ def _list_outputs(self):
257258
outputs['nifti_file'] = self._gen_output_file_name()
258259
return outputs
259260

260-
261261
class TSNRInputSpec(BaseInterfaceInputSpec):
262262
in_file = InputMultiPath(File(exists=True), mandatory=True,
263263
desc='realigned 4D file or a list of 3D files')
@@ -308,17 +308,7 @@ def _run_interface(self, runtime):
308308
data = data.astype(np.float32)
309309

310310
if isdefined(self.inputs.regress_poly):
311-
timepoints = img.shape[-1]
312-
X = np.ones((timepoints, 1))
313-
for i in range(self.inputs.regress_poly):
314-
X = np.hstack((X, legendre(
315-
i + 1)(np.linspace(-1, 1, timepoints))[:, None]))
316-
betas = np.dot(np.linalg.pinv(X), np.rollaxis(data, 3, 2))
317-
datahat = np.rollaxis(np.dot(X[:, 1:],
318-
np.rollaxis(
319-
betas[1:, :, :, :], 0, 3)),
320-
0, 4)
321-
data = data - datahat
311+
data = regress_poly(self.inputs.regress_poly, data)
322312
img = nb.Nifti1Image(data, img.get_affine(), header)
323313
nb.save(img, op.abspath(self.inputs.detrended_file))
324314

@@ -343,6 +333,32 @@ def _list_outputs(self):
343333
outputs['detrended_file'] = op.abspath(self.inputs.detrended_file)
344334
return outputs
345335

336+
def regress_poly(degree, data):
337+
''' returns data with degree polynomial regressed out.
338+
The last dimension (i.e. data.shape[-1]) should be time.
339+
'''
340+
datashape = data.shape
341+
timepoints = datashape[-1]
342+
343+
# Rearrange all voxel-wise time-series in rows
344+
data = data.reshape((-1, timepoints))
345+
346+
# Generate design matrix
347+
X = np.ones((timepoints, 1))
348+
for i in range(degree):
349+
polynomial_func = legendre(i+1)
350+
value_array = np.linspace(-1, 1, timepoints)
351+
X = np.hstack((X, polynomial_func(value_array)[:, np.newaxis]))
352+
353+
# Calculate coefficients
354+
betas = np.linalg.pinv(X).dot(data.T)
355+
356+
# Estimation
357+
datahat = X[:, 1:].dot(betas[1:, ...]).T
358+
regressed_data = data - datahat
359+
360+
# Back to original shape
361+
return regressed_data.reshape(datashape)
346362

347363
class GunzipInputSpec(BaseInterfaceInputSpec):
348364
in_file = File(exists=True, mandatory=True)

0 commit comments

Comments
 (0)