From 831eb394a119aef96f24e10347d99029a60da398 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jan 2025 19:25:14 +0100 Subject: [PATCH 01/14] feat: add std_dvars_threshold to fMRIPrepConfoundRemover --- .../confounds/fmriprep_confound_remover.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 64d0ba362..971388cb5 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -95,6 +95,8 @@ FMRIPREP_VALID_NAMES = [ # NOTE: Check with @fraimondo about the spike mapping intent # Add spike_name to FMRIPREP_VALID_NAMES FMRIPREP_VALID_NAMES.append("framewise_displacement") +# Add std_dvars to FMRIPREP_VALID_NAMES +FMRIPREP_VALID_NAMES.append("std_dvars") @register_preprocessor @@ -130,6 +132,12 @@ class fMRIPrepConfoundRemover(BasePreprocessor): If None, no spike regressor is added. If spike is a float, it will add a spike regressor for every point at which framewise displacement exceeds the specified float (default None). + std_dvars_threshold : float, optional + Standardized DVARS threshold for scrub. DVARs is defined as root mean + squared intensity difference of volume N to volume N+1. D referring to + temporal derivative of timecourses, VARS referring to root mean squared + variance over voxels. If None, no scrubbing is performed + (default None). detrend : bool, optional If True, detrending will be applied on timeseries, before confound removal (default True). @@ -157,6 +165,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor): self, strategy: Optional[dict[str, str]] = None, spike: Optional[float] = None, + std_dvars_threshold: Optional[float] = None, detrend: bool = True, standardize: bool = True, low_pass: Optional[float] = None, @@ -173,6 +182,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor): } self.strategy = strategy self.spike = spike + self.std_dvars_threshold = std_dvars_threshold self.detrend = detrend self.standardize = standardize self.low_pass = low_pass @@ -355,6 +365,14 @@ class fMRIPrepConfoundRemover(BasePreprocessor): "Check if this file is really an fmriprep confounds file. " "You can also deactivate spike (set spike = None)." ) + # Add std_dvars + if self.std_dvars_threshold is not None: + if "std_dvars" not in available_vars: + raise_error( + "Invalid confounds file. Missing std_dvars " + "(standardized DVARS) confound. " + "Check if this file is really an fMRIPrep confounds file. " + ) out = to_select, squares_to_compute, derivatives_to_compute, spike_name return out @@ -404,6 +422,14 @@ class fMRIPrepConfoundRemover(BasePreprocessor): out_df["spike"] = fd to_select.append("spike") + # add binary std_dvars regressor if needed at given threshold + if self.std_dvars_threshold is not None: + std_dvars = out_df["std_dvars"].copy() + std_dvars.loc[std_dvars > self.std_dvars_threshold] = 1 + std_dvars.loc[std_dvars != 1] = 0 + out_df["std_dvars"] = std_dvars + to_select.append("std_dvars") + # Now pick all the relevant confounds out_df = out_df[to_select] -- 2.52.0 From b33358f129be04b6fcd1b11e57dd948d424f15bd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jan 2025 19:25:38 +0100 Subject: [PATCH 02/14] chore: improve commentary in fMRIPrepConfoundRemover --- junifer/preprocess/confounds/fmriprep_confound_remover.py | 1 + 1 file changed, 1 insertion(+) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 971388cb5..9a496c944 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -356,6 +356,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor): to_select.append(x_derivative_2) if x_derivative_2 not in available_vars: squares_to_compute[x_derivative_2] = x_derivative + # Add spike spike_name = "framewise_displacement" if self.spike is not None: if spike_name not in available_vars: -- 2.52.0 From 2baa31719139c4e35d36d3da3f13e836f0c1426b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jan 2025 19:33:13 +0100 Subject: [PATCH 03/14] chore: add changelogs 421.{change,enh} --- docs/changes/newsfragments/421.change | 1 + docs/changes/newsfragments/421.enh | 1 + 2 files changed, 2 insertions(+) create mode 100644 docs/changes/newsfragments/421.change create mode 100644 docs/changes/newsfragments/421.enh diff --git a/docs/changes/newsfragments/421.change b/docs/changes/newsfragments/421.change new file mode 100644 index 000000000..4d4d9d3f9 --- /dev/null +++ b/docs/changes/newsfragments/421.change @@ -0,0 +1 @@ +Add ``std_vars_threshold`` parameter to :class:`.fMRIPrepConfoundRemover` by `Synchon Mandal`_ diff --git a/docs/changes/newsfragments/421.enh b/docs/changes/newsfragments/421.enh new file mode 100644 index 000000000..adf837b06 --- /dev/null +++ b/docs/changes/newsfragments/421.enh @@ -0,0 +1 @@ +Add scrubbing support to :class:`.fMRIPrepConfoundRemover` by `Synchon Mandal`_ -- 2.52.0 From c7f328dd648554dd699678c1449f42215faff48f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Jan 2025 18:14:33 +0100 Subject: [PATCH 04/14] fix: use sample_mask of nilearn.signal.clean for scrubbing in fMRIPrepConfoundRemover --- .../confounds/fmriprep_confound_remover.py | 59 ++++++++++++++----- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 9a496c944..b055cdd5c 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -366,14 +366,6 @@ class fMRIPrepConfoundRemover(BasePreprocessor): "Check if this file is really an fmriprep confounds file. " "You can also deactivate spike (set spike = None)." ) - # Add std_dvars - if self.std_dvars_threshold is not None: - if "std_dvars" not in available_vars: - raise_error( - "Invalid confounds file. Missing std_dvars " - "(standardized DVARS) confound. " - "Check if this file is really an fMRIPrep confounds file. " - ) out = to_select, squares_to_compute, derivatives_to_compute, spike_name return out @@ -423,14 +415,6 @@ class fMRIPrepConfoundRemover(BasePreprocessor): out_df["spike"] = fd to_select.append("spike") - # add binary std_dvars regressor if needed at given threshold - if self.std_dvars_threshold is not None: - std_dvars = out_df["std_dvars"].copy() - std_dvars.loc[std_dvars > self.std_dvars_threshold] = 1 - std_dvars.loc[std_dvars != 1] = 0 - out_df["std_dvars"] = std_dvars - to_select.append("std_dvars") - # Now pick all the relevant confounds out_df = out_df[to_select] @@ -442,6 +426,41 @@ class fMRIPrepConfoundRemover(BasePreprocessor): return out_df + def _get_scrub_mask(self, input: dict[str, Any]) -> np.ndarray: + """Get boolean mask for scrubbing. + + Parameters + ---------- + input : dict + Dictionary containing the ``BOLD.confounds`` value from the + Junifer Data object. + + Returns + ------- + numpy.ndarray + Index of volumes to be kept. + + Raises + ------ + RuntimeError + If ``std_dvars`` is not found in the confounds file. + + """ + confounds_df = input["data"] + # Check confounds file + if "std_dvars" not in confounds_df.columns: + raise_error( + "Invalid confounds file. Missing std_dvars " + "(standardized DVARS) confound. " + "Check if this file is really an fMRIPrep confounds file. " + ) + # Make first row 0 and then threshold + return np.flatnonzero( + ( + confounds_df["std_dvars"].fillna(0) > self.std_dvars_threshold + ).to_numpy() + ) + def _validate_data( self, input: dict[str, Any], @@ -605,6 +624,13 @@ class fMRIPrepConfoundRemover(BasePreprocessor): } } ) + + # Set up scrubbing mask if needed + if self.std_dvars_threshold is not None: + sample_mask = self._get_scrub_mask(input["confounds"]) + else: + sample_mask = None + # Clean image logger.info("Cleaning image using nilearn") logger.debug(f"\tdetrend: {self.detrend}") @@ -622,6 +648,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor): high_pass=self.high_pass, t_r=t_r, mask_img=mask_img, + clean__sample_mask=sample_mask, ) # Fix t_r as nilearn messes it up cleaned_img.header["pixdim"][4] = t_r -- 2.52.0 From 00b48d6ae244805fbf3a2ad18d139940b8ea7f36 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 29 Jan 2025 10:45:18 +0100 Subject: [PATCH 05/14] update: select std_dvars confounds when scrubbing in fMRIPrepConfoundRemover --- .../confounds/fmriprep_confound_remover.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index b055cdd5c..133ef0362 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -415,6 +415,10 @@ class fMRIPrepConfoundRemover(BasePreprocessor): out_df["spike"] = fd to_select.append("spike") + # Add std_dvars to to_select if scrubbing is required + if self.std_dvars_threshold is not None: + to_select.append("std_dvars") + # Now pick all the relevant confounds out_df = out_df[to_select] @@ -426,14 +430,13 @@ class fMRIPrepConfoundRemover(BasePreprocessor): return out_df - def _get_scrub_mask(self, input: dict[str, Any]) -> np.ndarray: + def _get_scrub_mask(self, confounds_df: pd.DataFrame) -> np.ndarray: """Get boolean mask for scrubbing. Parameters ---------- - input : dict - Dictionary containing the ``BOLD.confounds`` value from the - Junifer Data object. + confounds_df : pandas.DataFrame + pandas.DataFrame with selected confounds. Returns ------- @@ -443,11 +446,10 @@ class fMRIPrepConfoundRemover(BasePreprocessor): Raises ------ RuntimeError - If ``std_dvars`` is not found in the confounds file. + If ``std_dvars`` is not found in ``confounds_df``. """ - confounds_df = input["data"] - # Check confounds file + # Check confounds columns if "std_dvars" not in confounds_df.columns: raise_error( "Invalid confounds file. Missing std_dvars " -- 2.52.0 From 3fdbb958b4a85e528d33b3fa285bef33bddfe669 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 29 Jan 2025 10:46:08 +0100 Subject: [PATCH 06/14] update: improve logic for forwarding kwargs to nilearn.signal.clean in fMRIPrepConfoundRemover --- .../confounds/fmriprep_confound_remover.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 133ef0362..00fda6117 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -627,11 +627,15 @@ class fMRIPrepConfoundRemover(BasePreprocessor): } ) + signal_clean_kwargs = {} # Set up scrubbing mask if needed if self.std_dvars_threshold is not None: - sample_mask = self._get_scrub_mask(input["confounds"]) - else: - sample_mask = None + sample_mask = self._get_scrub_mask(confounds_df) + signal_clean_kwargs.update( + { + "clean__sample_mask": sample_mask, + } + ) # Clean image logger.info("Cleaning image using nilearn") @@ -650,7 +654,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor): high_pass=self.high_pass, t_r=t_r, mask_img=mask_img, - clean__sample_mask=sample_mask, + **signal_clean_kwargs, ) # Fix t_r as nilearn messes it up cleaned_img.header["pixdim"][4] = t_r -- 2.52.0 From f54f064afcba1c11c7bf2a983e3d47e266e79aa4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 11 Feb 2025 16:38:16 +0100 Subject: [PATCH 07/14] update: rework scrubbing and leverage nilearn completely for fMRIPrepConfoundRemover --- .../confounds/fmriprep_confound_remover.py | 178 ++++++++++++------ 1 file changed, 120 insertions(+), 58 deletions(-) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 00fda6117..36aa9e7b8 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -17,6 +17,8 @@ import numpy as np import pandas as pd from nilearn import image as nimg from nilearn._utils.niimg_conversions import check_niimg_4d +from nilearn.interfaces.fmriprep.load_confounds_components import _load_scrub +from nilearn.interfaces.fmriprep.load_confounds_utils import prepare_output from ...api.decorators import register_preprocessor from ...data import get_data @@ -95,8 +97,6 @@ FMRIPREP_VALID_NAMES = [ # NOTE: Check with @fraimondo about the spike mapping intent # Add spike_name to FMRIPREP_VALID_NAMES FMRIPREP_VALID_NAMES.append("framewise_displacement") -# Add std_dvars to FMRIPREP_VALID_NAMES -FMRIPREP_VALID_NAMES.append("std_dvars") @register_preprocessor @@ -112,13 +112,15 @@ class fMRIPrepConfoundRemover(BasePreprocessor): ---------- strategy : dict, optional The strategy to use for each component. If None, will use the *full* - strategy for all components (default None). + strategy for all components except ``"scrubbing"`` which will be set + to False (default None). The keys of the dictionary should correspond to names of noise components to include: * ``motion`` * ``wm_csf`` * ``global_signal`` + * ``scrubbing`` The values of dictionary should correspond to types of confounds extracted from each signal: @@ -128,15 +130,28 @@ class fMRIPrepConfoundRemover(BasePreprocessor): * ``derivatives`` : signal + derivatives * ``full`` : signal + deriv. + quadratic terms + power2 deriv. + except ``scrubbing`` which needs to be bool. spike : float, optional If None, no spike regressor is added. If spike is a float, it will add a spike regressor for every point at which framewise displacement exceeds the specified float (default None). + scrub : int, optional + After accounting for time frames with excessive motion, further remove + segments shorter than the given number. When the value is 0, remove + time frames based on excessive framewise displacement and DVARS only. + If None and no ``"scrubbing"`` in ``strategy``, no scrubbing is + performed, else the default value is 0. The default value is referred + as full scrubbing (default None). + fd_threshold : float, optional + Framewise displacement threshold for scrub in mm. If None no + ``"scrubbing"`` in ``strategy``, no scrubbing is performed, else the + default value is 0.5 (default None). std_dvars_threshold : float, optional Standardized DVARS threshold for scrub. DVARs is defined as root mean squared intensity difference of volume N to volume N+1. D referring to temporal derivative of timecourses, VARS referring to root mean squared - variance over voxels. If None, no scrubbing is performed + variance over voxels. If None and no ``"scrubbing"`` in ``strategy``, + no scrubbing is performed, else the default value is 1.5 (default None). detrend : bool, optional If True, detrending will be applied on timeseries, before confound @@ -163,8 +178,10 @@ class fMRIPrepConfoundRemover(BasePreprocessor): def __init__( self, - strategy: Optional[dict[str, str]] = None, + strategy: Optional[dict[str, Union[str, bool]]] = None, spike: Optional[float] = None, + scrub: Optional[int] = None, + fd_threshold: Optional[float] = None, std_dvars_threshold: Optional[float] = None, detrend: bool = True, standardize: bool = True, @@ -179,9 +196,12 @@ class fMRIPrepConfoundRemover(BasePreprocessor): "motion": "full", "wm_csf": "full", "global_signal": "full", + "scrubbing": False, } self.strategy = strategy self.spike = spike + self.scrub = scrub + self.fd_threshold = fd_threshold self.std_dvars_threshold = std_dvars_threshold self.detrend = detrend self.standardize = standardize @@ -190,13 +210,22 @@ class fMRIPrepConfoundRemover(BasePreprocessor): self.t_r = t_r self.masks = masks - self._valid_components = ["motion", "wm_csf", "global_signal"] + self._valid_components = [ + "motion", + "wm_csf", + "global_signal", + "scrubbing", + ] self._valid_confounds = ["basic", "power2", "derivatives", "full"] if any(not isinstance(k, str) for k in strategy.keys()): raise_error("Strategy keys must be strings", ValueError) - if any(not isinstance(v, str) for v in strategy.values()): + if any( + not isinstance(v, str) + for k, v in strategy.items() + if k != "scrubbing" + ): raise_error("Strategy values must be strings", ValueError) if any(x not in self._valid_components for x in strategy.keys()): @@ -209,7 +238,11 @@ class fMRIPrepConfoundRemover(BasePreprocessor): klass=ValueError, ) - if any(x not in self._valid_confounds for x in strategy.values()): + if any( + v not in self._valid_confounds + for k, v in strategy.items() + if k != "scrubbing" + ): raise_error( msg=f"Invalid confound types {list(strategy.values())}. " f"Valid confound types are {self._valid_confounds}.\n" @@ -325,37 +358,41 @@ class fMRIPrepConfoundRemover(BasePreprocessor): derivatives_to_compute = {} # the dictionary of missing derivatives for t_kind, t_strategy in self.strategy.items(): - t_basics = FMRIPREP_BASICS[t_kind] + if t_kind != "scrubbing": + t_basics = FMRIPREP_BASICS[t_kind] - if any(x not in available_vars for x in t_basics): - missing = [x for x in t_basics if x not in available_vars] - raise_error( - "Invalid confounds file. Missing basic confounds: " - f"{missing}. " - "Check if this file is really an fmriprep confounds file. " - "You can also modify the confound removal strategy." - ) + if any(x not in available_vars for x in t_basics): + missing = [x for x in t_basics if x not in available_vars] + raise_error( + "Invalid confounds file. Missing basic confounds: " + f"{missing}. " + "Check if this file is really an fmriprep confounds " + "file. You can also modify the confound removal " + "strategy." + ) - to_select.extend(t_basics) + to_select.extend(t_basics) - if t_strategy in ["power2", "full"]: - for x in t_basics: - x_2 = f"{x}_power2" - to_select.append(x_2) - if x_2 not in available_vars: - squares_to_compute[x_2] = x + if t_strategy in ["power2", "full"]: + for x in t_basics: + x_2 = f"{x}_power2" + to_select.append(x_2) + if x_2 not in available_vars: + squares_to_compute[x_2] = x - if t_strategy in ["derivatives", "full"]: - for x in t_basics: - x_derivative = f"{x}_derivative1" - to_select.append(x_derivative) - if x_derivative not in available_vars: - derivatives_to_compute[x_derivative] = x - if t_strategy == "full": - x_derivative_2 = f"{x_derivative}_power2" - to_select.append(x_derivative_2) - if x_derivative_2 not in available_vars: - squares_to_compute[x_derivative_2] = x_derivative + if t_strategy in ["derivatives", "full"]: + for x in t_basics: + x_derivative = f"{x}_derivative1" + to_select.append(x_derivative) + if x_derivative not in available_vars: + derivatives_to_compute[x_derivative] = x + if t_strategy == "full": + x_derivative_2 = f"{x_derivative}_power2" + to_select.append(x_derivative_2) + if x_derivative_2 not in available_vars: + squares_to_compute[x_derivative_2] = ( + x_derivative + ) # Add spike spike_name = "framewise_displacement" if self.spike is not None: @@ -388,14 +425,12 @@ class fMRIPrepConfoundRemover(BasePreprocessor): if confounds_format == "adhoc": self._map_adhoc_to_fmriprep(input) - processed_spec = self._process_fmriprep_spec(input) - ( to_select, squares_to_compute, derivatives_to_compute, spike_name, - ) = processed_spec + ) = self._process_fmriprep_spec(input) # Copy the confounds out_df = input["data"].copy() @@ -415,10 +450,6 @@ class fMRIPrepConfoundRemover(BasePreprocessor): out_df["spike"] = fd to_select.append("spike") - # Add std_dvars to to_select if scrubbing is required - if self.std_dvars_threshold is not None: - to_select.append("std_dvars") - # Now pick all the relevant confounds out_df = out_df[to_select] @@ -430,37 +461,59 @@ class fMRIPrepConfoundRemover(BasePreprocessor): return out_df - def _get_scrub_mask(self, confounds_df: pd.DataFrame) -> np.ndarray: - """Get boolean mask for scrubbing. + def _get_scrub_regressors( + self, confounds_df: pd.DataFrame + ) -> pd.DataFrame: + """Get motion outline regressors. Parameters ---------- confounds_df : pandas.DataFrame - pandas.DataFrame with selected confounds. + pandas.DataFrame with all confounds. Returns ------- - numpy.ndarray - Index of volumes to be kept. + pandas.DataFrame + pandas.DataFrame of motion outline regressors. Raises ------ RuntimeError - If ``std_dvars`` is not found in ``confounds_df``. + If ``std_dvars`` and / or ``framewise_displacement`` is not found + in ``confounds_df``. """ - # Check confounds columns - if "std_dvars" not in confounds_df.columns: + # Check columns + if ( + self.std_dvars_threshold is not None + and "std_dvars" not in confounds_df.columns + ): raise_error( "Invalid confounds file. Missing std_dvars " "(standardized DVARS) confound. " "Check if this file is really an fMRIPrep confounds file. " ) - # Make first row 0 and then threshold - return np.flatnonzero( - ( - confounds_df["std_dvars"].fillna(0) > self.std_dvars_threshold - ).to_numpy() + if ( + self.fd_threshold is not None + and "framewise_displacement" not in confounds_df.columns + ): + raise_error( + "Invalid confounds file. Missing framewise_displacement " + "confound. " + "Check if this file is really an fMRIPrep confounds file. " + ) + # Use function from nilearn to not reinvent the wheel + return _load_scrub( + confounds_raw=confounds_df, + scrub=self.scrub if self.scrub is not None else 0, + fd_threshold=( + self.fd_threshold if self.fd_threshold is not None else 0.5 + ), + std_dvars_threshold=( + self.std_dvars_threshold + if self.std_dvars_threshold is not None + else 1.5 + ), ) def _validate_data( @@ -629,14 +682,23 @@ class fMRIPrepConfoundRemover(BasePreprocessor): signal_clean_kwargs = {} # Set up scrubbing mask if needed - if self.std_dvars_threshold is not None: - sample_mask = self._get_scrub_mask(confounds_df) + if self.strategy.get("scrubbing", False): + motion_outline_regressors = self._get_scrub_regressors( + input["confounds"]["data"] + ) + # Add regressors to confounds + confounds_df = pd.concat( + [confounds_df, motion_outline_regressors], axis=1 + ) + # Get sample mask + sample_mask, confounds_df = prepare_output( + confounds=confounds_df, demean=False + ) signal_clean_kwargs.update( { "clean__sample_mask": sample_mask, } ) - # Clean image logger.info("Cleaning image using nilearn") logger.debug(f"\tdetrend: {self.detrend}") -- 2.52.0 From c1cf8c272650f29d6ed579f478aa04fa6f1d2743 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 11 Feb 2025 16:39:02 +0100 Subject: [PATCH 08/14] update: improve logging for fMRIPrepConfoundRemover --- junifer/preprocess/confounds/fmriprep_confound_remover.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 36aa9e7b8..6b74502c0 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -701,10 +701,16 @@ class fMRIPrepConfoundRemover(BasePreprocessor): ) # Clean image logger.info("Cleaning image using nilearn") + logger.debug(f"\tstrategy: {self.strategy}") + logger.debug(f"\tspike: {self.spike}") + logger.debug(f"\tscrub: {self.scrub}") + logger.debug(f"\tfd_threshold: {self.fd_threshold}") + logger.debug(f"\tstd_dvars_threshold: {self.std_dvars_threshold}") logger.debug(f"\tdetrend: {self.detrend}") logger.debug(f"\tstandardize: {self.standardize}") logger.debug(f"\tlow_pass: {self.low_pass}") logger.debug(f"\thigh_pass: {self.high_pass}") + logger.debug(f"\tt_r: {self.t_r}") # Deconfound data cleaned_img = nimg.clean_img( -- 2.52.0 From 2a017852f95f64d3c4540f564d90abbbff4dfd2a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 11 Feb 2025 16:39:18 +0100 Subject: [PATCH 09/14] chore: lint --- .../external/nilearn/tests/test_junifer_connectivity_measure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/external/nilearn/tests/test_junifer_connectivity_measure.py b/junifer/external/nilearn/tests/test_junifer_connectivity_measure.py index 61bb553d4..e5ccb1be5 100644 --- a/junifer/external/nilearn/tests/test_junifer_connectivity_measure.py +++ b/junifer/external/nilearn/tests/test_junifer_connectivity_measure.py @@ -185,7 +185,7 @@ def signals() -> list[np.ndarray]: @pytest.fixture def signals_and_covariances( - cov_estimator: Union[LedoitWolf, EmpiricalCovariance] + cov_estimator: Union[LedoitWolf, EmpiricalCovariance], ) -> tuple[list[np.ndarray], list[float]]: """Return signals and covariances for a covariance estimator. -- 2.52.0 From 552ec38375c2db9bbb853293f1f32faa9d0f7c0c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 11 Feb 2025 17:12:50 +0100 Subject: [PATCH 10/14] update: add scrubbing test for fMRIPrepConfoundRemover --- .../tests/test_fmriprep_confound_remover.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py index 97fbf9afb..dccd4bf25 100644 --- a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py @@ -567,3 +567,41 @@ def test_fMRIPrepConfoundRemover_fit_transform_masks() -> None: assert "dependencies" in output["BOLD"]["meta"] dependencies = output["BOLD"]["meta"]["dependencies"] assert dependencies == {"numpy", "nilearn"} + + +def test_fMRIPrepConfoundRemover_scrubbing() -> None: + """Test fMRIPrepConfoundRemover with scrubbing.""" + confound_remover = fMRIPrepConfoundRemover( + strategy={ + "motion": "full", + "wm_csf": "full", + "global_signal": "full", + "scrubbing": True, + }, + ) + + with PartlyCloudyTestingDataGrabber(reduce_confounds=False) as dg: + element_data = DefaultDataReader().fit_transform(dg["sub-01"]) + orig_bold = element_data["BOLD"]["data"].get_fdata().copy() + pre_input = element_data["BOLD"] + pre_extra_input = { + "BOLD": {"confounds": element_data["BOLD"]["confounds"]} + } + output, _ = confound_remover.preprocess(pre_input, pre_extra_input) + trans_bold = output["data"].get_fdata() + # Transformation is in place + assert_array_equal( + trans_bold, element_data["BOLD"]["data"].get_fdata() + ) + + # Data should have different shape + assert_raises( + AssertionError, + assert_array_equal, + orig_bold.shape, + trans_bold.shape, + ) + # and be different + assert_raises( + AssertionError, assert_array_equal, orig_bold, trans_bold + ) -- 2.52.0 From a87805bf1531db9796bcf6de389e4ad61a50e8f9 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 11 Feb 2025 17:24:22 +0100 Subject: [PATCH 11/14] chore: update changelog 421.change --- docs/changes/newsfragments/421.change | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes/newsfragments/421.change b/docs/changes/newsfragments/421.change index 4d4d9d3f9..8f1e41aa0 100644 --- a/docs/changes/newsfragments/421.change +++ b/docs/changes/newsfragments/421.change @@ -1 +1 @@ -Add ``std_vars_threshold`` parameter to :class:`.fMRIPrepConfoundRemover` by `Synchon Mandal`_ +Add ``scrub``, ``fd_threshold`` and ``std_vars_threshold`` parameters to :class:`.fMRIPrepConfoundRemover` and allow ``"scrubbing"`` key of type bool for ``strategy`` by `Synchon Mandal`_ -- 2.52.0 From e917c6967db301e52230926daef19f054bf422f4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Feb 2025 10:19:47 +0100 Subject: [PATCH 12/14] update: improve test for fMRIPrepConfoundRemover scrubbing --- .../tests/test_fmriprep_confound_remover.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py index dccd4bf25..d6d0b6f69 100644 --- a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py @@ -10,6 +10,7 @@ import numpy as np import pandas as pd import pytest from nilearn._utils.exceptions import DimensionError +from nilearn.interfaces.fmriprep.load_confounds_utils import prepare_output from numpy.testing import assert_array_equal, assert_raises from pandas.testing import assert_frame_equal @@ -583,6 +584,7 @@ def test_fMRIPrepConfoundRemover_scrubbing() -> None: with PartlyCloudyTestingDataGrabber(reduce_confounds=False) as dg: element_data = DefaultDataReader().fit_transform(dg["sub-01"]) orig_bold = element_data["BOLD"]["data"].get_fdata().copy() + orig_confounds = element_data["BOLD"]["confounds"].copy() pre_input = element_data["BOLD"] pre_extra_input = { "BOLD": {"confounds": element_data["BOLD"]["confounds"]} @@ -605,3 +607,25 @@ def test_fMRIPrepConfoundRemover_scrubbing() -> None: assert_raises( AssertionError, assert_array_equal, orig_bold, trans_bold ) + + # Check scrubbing process + # Should be at the start + confounds_df = confound_remover._pick_confounds(orig_confounds) + assert confounds_df.shape == (168, 36) + # Should have 4 motion outliers based on threshold + motion_outline_regressors = confound_remover._get_scrub_regressors( + orig_confounds["data"] + ) + assert motion_outline_regressors.shape == (168, 4) + # Add regressors to confounds + concat_confounds_df = pd.concat( + [confounds_df, motion_outline_regressors], axis=1 + ) + assert concat_confounds_df.shape == (168, 40) + # Get sample mask and correct confounds + sample_mask, final_confounds_df = prepare_output( + confounds=concat_confounds_df, demean=False + ) + assert not confounds_df.equals(final_confounds_df) + assert sample_mask.shape == (164,) + assert not (np.isin([91, 92, 93, 113], sample_mask)).all() -- 2.52.0 From 5b287ea5fde16c00b15f321d41927cc749a42356 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Feb 2025 09:54:12 +0100 Subject: [PATCH 13/14] update: improve tests for fMRIPrepConfoundRemover --- .../confounds/fmriprep_confound_remover.py | 18 ++++++++----- .../tests/test_fmriprep_confound_remover.py | 26 +++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 6b74502c0..92e95b76a 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -489,18 +489,24 @@ class fMRIPrepConfoundRemover(BasePreprocessor): and "std_dvars" not in confounds_df.columns ): raise_error( - "Invalid confounds file. Missing std_dvars " - "(standardized DVARS) confound. " - "Check if this file is really an fMRIPrep confounds file. " + msg=( + "Invalid confounds file. Missing std_dvars " + "(standardized DVARS) confound. " + "Check if this file is really an fMRIPrep confounds file. " + ), + klass=RuntimeError, ) if ( self.fd_threshold is not None and "framewise_displacement" not in confounds_df.columns ): raise_error( - "Invalid confounds file. Missing framewise_displacement " - "confound. " - "Check if this file is really an fMRIPrep confounds file. " + msg=( + "Invalid confounds file. Missing framewise_displacement " + "confound. " + "Check if this file is really an fMRIPrep confounds file. " + ), + klass=RuntimeError, ) # Use function from nilearn to not reinvent the wheel return _load_scrub( diff --git a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py index d6d0b6f69..8d547b30b 100644 --- a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py @@ -321,6 +321,32 @@ def test_fMRIPRepConfoundRemover__pick_confounds_fmriprep_compute() -> None: assert_frame_equal(out_junifer, out_fmriprep) +@pytest.mark.parametrize( + "preprocessor", + [ + fMRIPrepConfoundRemover( + std_dvars_threshold=1.5, + ), + fMRIPrepConfoundRemover( + fd_threshold=0.5, + ), + ], +) +def test_fMRIPrepConfoundRemover__get_scrub_regressors_errors( + preprocessor: type, +) -> None: + """Test fMRIPrepConfoundRemover scrub regressors errors. + + Parameters + ---------- + preprocessor : object + The parametrized preprocessor. + + """ + with pytest.raises(RuntimeError, match="Invalid confounds file."): + preprocessor._get_scrub_regressors(pd.DataFrame({"a": [1, 2]})) + + def test_fMRIPrepConfoundRemover__validate_data() -> None: """Test fMRIPrepConfoundRemover validate data.""" confound_remover = fMRIPrepConfoundRemover(strategy={"wm_csf": "full"}) -- 2.52.0 From e5ada553ac8db5a07299720a80c8cc37baad096d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Feb 2025 09:54:37 +0100 Subject: [PATCH 14/14] update: improve exception type in fMRIPrepConfoundRemover --- .../confounds/fmriprep_confound_remover.py | 27 ++++++++++++------- .../tests/test_fmriprep_confound_remover.py | 4 +-- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/junifer/preprocess/confounds/fmriprep_confound_remover.py b/junifer/preprocess/confounds/fmriprep_confound_remover.py index 92e95b76a..2cce6714b 100644 --- a/junifer/preprocess/confounds/fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/fmriprep_confound_remover.py @@ -345,7 +345,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor): Raises ------ - ValueError + RuntimeError If invalid confounds file is found. """ @@ -364,11 +364,14 @@ class fMRIPrepConfoundRemover(BasePreprocessor): if any(x not in available_vars for x in t_basics): missing = [x for x in t_basics if x not in available_vars] raise_error( - "Invalid confounds file. Missing basic confounds: " - f"{missing}. " - "Check if this file is really an fmriprep confounds " - "file. You can also modify the confound removal " - "strategy." + msg=( + "Invalid confounds file. Missing basic confounds: " + f"{missing}. " + "Check if this file is really an fmriprep " + "confounds file. You can also modify the confound " + "removal strategy." + ), + klass=RuntimeError, ) to_select.extend(t_basics) @@ -398,10 +401,14 @@ class fMRIPrepConfoundRemover(BasePreprocessor): if self.spike is not None: if spike_name not in available_vars: raise_error( - "Invalid confounds file. Missing framewise_displacement " - "(spike) confound. " - "Check if this file is really an fmriprep confounds file. " - "You can also deactivate spike (set spike = None)." + msg=( + "Invalid confounds file. Missing " + "framewise_displacement (spike) confound. " + "Check if this file is really an fmriprep confounds " + "file. You can also deactivate spike " + "(set spike = None)." + ), + klass=RuntimeError, ) out = to_select, squares_to_compute, derivatives_to_compute, spike_name return out diff --git a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py index 8d547b30b..51bfafb01 100644 --- a/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py +++ b/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py @@ -212,7 +212,7 @@ def test_fMRIPrepConfoundRemover__process_fmriprep_spec() -> None: ) msg = r"Missing basic confounds: \['white_matter'\]" - with pytest.raises(ValueError, match=msg): + with pytest.raises(RuntimeError, match=msg): confound_remover._process_fmriprep_spec({"data": confounds_df}) var_names = ["csf", "white_matter"] @@ -221,7 +221,7 @@ def test_fMRIPrepConfoundRemover__process_fmriprep_spec() -> None: ) msg = r"Missing framewise_displacement" - with pytest.raises(ValueError, match=msg): + with pytest.raises(RuntimeError, match=msg): confound_remover._process_fmriprep_spec({"data": confounds_df}) -- 2.52.0