From 2d9679bd2f643fd567ef2db77b65ab7010a7e335 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 19 Nov 2024 15:00:52 +0100 Subject: [PATCH 1/7] fix: correct meta update in MultipleDataGrabber --- junifer/datagrabber/multiple.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/junifer/datagrabber/multiple.py b/junifer/datagrabber/multiple.py index fef6e409d..5f9f3c9d8 100644 --- a/junifer/datagrabber/multiple.py +++ b/junifer/datagrabber/multiple.py @@ -111,8 +111,12 @@ class MultipleDataGrabber(BaseDataGrabber): # Update all the metas again for kind in out: - self.update_meta(out[kind], "datagrabber") - out[kind]["meta"]["datagrabber"]["datagrabbers"] = metas + to_update = out[kind] + if not isinstance(to_update, list): + to_update = [to_update] + for t_kind in to_update: + self.update_meta(t_kind, "datagrabber") + t_kind["meta"]["datagrabber"]["datagrabbers"] = metas return out def __enter__(self) -> "MultipleDataGrabber": -- 2.52.0 From 433a8bd9d0575c2f2d92c35d9f1e86f6db50d4b6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 19 Nov 2024 17:04:25 +0100 Subject: [PATCH 2/7] chore: update testing datasets' scripts --- tools/create_bids_example_dataset.py | 2 ++ tools/create_bids_example_dataset_sessions.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/create_bids_example_dataset.py b/tools/create_bids_example_dataset.py index 7b755eccb..9dc56bd2e 100644 --- a/tools/create_bids_example_dataset.py +++ b/tools/create_bids_example_dataset.py @@ -26,6 +26,8 @@ with TemporaryDirectory() as tmpdir_name: fnames = [ f"anat/{t_sub}_T1w.nii.gz", f"anat/{t_sub}_brain_mask.nii.gz", + f"anat/{t_sub}_from-MNI152NLin2009cAsym_to-T1w_xfm.h5", + f"anat/{t_sub}_from-T1w_to-MNI152NLin2009cAsym_xfm.h5", f"func/{t_sub}_task-rest_bold.nii.gz", f"func/{t_sub}_task-rest_bold.json", f"func/{t_sub}_task-rest_brain_mask.nii.gz", diff --git a/tools/create_bids_example_dataset_sessions.py b/tools/create_bids_example_dataset_sessions.py index b37120bb2..b642eab1d 100644 --- a/tools/create_bids_example_dataset_sessions.py +++ b/tools/create_bids_example_dataset_sessions.py @@ -31,6 +31,8 @@ with TemporaryDirectory() as tmpdir_name: fnames = [ f"anat/{t_sub}_{t_ses}_T1w.nii.gz", f"anat/{t_sub}_{t_ses}_brain_mask.nii.gz", + f"anat/{t_sub}_{t_ses}_from-MNI152NLin2009cAsym_to-T1w_xfm.h5", + f"anat/{t_sub}_{t_ses}_from-T1w_to-MNI152NLin2009cAsym_xfm.h5", ] if i_ses != 3: # Session 3 does not have functional data fnames.extend( -- 2.52.0 From ca4b333c1ce77bf49cd2981fb6dffa7084806e6b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 19 Nov 2024 17:05:01 +0100 Subject: [PATCH 3/7] update: adapt tests for MultipleDataGrabber --- junifer/datagrabber/tests/test_multiple.py | 50 +++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/junifer/datagrabber/tests/test_multiple.py b/junifer/datagrabber/tests/test_multiple.py index 24e7eb301..78afd2889 100644 --- a/junifer/datagrabber/tests/test_multiple.py +++ b/junifer/datagrabber/tests/test_multiple.py @@ -29,7 +29,7 @@ def test_MultipleDataGrabber() -> None: dg1 = PatternDataladDataGrabber( rootdir=rootdir, uri=repo_uri, - types=["T1w"], + types=["T1w", "Warp"], patterns={ "T1w": { "pattern": ( @@ -44,6 +44,28 @@ def test_MultipleDataGrabber() -> None: "space": "native", }, }, + "Warp": [ + { + "pattern": ( + "{subject}/{session}/anat/" + "{subject}_{session}_from-MNI152NLin2009cAsym_to-T1w_" + "xfm.h5" + ), + "src": "MNI152NLin2009cAsym", + "dst": "native", + "warper": "ants", + }, + { + "pattern": ( + "{subject}/{session}/anat/" + "{subject}_{session}_from-T1w_to-MNI152NLin2009cAsym_" + "xfm.h5" + ), + "src": "native", + "dst": "MNI152NLin2009cAsym", + "warper": "ants", + }, + ], }, replacements=replacements, ) @@ -75,6 +97,7 @@ def test_MultipleDataGrabber() -> None: types = dg.get_types() assert "T1w" in types + assert "Warp" in types assert "BOLD" in types expected_subs = [ @@ -90,6 +113,7 @@ def test_MultipleDataGrabber() -> None: elem = dg[("sub-01", "ses-01")] # Check data types assert "T1w" in elem + assert "Warp" in elem assert "BOLD" in elem # Check meta assert "meta" in elem["BOLD"] @@ -111,7 +135,7 @@ def test_MultipleDataGrabber_no_intersection() -> None: dg1 = PatternDataladDataGrabber( rootdir=rootdir, uri=_testing_dataset["example_bids"]["uri"], - types=["T1w"], + types=["T1w", "Warp"], patterns={ "T1w": { "pattern": ( @@ -119,6 +143,28 @@ def test_MultipleDataGrabber_no_intersection() -> None: ), "space": "native", }, + "Warp": [ + { + "pattern": ( + "{subject}/{session}/anat/" + "{subject}_{session}_from-MNI152NLin2009cAsym_to-T1w_" + "xfm.h5" + ), + "src": "MNI152NLin2009cAsym", + "dst": "native", + "warper": "ants", + }, + { + "pattern": ( + "{subject}/{session}/anat/" + "{subject}_{session}_from-T1w_to-MNI152NLin2009cAsym_" + "xfm.h5" + ), + "src": "native", + "dst": "MNI152NLin2009cAsym", + "warper": "ants", + }, + ], }, replacements=replacements, ) -- 2.52.0 From 5059a26498777254ad723466fcefb30341717612 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 19 Nov 2024 17:15:43 +0100 Subject: [PATCH 4/7] refactor: allow list like data type value to be checked in PatternDataGrabber.get_elements() --- junifer/datagrabber/pattern.py | 105 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/junifer/datagrabber/pattern.py b/junifer/datagrabber/pattern.py index e57c8f156..7e1d64245 100644 --- a/junifer/datagrabber/pattern.py +++ b/junifer/datagrabber/pattern.py @@ -478,58 +478,63 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin): t_type = self.types[t_idx] types_element = set() - # Get the pattern dict - t_pattern = self.patterns[t_type] - # Conditional fetch of base pattern for getting elements - pattern = None - # Try for data type pattern - pattern = t_pattern.get("pattern") - # Try for nested data type pattern - if pattern is None and self.partial_pattern_ok: - for v in t_pattern.values(): - if isinstance(v, dict) and "pattern" in v: - pattern = v["pattern"] - break + # Data type dictionary + patterns = self.patterns[t_type] + # Conditional for list dtype vals like Warp + if not isinstance(patterns, list): + patterns = [patterns] + for t_pattern in patterns: + # Conditional fetch of base pattern for getting elements + pattern = None + # Try for data type pattern + pattern = t_pattern.get("pattern") + # Try for nested data type pattern + if pattern is None and self.partial_pattern_ok: + for v in t_pattern.values(): + if isinstance(v, dict) and "pattern" in v: + pattern = v["pattern"] + break - # Replace the pattern - ( - re_pattern, - glob_pattern, - t_replacements, - ) = self._replace_patterns_regex(pattern) - for fname in self.datadir.glob(glob_pattern): - suffix = fname.relative_to(self.datadir).as_posix() - m = re.match(re_pattern, suffix) - if m is not None: - # Find the groups of replacements present in the pattern - # If one replacement is not present, set it to None. - # We will take care of this in the intersection - t_element = tuple([m.group(k) for k in t_replacements]) - if len(self.replacements) == 1: - t_element = t_element[0] - types_element.add(t_element) - # TODO: does this make sense as elements is always None - if elements is None: - elements = types_element - else: - # Do the intersection by filtering out elements in which - # the replacements are not None - if t_replacements == self.replacements: - elements.intersection(types_element) + # Replace the pattern + ( + re_pattern, + glob_pattern, + t_replacements, + ) = self._replace_patterns_regex(pattern) + for fname in self.datadir.glob(glob_pattern): + suffix = fname.relative_to(self.datadir).as_posix() + m = re.match(re_pattern, suffix) + if m is not None: + # Find the groups of replacements present in the + # pattern. If one replacement is not present, set it + # to None. We will take care of this in the + # intersection. + t_element = tuple([m.group(k) for k in t_replacements]) + if len(self.replacements) == 1: + t_element = t_element[0] + types_element.add(t_element) + # TODO: does this make sense as elements is always None + if elements is None: + elements = types_element else: - t_repl_idx = [ - i - for i, v in enumerate(self.replacements) - if v in t_replacements - ] - new_elements = set() - for t_element in elements: - if ( - tuple(np.array(t_element)[t_repl_idx]) - in types_element - ): - new_elements.add(t_element) - elements = new_elements + # Do the intersection by filtering out elements in which + # the replacements are not None + if t_replacements == self.replacements: + elements.intersection(types_element) + else: + t_repl_idx = [ + i + for i, v in enumerate(self.replacements) + if v in t_replacements + ] + new_elements = set() + for t_element in elements: + if ( + tuple(np.array(t_element)[t_repl_idx]) + in types_element + ): + new_elements.add(t_element) + elements = new_elements if elements is None: elements = set() return list(elements) -- 2.52.0 From 01a8efbd7e8d4390ddbf4e7d30e7d847268ccbd2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 19 Nov 2024 17:16:22 +0100 Subject: [PATCH 5/7] style: introduce and use new DataGrabberPatterns type --- junifer/datagrabber/pattern.py | 3 ++- junifer/datagrabber/pattern_validation_mixin.py | 6 +++--- junifer/typing/__init__.pyi | 2 ++ junifer/typing/_typing.py | 4 ++++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/junifer/datagrabber/pattern.py b/junifer/datagrabber/pattern.py index 7e1d64245..dafbe4893 100644 --- a/junifer/datagrabber/pattern.py +++ b/junifer/datagrabber/pattern.py @@ -13,6 +13,7 @@ from typing import Optional, Union import numpy as np from ..api.decorators import register_datagrabber +from ..typing import DataGrabberPatterns from ..utils import logger, raise_error from .base import BaseDataGrabber from .pattern_validation_mixin import PatternValidationMixin @@ -171,7 +172,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin): def __init__( self, types: list[str], - patterns: dict[str, dict[str, str]], + patterns: DataGrabberPatterns, replacements: Union[list[str], str], datadir: Union[str, Path], confounds_format: Optional[str] = None, diff --git a/junifer/datagrabber/pattern_validation_mixin.py b/junifer/datagrabber/pattern_validation_mixin.py index c808cb134..6acf8fab1 100644 --- a/junifer/datagrabber/pattern_validation_mixin.py +++ b/junifer/datagrabber/pattern_validation_mixin.py @@ -3,8 +3,8 @@ # Authors: Synchon Mandal # License: AGPL -from typing import Union +from ..typing import DataGrabberPatterns from ..utils import logger, raise_error, warn_with_log @@ -96,7 +96,7 @@ class PatternValidationMixin: def _validate_replacements( self, replacements: list[str], - patterns: dict[str, Union[dict[str, str], list[dict[str, str]]]], + patterns: DataGrabberPatterns, partial_pattern_ok: bool, ) -> None: """Validate the replacements. @@ -263,7 +263,7 @@ class PatternValidationMixin: self, types: list[str], replacements: list[str], - patterns: dict[str, Union[dict[str, str], list[dict[str, str]]]], + patterns: DataGrabberPatterns, partial_pattern_ok: bool = False, ) -> None: """Validate the patterns. diff --git a/junifer/typing/__init__.pyi b/junifer/typing/__init__.pyi index 8a25d5d73..43ab84c02 100644 --- a/junifer/typing/__init__.pyi +++ b/junifer/typing/__init__.pyi @@ -8,6 +8,7 @@ __all__ = [ "ConditionalDependencies", "ExternalDependencies", "MarkerInOutMappings", + "DataGrabberPatterns", ] from ._typing import ( @@ -20,4 +21,5 @@ from ._typing import ( ConditionalDependencies, ExternalDependencies, MarkerInOutMappings, + DataGrabberPatterns, ) diff --git a/junifer/typing/_typing.py b/junifer/typing/_typing.py index f4173a6fe..b7be28415 100644 --- a/junifer/typing/_typing.py +++ b/junifer/typing/_typing.py @@ -28,6 +28,7 @@ __all__ = [ "ConditionalDependencies", "ExternalDependencies", "MarkerInOutMappings", + "DataGrabberPatterns", ] @@ -56,3 +57,6 @@ ConditionalDependencies = Sequence[ ] ExternalDependencies = Sequence[MutableMapping[str, Union[str, Sequence[str]]]] MarkerInOutMappings = MutableMapping[str, MutableMapping[str, str]] +DataGrabberPatterns = dict[ + str, Union[dict[str, str], Sequence[dict[str, str]]] +] -- 2.52.0 From cd0b0278407f65134309d6e5835e5f9930f6880f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 19 Nov 2024 17:24:03 +0100 Subject: [PATCH 6/7] chore: add changelog 398.bugfix --- docs/changes/newsfragments/398.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/398.bugfix diff --git a/docs/changes/newsfragments/398.bugfix b/docs/changes/newsfragments/398.bugfix new file mode 100644 index 000000000..abacb5389 --- /dev/null +++ b/docs/changes/newsfragments/398.bugfix @@ -0,0 +1 @@ +Fix metadata update for :class:`.MultipleDataGrabber` and adjust :meth:`.PatternDataGrabber.get_elements` to check ``list``-like data type values by `Fede Raimondo`_ and `Synchon Mandal`_ -- 2.52.0 From 92bbd72c2d960e490071884fb177ea0a328196e8 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Nov 2024 10:50:39 +0100 Subject: [PATCH 7/7] chore: fix tests for mock datagrabbers on gin --- junifer/datagrabber/tests/test_datalad_base.py | 4 ++-- junifer/datagrabber/tests/test_pattern_datalad.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/datagrabber/tests/test_datalad_base.py b/junifer/datagrabber/tests/test_datalad_base.py index ac36afc1d..02a94758f 100644 --- a/junifer/datagrabber/tests/test_datalad_base.py +++ b/junifer/datagrabber/tests/test_datalad_base.py @@ -14,8 +14,8 @@ from junifer.datagrabber import DataladDataGrabber _testing_dataset = { "example_bids": { "uri": "https://gin.g-node.org/juaml/datalad-example-bids", - "commit": "b87897cbe51bf0ee5514becaa5c7dd76491db5ad", - "id": "8fddff30-6993-420a-9d1e-b5b028c59468", + "commit": "3f288c8725207ae0c9b3616e093e78cda192b570", + "id": "582b9696-f13f-42e4-9587-b4e62aa2a8e7", }, "example_bids_ses": { "uri": "https://gin.g-node.org/juaml/datalad-example-bids-ses", diff --git a/junifer/datagrabber/tests/test_pattern_datalad.py b/junifer/datagrabber/tests/test_pattern_datalad.py index b27668142..c4529b1d8 100644 --- a/junifer/datagrabber/tests/test_pattern_datalad.py +++ b/junifer/datagrabber/tests/test_pattern_datalad.py @@ -15,7 +15,7 @@ from junifer.datagrabber import PatternDataladDataGrabber _testing_dataset = { "example_bids": { "uri": "https://gin.g-node.org/juaml/datalad-example-bids", - "commit": "b87897cbe51bf0ee5514becaa5c7dd76491db5ad", + "commit": "3f288c8725207ae0c9b3616e093e78cda192b570", "id": "8fddff30-6993-420a-9d1e-b5b028c59468", }, "example_bids_ses": { -- 2.52.0