juaml/junifer
Watch
3
0
Fork
You've already forked junifer
 0
Code Issues 14 Pull requests 2 Packages Activity Actions

[BUG]: Fix metadata update for MultipleDataGrabber #398

Merged
synchon merged 7 commits from fix/multiple-dg-meta-update into main 2024-11-21 13:14:10 +00:00
Conversation 3 Commits 7 Files changed 11 +129 -62
11 changed files with 129 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/changes/newsfragments/398.bugfix Normal file
View file

@ -0,0 +1 @@
Fix metadata update for :class:`.MultipleDataGrabber` and adjust :meth:`.PatternDataGrabber.get_elements` to check ``list``-like data type values by `Fede Raimondo`_ and `Synchon Mandal`_

8
junifer/datagrabber/multiple.py
View file

@ -111,8 +111,12 @@ class MultipleDataGrabber(BaseDataGrabber):
# Update all the metas again
for kind in out:
self.update_meta(out[kind], "datagrabber")
out[kind]["meta"]["datagrabber"]["datagrabbers"] = metas
to_update = out[kind]
if not isinstance(to_update, list):
to_update = [to_update]
for t_kind in to_update:
self.update_meta(t_kind, "datagrabber")
t_kind["meta"]["datagrabber"]["datagrabbers"] = metas
return out
def __enter__(self) -> "MultipleDataGrabber":

18
junifer/datagrabber/pattern.py
View file

@ -13,6 +13,7 @@ from typing import Optional, Union
import numpy as np
from ..api.decorators import register_datagrabber
from ..typing import DataGrabberPatterns
from ..utils import logger, raise_error
from .base import BaseDataGrabber
from .pattern_validation_mixin import PatternValidationMixin
@ -171,7 +172,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
def __init__(
self,
types: list[str],
patterns: dict[str, dict[str, str]],
patterns: DataGrabberPatterns,
replacements: Union[list[str], str],
datadir: Union[str, Path],
confounds_format: Optional[str] = None,
@ -478,8 +479,12 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
t_type = self.types[t_idx]
types_element = set()
# Get the pattern dict
t_pattern = self.patterns[t_type]
# Data type dictionary
patterns = self.patterns[t_type]
# Conditional for list dtype vals like Warp
if not isinstance(patterns, list):
patterns = [patterns]
for t_pattern in patterns:
# Conditional fetch of base pattern for getting elements
pattern = None
# Try for data type pattern
@ -501,9 +506,10 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
suffix = fname.relative_to(self.datadir).as_posix()
m = re.match(re_pattern, suffix)
if m is not None:
# Find the groups of replacements present in the pattern
# If one replacement is not present, set it to None.
# We will take care of this in the intersection
# Find the groups of replacements present in the
# pattern. If one replacement is not present, set it
# to None. We will take care of this in the
# intersection.
t_element = tuple([m.group(k) for k in t_replacements])
if len(self.replacements) == 1:
t_element = t_element[0]

6
junifer/datagrabber/pattern_validation_mixin.py
View file

@ -3,8 +3,8 @@
# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
# License: AGPL
from typing import Union
from ..typing import DataGrabberPatterns
from ..utils import logger, raise_error, warn_with_log
@ -96,7 +96,7 @@ class PatternValidationMixin:
def _validate_replacements(
self,
replacements: list[str],
patterns: dict[str, Union[dict[str, str], list[dict[str, str]]]],
patterns: DataGrabberPatterns,
partial_pattern_ok: bool,
) -> None:
"""Validate the replacements.
@ -263,7 +263,7 @@ class PatternValidationMixin:
self,
types: list[str],
replacements: list[str],
patterns: dict[str, Union[dict[str, str], list[dict[str, str]]]],
patterns: DataGrabberPatterns,
partial_pattern_ok: bool = False,
) -> None:
"""Validate the patterns.

4
junifer/datagrabber/tests/test_datalad_base.py
View file

@ -14,8 +14,8 @@ from junifer.datagrabber import DataladDataGrabber
_testing_dataset = {
"example_bids": {
"uri": "https://gin.g-node.org/juaml/datalad-example-bids",
"commit": "b87897cbe51bf0ee5514becaa5c7dd76491db5ad",
"id": "8fddff30-6993-420a-9d1e-b5b028c59468",
"commit": "3f288c8725207ae0c9b3616e093e78cda192b570",
"id": "582b9696-f13f-42e4-9587-b4e62aa2a8e7",
},
"example_bids_ses": {
"uri": "https://gin.g-node.org/juaml/datalad-example-bids-ses",

50
junifer/datagrabber/tests/test_multiple.py
View file

@ -29,7 +29,7 @@ def test_MultipleDataGrabber() -> None:
dg1 = PatternDataladDataGrabber(
rootdir=rootdir,
uri=repo_uri,
types=["T1w"],
types=["T1w", "Warp"],
patterns={
"T1w": {
"pattern": (
@ -44,6 +44,28 @@ def test_MultipleDataGrabber() -> None:
"space": "native",
},
},
"Warp": [
{
"pattern": (
"{subject}/{session}/anat/"
"{subject}_{session}_from-MNI152NLin2009cAsym_to-T1w_"
"xfm.h5"
),
"src": "MNI152NLin2009cAsym",
"dst": "native",
"warper": "ants",
},
{
"pattern": (
"{subject}/{session}/anat/"
"{subject}_{session}_from-T1w_to-MNI152NLin2009cAsym_"
"xfm.h5"
),
"src": "native",
"dst": "MNI152NLin2009cAsym",
"warper": "ants",
},
],
},
replacements=replacements,
)
@ -75,6 +97,7 @@ def test_MultipleDataGrabber() -> None:
types = dg.get_types()
assert "T1w" in types
assert "Warp" in types
assert "BOLD" in types
expected_subs = [
@ -90,6 +113,7 @@ def test_MultipleDataGrabber() -> None:
elem = dg[("sub-01", "ses-01")]
# Check data types
assert "T1w" in elem
assert "Warp" in elem
assert "BOLD" in elem
# Check meta
assert "meta" in elem["BOLD"]
@ -111,7 +135,7 @@ def test_MultipleDataGrabber_no_intersection() -> None:
dg1 = PatternDataladDataGrabber(
rootdir=rootdir,
uri=_testing_dataset["example_bids"]["uri"],
types=["T1w"],
types=["T1w", "Warp"],
patterns={
"T1w": {
"pattern": (
@ -119,6 +143,28 @@ def test_MultipleDataGrabber_no_intersection() -> None:
),
"space": "native",
},
"Warp": [
{
"pattern": (
"{subject}/{session}/anat/"
"{subject}_{session}_from-MNI152NLin2009cAsym_to-T1w_"
"xfm.h5"
),
"src": "MNI152NLin2009cAsym",
"dst": "native",
"warper": "ants",
},
{
"pattern": (
"{subject}/{session}/anat/"
"{subject}_{session}_from-T1w_to-MNI152NLin2009cAsym_"
"xfm.h5"
),
"src": "native",
"dst": "MNI152NLin2009cAsym",
"warper": "ants",
},
],
},
replacements=replacements,
)

2
junifer/datagrabber/tests/test_pattern_datalad.py
View file

@ -15,7 +15,7 @@ from junifer.datagrabber import PatternDataladDataGrabber
_testing_dataset = {
"example_bids": {
"uri": "https://gin.g-node.org/juaml/datalad-example-bids",
"commit": "b87897cbe51bf0ee5514becaa5c7dd76491db5ad",
"commit": "3f288c8725207ae0c9b3616e093e78cda192b570",
"id": "8fddff30-6993-420a-9d1e-b5b028c59468",
},
"example_bids_ses": {

2
junifer/typing/__init__.pyi
View file

@ -8,6 +8,7 @@ __all__ = [
"ConditionalDependencies",
"ExternalDependencies",
"MarkerInOutMappings",
"DataGrabberPatterns",
]
from ._typing import (
@ -20,4 +21,5 @@ from ._typing import (
ConditionalDependencies,
ExternalDependencies,
MarkerInOutMappings,
DataGrabberPatterns,
)

4
junifer/typing/_typing.py
View file

@ -28,6 +28,7 @@ __all__ = [
"ConditionalDependencies",
"ExternalDependencies",
"MarkerInOutMappings",
"DataGrabberPatterns",
]
@ -56,3 +57,6 @@ ConditionalDependencies = Sequence[
]
ExternalDependencies = Sequence[MutableMapping[str, Union[str, Sequence[str]]]]
MarkerInOutMappings = MutableMapping[str, MutableMapping[str, str]]
DataGrabberPatterns = dict[
str, Union[dict[str, str], Sequence[dict[str, str]]]
]

2
tools/create_bids_example_dataset.py
View file

@ -26,6 +26,8 @@ with TemporaryDirectory() as tmpdir_name:
fnames = [
f"anat/{t_sub}_T1w.nii.gz",
f"anat/{t_sub}_brain_mask.nii.gz",
f"anat/{t_sub}_from-MNI152NLin2009cAsym_to-T1w_xfm.h5",
f"anat/{t_sub}_from-T1w_to-MNI152NLin2009cAsym_xfm.h5",
f"func/{t_sub}_task-rest_bold.nii.gz",
f"func/{t_sub}_task-rest_bold.json",
f"func/{t_sub}_task-rest_brain_mask.nii.gz",

2
tools/create_bids_example_dataset_sessions.py
View file

@ -31,6 +31,8 @@ with TemporaryDirectory() as tmpdir_name:
fnames = [
f"anat/{t_sub}_{t_ses}_T1w.nii.gz",
f"anat/{t_sub}_{t_ses}_brain_mask.nii.gz",
f"anat/{t_sub}_{t_ses}_from-MNI152NLin2009cAsym_to-T1w_xfm.h5",
f"anat/{t_sub}_{t_ses}_from-T1w_to-MNI152NLin2009cAsym_xfm.h5",
]
if i_ses != 3: # Session 3 does not have functional data
fnames.extend(