refactor: consistent use of DataReader #227

Merged
synchon merged 7 commits from chore/dr-cleanup into main 2023-06-20 10:28:05 +00:00
8 changed files with 29 additions and 23 deletions

View file

@ -0,0 +1 @@
Adopt ``DataReader`` consistently throughout codebase to match with the documentation by `Synchon Mandal`_

View file

@ -39,7 +39,7 @@ step only contains information about the datagrabber used.
The :ref:`Data Reader <datareader>` step adds the ``data`` second-level key The :ref:`Data Reader <datareader>` step adds the ``data`` second-level key
which is the actual data loaded into memory. The ``meta`` key in this step which is the actual data loaded into memory. The ``meta`` key in this step
adds information about the datareader used to read the data. adds information about the DataReader used to read the data.
.. code-block:: python .. code-block:: python

View file

@ -14,7 +14,7 @@ files in junifer. It reads the value of the key ``path`` for each
them into memory. After reading the data into memory, it adds the key ``data`` them into memory. After reading the data into memory, it adds the key ``data``
to the same level as ``path`` and the value is the actual data in the memory. to the same level as ``path`` and the value is the actual data in the memory.
Datareaders are meant to be used inside the datagrabber context but you can DataReaders are meant to be used inside the datagrabber context but you can
operate on them outside the context as long as the actual data is in the memory operate on them outside the context as long as the actual data is in the memory
and the Python runtime has not garbage-collected it. and the Python runtime has not garbage-collected it.

View file

@ -107,12 +107,12 @@ Data Reader
^^^^^^^^^^^ ^^^^^^^^^^^
As mentioned before, this section is entirely optional, as junifer only provides As mentioned before, this section is entirely optional, as junifer only provides
one data reader (:class:`.DefaultDataReader`), which is the default in case the one DataReader (:class:`.DefaultDataReader`), which is the default in case the
section is not specified. section is not specified.
In any case, the syntax of the section is the same as for the ``datagrabber`` In any case, the syntax of the section is the same as for the ``datagrabber``
section, using the ``kind`` key to specify the datareader to use, and additional section, using the ``kind`` key to specify the DataReader to use, and additional
keys to pass parameters to the datareader: keys to pass parameters to the DataReader constructor:
.. code-block:: yaml .. code-block:: yaml

View file

@ -35,20 +35,24 @@ def register_datagrabber(klass: Type) -> Type:
def register_datareader(klass: Type) -> Type: def register_datareader(klass: Type) -> Type:
"""Datareader registration decorator. """Register DataReader.
Registers the datareader so it can be used by name. Registers the DataReader so it can be used by name.
Parameters Parameters
---------- ----------
klass: class klass: class
The class of the datareader to register. The class of the DataReader to register.
Returns Returns
------- -------
klass: class klass: class
The unmodified input class. The unmodified input class.
Notes
-----
It should only be used as a decorator.
""" """
register( register(
step="datareader", step="datareader",

View file

@ -1,4 +1,4 @@
"""Provide class for default data reader.""" """Provide concrete implementation for default DataReader."""
# Authors: Federico Raimondo <f.raimondo@fz-juelich.de> # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
# Synchon Mandal <s.mandal@fz-juelich.de> # Synchon Mandal <s.mandal@fz-juelich.de>
@ -32,7 +32,7 @@ _readers["TSV"] = {"func": pd.read_csv, "params": {"sep": "\t"}}
@register_datareader @register_datareader
class DefaultDataReader(PipelineStepMixin, UpdateMetaMixin): class DefaultDataReader(PipelineStepMixin, UpdateMetaMixin):
"""Mixin class for default data reader.""" """Concrete implementation for common data reading."""
def validate_input(self, input: List[str]) -> List[str]: def validate_input(self, input: List[str]) -> List[str]:
"""Validate input. """Validate input.
@ -48,6 +48,7 @@ class DefaultDataReader(PipelineStepMixin, UpdateMetaMixin):
list of str list of str
The actual elements of the input that will be processed by this The actual elements of the input that will be processed by this
pipeline step. pipeline step.
""" """
# Nothing to validate, any input is fine # Nothing to validate, any input is fine
return input return input

View file

@ -1,4 +1,4 @@
"""Provide tests for default data reader.""" """Provide tests for DefaultDataReader."""
# Authors: Federico Raimondo <f.raimondo@fz-juelich.de> # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
# Synchon Mandal <s.mandal@fz-juelich.de> # Synchon Mandal <s.mandal@fz-juelich.de>
@ -19,8 +19,8 @@ from junifer.datareader import DefaultDataReader
@pytest.mark.parametrize( @pytest.mark.parametrize(
"type_", [["T1w", "BOLD", "T2", "dwi"], [], ["whatever"]] "type_", [["T1w", "BOLD", "T2", "dwi"], [], ["whatever"]]
) )
def test_validation(type_) -> None: def test_DefaultDataReader_validation(type_) -> None:
"""Test validating input/output. """Test DefaultDataReader validating input/output.
Parameters Parameters
---------- ----------
@ -34,8 +34,8 @@ def test_validation(type_) -> None:
assert reader.validate(type_) == type_ assert reader.validate(type_) == type_
def test_meta() -> None: def test_DefaultDataReader_meta() -> None:
"""Test reader metadata.""" """Test DefaultDataReader metadata."""
reader = DefaultDataReader() reader = DefaultDataReader()
nib_data_path = Path(nib_testing.data_path) nib_data_path = Path(nib_testing.data_path)
@ -52,8 +52,8 @@ def test_meta() -> None:
@pytest.mark.parametrize( @pytest.mark.parametrize(
"fname", ["example4d.nii.gz", "reoriented_anat_moved.nii"] "fname", ["example4d.nii.gz", "reoriented_anat_moved.nii"]
) )
def test_read_nifti(fname: str) -> None: def test_DefaultDataReader_nifti(fname: str) -> None:
"""Test reading NIFTI files. """Test DefaultDataReader reading NIfTI files.
Parameters Parameters
---------- ----------
@ -85,8 +85,8 @@ def test_read_nifti(fname: str) -> None:
assert output["BOLD"]["path"] == output2["BOLD"]["path"] assert output["BOLD"]["path"] == output2["BOLD"]["path"]
def test_read_unknown() -> None: def test_DefaultDataReader_unknown() -> None:
"""Test (not) reading unknown files.""" """Test DefaultDataReader (not) reading unknown files."""
reader = DefaultDataReader() reader = DefaultDataReader()
nib_data_path = Path(nib_testing.data_path) nib_data_path = Path(nib_testing.data_path)
@ -115,8 +115,8 @@ def test_read_unknown() -> None:
reader.fit_transform(input) reader.fit_transform(input)
def test_read_csv(tmp_path: Path) -> None: def test_DefaultDataReader_csv(tmp_path: Path) -> None:
"""Test reading CSV files. """Test DefaultDataReader reading CSV files.
Parameters Parameters
---------- ----------

View file

@ -25,8 +25,8 @@ class MarkerCollection:
---------- ----------
markers : list of marker-like markers : list of marker-like
The markers to compute. The markers to compute.
datareader : datareader-like, optional datareader : DataReader-like object, optional
The datareader to use (default None). The DataReader to use (default None).
preprocessing : preprocessing-like, optional preprocessing : preprocessing-like, optional
The preprocessing steps to apply. The preprocessing steps to apply.
storage : storage-like, optional storage : storage-like, optional