From a47210435839a1eb0033e8f1bd8da8a8613338cd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 21 Sep 2023 09:08:17 +0200 Subject: [PATCH 1/6] update: support multiple preprocessors in the pipeline --- junifer/api/cli.py | 5 +++-- junifer/api/functions.py | 22 +++++++++++++--------- junifer/markers/collection.py | 33 ++++++++++++++++++++++----------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/junifer/api/cli.py b/junifer/api/cli.py index 84f516774..2000c75cf 100644 --- a/junifer/api/cli.py +++ b/junifer/api/cli.py @@ -206,7 +206,8 @@ def run( datagrabber = config["datagrabber"] markers = config["markers"] storage = config["storage"] - preprocessor = config.get("preprocess") + # Fetch preprocessors + preprocessors = config.get("preprocess") elements = _parse_elements(element, config) # Perform operation api_run( @@ -214,7 +215,7 @@ def run( datagrabber=datagrabber, markers=markers, storage=storage, - preprocessor=preprocessor, + preprocessors=preprocessors, elements=elements, ) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 774f29c82..b54ed1733 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -81,7 +81,7 @@ def run( datagrabber: Dict, markers: List[Dict], storage: Dict, - preprocessor: Optional[Dict] = None, + preprocessors: Optional[List[Dict]] = None, elements: Union[str, List[Union[str, Tuple]], Tuple, None] = None, ) -> None: """Run the pipeline on the selected element. @@ -104,10 +104,10 @@ def run( Storage to use. Must have a key ``kind`` with the kind of storage to use. All other keys are passed to the storage init function. - preprocessor : dict, optional - Preprocessor to use. Must have a key ``kind`` with the kind of - preprocessor to use. All other keys are passed to the preprocessor - init function (default None). + preprocessors : list of dict, optional + List of preprocessors to use. Each preprocessor is a dict with at + least a key ``kind`` specifying the preprocessor to use. All other keys + are passed to the preprocessor init function (default None). elements : str or tuple or list of str or tuple, optional Element(s) to process. Will be used to index the DataGrabber (default None). @@ -152,15 +152,19 @@ def run( storage_object = typing.cast(BaseFeatureStorage, storage_object) # Get preprocessor to use (if provided) - if preprocessor is not None: - preprocessor_object = _get_preprocessor(preprocessor) + if preprocessors is not None: + _preprocessors = [x.copy() for x in preprocessors] + built_preprocessors = [] + for preprocessor in _preprocessors: + preprocessor_object = _get_preprocessor(preprocessor) + built_preprocessors.append(preprocessor_object) else: - preprocessor_object = None + built_preprocessors = None # Create new marker collection mc = MarkerCollection( markers=built_markers, - preprocessing=preprocessor_object, + preprocessors=built_preprocessors, storage=storage_object, ) # Fit elements diff --git a/junifer/markers/collection.py b/junifer/markers/collection.py index cce42855a..bf40c8bb2 100644 --- a/junifer/markers/collection.py +++ b/junifer/markers/collection.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional from ..datareader.default import DefaultDataReader from ..markers.base import BaseMarker from ..pipeline import PipelineStepMixin +from ..preprocess.base import BasePreprocessor from ..storage.base import BaseFeatureStorage from ..utils import logger @@ -27,8 +28,8 @@ class MarkerCollection: The markers to compute. datareader : DataReader-like object, optional The DataReader to use (default None). - preprocessing : preprocessing-like, optional - The preprocessing steps to apply. + preprocessors : list of preprocessing-like, optional + The preprocessors to apply (default None). storage : storage-like, optional The storage to use (default None). @@ -38,7 +39,7 @@ class MarkerCollection: self, markers: List[BaseMarker], datareader: Optional[PipelineStepMixin] = None, - preprocessing: Optional[PipelineStepMixin] = None, + preprocessors: Optional[List[BasePreprocessor]] = None, storage: Optional[BaseFeatureStorage] = None, ): # Check that the markers have different names @@ -53,7 +54,7 @@ class MarkerCollection: if datareader is None: datareader = DefaultDataReader() self._datareader = datareader - self._preprocessing = preprocessing + self._preprocessors = preprocessors self._storage = storage def fit(self, input: Dict[str, Dict]) -> Optional[Dict]: @@ -79,9 +80,14 @@ class MarkerCollection: data = self._datareader.fit_transform(input) # Apply preprocessing steps - if self._preprocessing is not None: - logger.info("Preprocessing data") - data = self._preprocessing.fit_transform(data) + if self._preprocessors is not None: + for preprocessor in self._preprocessors: + logger.info( + "Preprocessing data with " + f"{preprocessor.__class__.__name__}" + ) + # Mutate data after every iteration + data = preprocessor.fit_transform(data) # Compute markers out = {} @@ -116,10 +122,15 @@ class MarkerCollection: t_data = self._datareader.validate(t_data) logger.info(f"Data Reader output type: {t_data}") - if self._preprocessing is not None: - logger.info("Validating Preprocessor:") - t_data = self._preprocessing.validate(t_data) - logger.info(f"Preprocess output type: {t_data}") + if self._preprocessors is not None: + for preprocessor in self._preprocessors: + logger.info( + "Validating Preprocessor: " + f"{preprocessor.__class__.__name__}" + ) + # Validate preprocessor + t_data = preprocessor.validate(t_data) + logger.info(f"Preprocess output type: {t_data}") for marker in self._markers: logger.info(f"Validating Marker: {marker.name}") -- 2.52.0 From 495a0f7cedd4bea087dbd1a983399412df1a95f7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 21 Sep 2023 09:10:39 +0200 Subject: [PATCH 2/6] chore: improve commentary in api.cli --- junifer/api/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/junifer/api/cli.py b/junifer/api/cli.py index 2000c75cf..7ff2f15e3 100644 --- a/junifer/api/cli.py +++ b/junifer/api/cli.py @@ -200,14 +200,20 @@ def run( """ configure_logging(level=verbose) - # TODO: add validation + # TODO(synchon): add validation + # Parse YAML config = parse_yaml(filepath) # type: ignore + # Retrieve working directory workdir = config["workdir"] + # Fetch datagrabber datagrabber = config["datagrabber"] + # Fetch markers markers = config["markers"] + # Fetch storage storage = config["storage"] # Fetch preprocessors preprocessors = config.get("preprocess") + # Parse elements elements = _parse_elements(element, config) # Perform operation api_run( -- 2.52.0 From d39daada1fee1422e847468a854f2f607f282372 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 21 Sep 2023 10:48:52 +0200 Subject: [PATCH 3/6] update: allow single or multiple preprocessor to be specified in YAML --- junifer/api/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/junifer/api/cli.py b/junifer/api/cli.py index 7ff2f15e3..873970eca 100644 --- a/junifer/api/cli.py +++ b/junifer/api/cli.py @@ -213,6 +213,9 @@ def run( storage = config["storage"] # Fetch preprocessors preprocessors = config.get("preprocess") + # Convert to list if single preprocessor + if preprocessors is not None and not isinstance(preprocessors, list): + preprocessors = [preprocessors] # Parse elements elements = _parse_elements(element, config) # Perform operation -- 2.52.0 From e4817dc3b019528fc897019a3c83d91f38ed89c2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 17 Oct 2023 15:15:35 +0200 Subject: [PATCH 4/6] chore: add changelog 263.change --- docs/changes/newsfragments/263.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/263.change diff --git a/docs/changes/newsfragments/263.change b/docs/changes/newsfragments/263.change new file mode 100644 index 000000000..9b585d90b --- /dev/null +++ b/docs/changes/newsfragments/263.change @@ -0,0 +1 @@ +Modify ``preprocessor`` to ``preprocessors`` in :func:`.run` and ``preprocessing`` to ``preprocessors`` in :class:`.MarkerCollection` to accept multiple preprocessors by `Synchon Mandal`_ -- 2.52.0 From 3f08aa1784c4ce5646af3177e8fa373915b1a17a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 18 Oct 2023 12:47:05 +0200 Subject: [PATCH 5/6] fix: adapt tests to multiple preprocessor support --- junifer/api/tests/test_functions.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 008843615..af87a1223 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -117,9 +117,11 @@ def test_run_single_element_with_preprocessing(tmp_path: Path) -> None: } ], storage=storage, - preprocessor={ - "kind": "fMRIPrepConfoundRemover", - }, + preprocessors=[ + { + "kind": "fMRIPrepConfoundRemover", + } + ], elements=["sub-01"], ) # Check files -- 2.52.0 From f80d532cf6331d6220abe45b518980e1761916b8 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 18 Oct 2023 13:20:40 +0200 Subject: [PATCH 6/6] fix: further test fixes in test_collection.py --- junifer/markers/tests/test_collection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/junifer/markers/tests/test_collection.py b/junifer/markers/tests/test_collection.py index e1dc07a76..5f978ee70 100644 --- a/junifer/markers/tests/test_collection.py +++ b/junifer/markers/tests/test_collection.py @@ -64,7 +64,7 @@ def test_marker_collection() -> None: ] mc = MarkerCollection(markers=markers) # type: ignore assert mc._markers == markers - assert mc._preprocessing is None + assert mc._preprocessors is None assert mc._storage is None assert isinstance(mc._datareader, DefaultDataReader) @@ -97,7 +97,7 @@ def test_marker_collection() -> None: mc2 = MarkerCollection( markers=markers, # type: ignore - preprocessing=BypassPreprocessing(), + preprocessors=[BypassPreprocessing()], # type: ignore datareader=DefaultDataReader(), ) assert isinstance(mc2._datareader, DefaultDataReader) @@ -128,10 +128,10 @@ def test_marker_collection_with_preprocessing() -> None: ] mc = MarkerCollection( markers=markers, # type: ignore - preprocessing=fMRIPrepConfoundRemover(), + preprocessors=[fMRIPrepConfoundRemover()], ) assert mc._markers == markers - assert mc._preprocessing is not None + assert mc._preprocessors is not None assert mc._storage is None assert isinstance(mc._datareader, DefaultDataReader) -- 2.52.0