diff --git a/docs/builtin.rst b/docs/builtin.rst index 574b645f3..7cf1117fa 100644 --- a/docs/builtin.rst +++ b/docs/builtin.rst @@ -437,7 +437,7 @@ Available | (2008). | https://doi.org/10.1196/annals.1440.011 * - Missing formal name - - ``extDMN`` + - ``eMDN`` - 0.0.1 - Missing publication details * - Empathic processing @@ -457,7 +457,7 @@ Available | Brain structure & function, Volume 220, Pages 1031–1049 (2015). | https://doi.org/10.1007/s00429-013-0698-0 * - Extended multiple-demand network - - ``eMDN`` + - ``extMDN`` - 0.0.1 - | Camilleri, J.A., Müller, V.I., Fox, P. et al. | Definition and characterization of an extended multiple-demand network. diff --git a/docs/changes/newsfragments/182.enh b/docs/changes/newsfragments/182.enh new file mode 100644 index 000000000..615c16ade --- /dev/null +++ b/docs/changes/newsfragments/182.enh @@ -0,0 +1 @@ +Support element(s) to be specified via text file for ``--element`` option of ``junifer run`` by `Synchon Mandal`_ diff --git a/docs/changes/newsfragments/251.doc b/docs/changes/newsfragments/251.doc deleted file mode 100644 index 4ee20dc01..000000000 --- a/docs/changes/newsfragments/251.doc +++ /dev/null @@ -1 +0,0 @@ -Rename ``extMDN`` to ``extDMN`` (extended default mode network) and fix listing for ``eMDN`` (extended multiple-demand network) by `Synchon Mandal`_ diff --git a/docs/using/running.rst b/docs/using/running.rst index 168f58f75..118e01a3a 100644 --- a/docs/using/running.rst +++ b/docs/using/running.rst @@ -29,20 +29,68 @@ The ``run`` command accepts the following additional arguments: * ``--element``: The *element* to run. If not specified, all elements will be run. This parameter can be specified multiple times to run multiple elements. If the *element* requires several parameters, they can be specified by - separating them with ``,``. + separating them with ``,``. It also accepts a file (e.g., ``elements.txt``) + containing complete or partial element(s). Example of running two elements: +-------------------------------- .. code-block:: bash junifer run config.yaml --element sub-01 --element sub-02 +You can also specify the elements via a text file like so: + +.. code-block:: bash + + junifer run config.yaml --element elements.txt + +And the corresponding ``elements.txt`` would be like so: + +.. code-block:: text + + sub-01 + sub-02 + Example of elements with multiple parameters and verbose output: +---------------------------------------------------------------- .. code-block:: bash junifer run --verbose info config.yaml --element sub-01,ses-01 +You can also specify the elements via a text file like so: + +.. code-block:: bash + + junifer run --verbose info config.yaml --element elements.txt + +And the corresponding ``elements.txt`` would be like so: + +.. code-block:: text + + sub-01,ses-01 + +In case you wanted to run for all possible sessions (e.g., ``ses-01``, +``ses-02``, ``ses-03``) but only for ``sub-01``, you could also do: + +.. code-block:: bash + + junifer run --verbose info config.yaml --element sub-01 + +or, + +.. code-block:: bash + + junifer run --verbose info config.yaml --element elements.txt + +and then the ``elements.txt`` would be like so: + +.. code-block:: text + + sub-01 + + .. _collect: Collecting Results diff --git a/junifer/api/cli.py b/junifer/api/cli.py index 62426da2b..efa4707ac 100644 --- a/junifer/api/cli.py +++ b/junifer/api/cli.py @@ -8,9 +8,10 @@ import pathlib import subprocess import sys from pathlib import Path -from typing import Dict, List, Union +from typing import Dict, List, Tuple, Union import click +import pandas as pd from ..utils.logging import ( configure_logging, @@ -32,27 +33,45 @@ from .utils import ( ) -def _parse_elements(element: str, config: Dict) -> Union[List, None]: +def _parse_elements(element: Tuple[str], config: Dict) -> Union[List, None]: """Parse elements from cli. Parameters ---------- - element : str - The element to operate on. + element : tuple of str + The element(s) to operate on. config : dict The configuration to operate using. Returns ------- - list - The element(s) as list. + list or None + The element(s) as list or None. + + Raises + ------ + ValueError + If no element is found either in the command-line options or + the configuration file. + + Warns + ----- + RuntimeWarning + If elements are specified both via the command-line options and + the configuration file. """ logger.debug(f"Parsing elements: {element}") + # Early return None to continue with all elements if len(element) == 0: return None - # TODO: If len == 1, check if its a file, then parse elements from file - elements = [tuple(x.split(",")) if "," in x else x for x in element] + # Check if the element is a file for single element; + # if yes, then parse elements from it + if len(element) == 1 and Path(element[0]).resolve().is_file(): + elements = _parse_elements_file(Path(element[0]).resolve()) + else: + # Process multi-keyed elements + elements = [tuple(x.split(",")) if "," in x else x for x in element] logger.debug(f"Parsed elements: {elements}") if elements is not None and "elements" in config: warn_with_log( @@ -61,19 +80,49 @@ def _parse_elements(element: str, config: Dict) -> Union[List, None]: "over the configuration file. That is, the elements specified " "in the command line will be used. The elements specified in " "the configuration file will be ignored. To remove this warning, " - 'please remove the "elements" item from the configuration file.' + "please remove the `elements` item from the configuration file." ) elif elements is None: + # Check in config elements = config.get("elements", None) if elements is None: raise_error( - "The 'elements' key is set in the configuration, but its value" - " is 'None'. It is likely that there is an empty 'elements' " + "The `elements` key is set in the configuration, but its value" + " is `None`. It is likely that there is an empty `elements` " "section in the yaml configuration file." ) return elements +def _parse_elements_file(filepath: Path) -> List[Tuple[str, ...]]: + """Parse elements from file. + + Parameters + ---------- + filepath : pathlib.Path + The path to the element file. + + Returns + ------- + list of tuple of str + The element(s) as list. + + """ + # Read CSV into dataframe + csv_df = pd.read_csv( + filepath, + header=None, # no header # type: ignore + index_col=False, # no index column + skipinitialspace=True, # no leading space after delimiter + ) + # Remove trailing whitespace in cell entries + csv_df_trimmed = csv_df.apply( + lambda x: x.str.strip() if x.dtype == "object" else x + ) + # Convert to list of tuple of str + return list(map(tuple, csv_df_trimmed.to_numpy().astype(str))) + + def _validate_verbose( ctx: click.Context, param: str, value: str ) -> Union[str, int]: @@ -133,7 +182,9 @@ def cli() -> None: # pragma: no cover callback=_validate_verbose, default="info", ) -def run(filepath: click.Path, element: str, verbose: Union[str, int]) -> None: +def run( + filepath: click.Path, element: Tuple[str], verbose: Union[str, int] +) -> None: """Run command for CLI. \f @@ -142,8 +193,8 @@ def run(filepath: click.Path, element: str, verbose: Union[str, int]) -> None: ---------- filepath : click.Path The filepath to the configuration file. - element : str - The element to operate using. + element : tuple of str + The element to operate on. verbose : click.Choice The verbosity level: warning, info or debug (default "info"). diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 37e94ad2d..727d4581a 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -163,7 +163,9 @@ def run( # Fit elements with datagrabber_object: if elements is not None: - for t_element in elements: + for t_element in datagrabber_object.filter( + elements # type: ignore + ): mc.fit(datagrabber_object[t_element]) else: for t_element in datagrabber_object: diff --git a/junifer/api/tests/test_cli.py b/junifer/api/tests/test_cli.py index aa01285e4..7d94ca35f 100644 --- a/junifer/api/tests/test_cli.py +++ b/junifer/api/tests/test_cli.py @@ -5,13 +5,13 @@ # License: AGPL from pathlib import Path -from typing import Tuple +from typing import List, Tuple import pytest from click.testing import CliRunner from ruamel.yaml import YAML -from junifer.api.cli import collect, run, selftest, wtf +from junifer.api.cli import _parse_elements_file, collect, run, selftest, wtf # Configure YAML class @@ -35,7 +35,16 @@ runner = CliRunner() def test_run_and_collect_commands( tmp_path: Path, elements: Tuple[str, ...] ) -> None: - """Test run and collect commands.""" + """Test run and collect commands. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + elements : tuple of str + The parametrized elements to operate on. + + """ # Get test config infile = Path(__file__).parent / "data" / "gmd_mean.yaml" # Read test config @@ -74,6 +83,104 @@ def test_run_and_collect_commands( assert collect_result.exit_code == 0 +@pytest.mark.parametrize( + "elements", + [ + "sub-01", + "sub-01\nsub-02", + " sub-01 ", + "sub-01\n sub-02", + ], +) +def test_run_using_element_file(tmp_path: Path, elements: str) -> None: + """Test run command using element file. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + elements : str + The parametrized elements to write to the element file. + + """ + # Create test file + test_file_path = tmp_path / "elements.txt" + with open(test_file_path, "w") as f: + f.write(elements) + + # Get test config + infile = Path(__file__).parent / "data" / "gmd_mean.yaml" + # Read test config + contents = yaml.load(infile) + # Working directory + workdir = tmp_path / "workdir" + contents["workdir"] = str(workdir.resolve()) + # Output directory + outdir = tmp_path / "outdir" + # Storage + contents["storage"]["uri"] = str(outdir.resolve()) + # Write new test config + outfile = tmp_path / "in.yaml" + yaml.dump(contents, stream=outfile) + # Run command arguments + run_args = [ + str(outfile.absolute()), + "--verbose", + "debug", + "--element", + str(test_file_path.resolve()), + ] + # Invoke run command + run_result = runner.invoke(run, run_args) + # Check + assert run_result.exit_code == 0 + + +@pytest.mark.parametrize( + "elements, expected_list", + [ + ("sub-01,ses-01", [("sub-01", "ses-01")]), + ( + "sub-01,ses-01\nsub-02,ses-01", + [("sub-01", "ses-01"), ("sub-02", "ses-01")], + ), + ("sub-01, ses-01", [("sub-01", "ses-01")]), + ( + "sub-01, ses-01\nsub-02, ses-01", + [("sub-01", "ses-01"), ("sub-02", "ses-01")], + ), + (" sub-01 , ses-01 ", [("sub-01", "ses-01")]), + ( + " sub-01 , ses-01 \n sub-02, ses-01 ", + [("sub-01", "ses-01"), ("sub-02", "ses-01")], + ), + ], +) +def test_multi_element_access( + tmp_path: Path, elements: str, expected_list: List[Tuple[str, ...]] +) -> None: + """Test mulit-element parsing. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + elements : str + The parametrized elements to write to the element file. + expected_list : list of tuple of str + The parametrized list of element tuples to expect. + + """ + # Create test file + test_file_path = tmp_path / "elements_multi.txt" + with open(test_file_path, "w") as f: + f.write(elements) + # Load element file + read_elements = _parse_elements_file(test_file_path) + # Check + assert read_elements == expected_list + + def test_wtf_short() -> None: """Test short version of wtf command.""" # Invoke wtf command diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 83f230b82..008843615 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -120,7 +120,7 @@ def test_run_single_element_with_preprocessing(tmp_path: Path) -> None: preprocessor={ "kind": "fMRIPrepConfoundRemover", }, - elements=["sub-001"], + elements=["sub-01"], ) # Check files files = list(outdir.glob("*.sqlite")) diff --git a/junifer/datagrabber/base.py b/junifer/datagrabber/base.py index 37e122033..a34db6bf2 100644 --- a/junifer/datagrabber/base.py +++ b/junifer/datagrabber/base.py @@ -58,12 +58,12 @@ class BaseDataGrabber(ABC, UpdateMetaMixin): """ yield from self.get_elements() - def __getitem__(self, element: Union[str, Tuple]) -> Dict[str, Dict]: + def __getitem__(self, element: Union[str, Tuple[str]]) -> Dict[str, Dict]: """Enable indexing support. Parameters ---------- - element : str or tuple + element : str or tuple of str The element to be indexed. Returns @@ -117,6 +117,51 @@ class BaseDataGrabber(ABC, UpdateMetaMixin): """ return self._datadir + def filter(self, selection: List[Union[str, Tuple[str]]]) -> Iterator: + """Filter elements to be grabbed. + + Parameters + ---------- + selection : list of str or tuple + The list of partial element key values to filter using. + + Yields + ------ + object + An element that can be indexed by the DataGrabber. + + """ + + def filter_func(element: Union[str, Tuple[str]]) -> bool: + """Filter element based on selection. + + Parameters + ---------- + element : str or tuple of str + The element to be filtered. + + Returns + ------- + bool + If the element passes the filter or not. + + """ + # Convert element to tuple + if not isinstance(element, tuple): + element = (element,) + # Filter based on selection kind + if isinstance(selection[0], str): + for opt in selection: + if opt in element: + return True + elif isinstance(selection[0], tuple): + for opt in selection: + if set(opt).issubset(element): + return True + return False + + yield from filter(filter_func, self.get_elements()) + @abstractmethod def get_element_keys(self) -> List[str]: """Get element keys. @@ -136,7 +181,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin): ) @abstractmethod - def get_elements(self) -> List: + def get_elements(self) -> List[Union[str, Tuple[str]]]: """Get elements. Returns diff --git a/junifer/datagrabber/tests/test_base.py b/junifer/datagrabber/tests/test_base.py index 676903303..dd8b75ac7 100644 --- a/junifer/datagrabber/tests/test_base.py +++ b/junifer/datagrabber/tests/test_base.py @@ -67,3 +67,60 @@ def test_BaseDataGrabber() -> None: with pytest.raises(NotImplementedError): dg.get_item(subject=1) # type: ignore + + +def test_BaseDataGrabber_filter_single() -> None: + """Test single-keyed element filter for BaseDataGrabber.""" + + # Create concrete class + class FilterDataGrabber(BaseDataGrabber): + def get_item(self, subject): + return {"BOLD": {}} + + def get_elements(self): + return ["sub01", "sub02", "sub03"] + + def get_element_keys(self): + return ["subject"] + + dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"]) + with dg: + assert "sub01" in list(dg.filter(["sub01"])) + assert "sub02" not in list(dg.filter(["sub01"])) + + +def test_BaseDataGrabber_filter_multi() -> None: + """Test multi-keyed element filter for BaseDataGrabber.""" + + # Create concrete class + class FilterDataGrabber(BaseDataGrabber): + def get_item(self, subject): + return {"BOLD": {}} + + def get_elements(self): + return [ + ("sub01", "rest"), + ("sub01", "movie"), + ("sub02", "rest"), + ("sub02", "movie"), + ("sub03", "rest"), + ("sub03", "movie"), + ] + + def get_element_keys(self): + return ["subject", "task"] + + dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"]) + with dg: + assert ("sub01", "rest") in list( + dg.filter([("sub01", "rest")]) # type: ignore + ) + assert ("sub01", "movie") not in list( + dg.filter([("sub01", "rest")]) # type: ignore + ) + assert ("sub02", "rest") not in list( + dg.filter([("sub01", "rest")]) # type: ignore + ) + assert ("sub02", "movie") not in list( + dg.filter([("sub01", "rest")]) # type: ignore + )