diff --git a/README.md b/README.md index 4808f0659..58326dd66 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,9 @@ The documentation is available at [https://juaml.github.io/junifer](https://juam * `data`: Module that handles data required for the library to work (e.g. parcels, coordinates). * `datagrabber`: DataGrabber module. * `datareader`: DataReader module. + * `external`: Module for external libraries and tools. * `markers`: Markers module. + * `onthefly`: Transformation components (on-the-fly) module. * `pipeline`: Pipeline module. * `preprocess`: Preprocessing module. * `storage`: Storage module. diff --git a/docs/api/index.rst b/docs/api/index.rst index 482837a30..f5812db0d 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -27,6 +27,7 @@ Utilities api utils testing + onthefly Configs ------- diff --git a/docs/api/onthefly.rst b/docs/api/onthefly.rst new file mode 100644 index 000000000..c5599e560 --- /dev/null +++ b/docs/api/onthefly.rst @@ -0,0 +1,8 @@ +On-the-fly +========== + +This package provides functions for quick transform operations on stored data. + +.. automodule:: junifer.onthefly + :members: + :imported-members: diff --git a/docs/changes/newsfragments/237.feature b/docs/changes/newsfragments/237.feature new file mode 100644 index 000000000..1a7dbd91d --- /dev/null +++ b/docs/changes/newsfragments/237.feature @@ -0,0 +1 @@ +Introduce :mod:`.onthefly` sub-module and :func:`.read_transform` for quick transform operations on stored data by `Synchon Mandal`_ diff --git a/junifer/__init__.py b/junifer/__init__.py index 9ae266782..500fde796 100644 --- a/junifer/__init__.py +++ b/junifer/__init__.py @@ -17,5 +17,6 @@ from . import ( storage, utils, external, + onthefly, ) from ._version import __version__ diff --git a/junifer/onthefly/__init__.py b/junifer/onthefly/__init__.py new file mode 100644 index 000000000..bfaff947f --- /dev/null +++ b/junifer/onthefly/__init__.py @@ -0,0 +1,6 @@ +"""Provide imports for onthefly sub-package.""" + +# Authors: Synchon Mandal +# License: AGPL + +from .read_transform import read_transform diff --git a/junifer/onthefly/read_transform.py b/junifer/onthefly/read_transform.py new file mode 100644 index 000000000..c59c70a0b --- /dev/null +++ b/junifer/onthefly/read_transform.py @@ -0,0 +1,134 @@ +"""Provide implementation for read-and-transform function.""" + +# Authors: Synchon Mandal +# License: AGPL + + +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Type + +import pandas as pd + +from ..utils import logger, raise_error, warn_with_log + + +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + +def read_transform( + storage: Type["BaseFeatureStorage"], + feature_name: str, + transform: str, + transform_args: Optional[Tuple] = None, + transform_kw_args: Optional[Dict] = None, +) -> pd.DataFrame: + """Read stored feature and transform to specific statistical output. + + Parameters + ---------- + storage : storage-like + The storage class, for example, SQLiteFeatureStorage. + feature_name : str + Name of the feature to read. + transform : str + The kind of transform formatted as ``_``, + for example, ``bctpy_degrees_und``. + transform_args : tuple, optional + The positional arguments for the callable of ``transform`` + (default None). + transform_kw_args : dict, optional + The keyword arguments for the callable of ``transform`` + (default None). + + Returns + ------- + pandas.DataFrame + The transformed feature as a dataframe. + + Notes + ----- + This function has been only tested for: + + * ``bct.degrees_und`` + * ``bct.strengths_und`` + * ``bct.clustering_coef_wu`` + * ``bct.eigenvector_centrality_und`` + + Using other functions may fail and require tweaking. + + """ + # Set default values for args and kwargs + transform_args = transform_args or () + transform_kw_args = transform_kw_args or {} + + # Read storage + stored_data = storage.read(feature_name=feature_name) # type: ignore + # Retrieve package and function + package, func_str = transform.split("_", 1) + # Condition for package + if package == "bctpy": + # Check that "matrix" is the feature data kind + if stored_data["kind"] != "matrix": + raise_error( + msg=( + f"'{stored_data['kind']}' is not valid data kind for " + f"'{package}'" + ), + klass=RuntimeError, + ) + + # Check bctpy import + try: + import bct + except ImportError as err: + raise_error(msg=str(err), klass=ImportError) + + # Warning about function usage + if func_str not in [ + "degrees_und", + "strengths_und", + "clustering_coef_wu", + "eigenvector_centrality_und", + ]: + warn_with_log( + f"You are about to use '{package}.{func_str}' which has not " + "been tested to run. In case it fails, you will need to tweak" + " the code yourself." + ) + + # Retrieve callable object + try: + func = getattr(bct, func_str) + except AttributeError as err: + raise_error(msg=str(err), klass=AttributeError) + + # Apply function and store subject-wise + output_list = [] + logger.debug( + f"Computing '{package}.{func_str}' for feature {feature_name} ..." + ) + for subject in range(stored_data["data"].shape[2]): + output = func( + stored_data["data"][:, :, subject], + *transform_args, + **transform_kw_args, + ) + output_list.append(output) + + # Create dataframe for index + idx_df = pd.DataFrame(data=stored_data["element"]) + # Create multiindex from dataframe + logger.debug( + f"Generating pandas.MultiIndex for feature {feature_name} ..." + ) + data_idx = pd.MultiIndex.from_frame(df=idx_df) + + # Create dataframe + df = pd.DataFrame( + data=output_list, + index=data_idx, + columns=stored_data["row_headers"], + ) + return df + else: + raise_error(f"Unknown package: {package}") diff --git a/junifer/onthefly/tests/test_read_transform.py b/junifer/onthefly/tests/test_read_transform.py new file mode 100644 index 000000000..0eec389f5 --- /dev/null +++ b/junifer/onthefly/tests/test_read_transform.py @@ -0,0 +1,179 @@ +"""Provide tests for read-and-transform.""" + +# Authors: Synchon Mandal +# License: AGPL + + +import logging +from pathlib import Path + +import numpy as np +import pytest + +from junifer.onthefly import read_transform +from junifer.storage.hdf5 import HDF5FeatureStorage + + +@pytest.fixture +def vector_storage(tmp_path: Path) -> HDF5FeatureStorage: + """Return a HDF5FeatureStorage with vector data. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + storage = HDF5FeatureStorage(tmp_path / "vector_store.hdf5") + storage.store( + kind="vector", + meta={ + "element": {"subject": "test"}, + "dependencies": [], + "marker": {"name": "vector"}, + "type": "BOLD", + }, + data=np.arange(2).reshape(1, 2), + col_names=["f1", "f2"], + ) + return storage + + +@pytest.fixture +def matrix_storage(tmp_path: Path) -> HDF5FeatureStorage: + """Return a HDF5FeatureStorage with matrix data. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + storage = HDF5FeatureStorage(tmp_path / "matrix_store.hdf5") + storage.store( + kind="matrix", + meta={ + "element": {"subject": "test"}, + "dependencies": [], + "marker": {"name": "matrix"}, + "type": "BOLD", + }, + data=np.arange(4).reshape(2, 2), + col_names=["f1", "f2"], + row_names=["g1", "g2"], + ) + return storage + + +def test_incorrect_package(matrix_storage: HDF5FeatureStorage) -> None: + """Test error check for incorrect package name. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + + """ + with pytest.raises(ValueError, match="Unknown package"): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform="godspeed_makemake", + ) + + +def test_incorrect_data_kind(vector_storage: HDF5FeatureStorage) -> None: + """Test error check for incorrect data kind. + + Parameters + ---------- + vector_storage : HDF5FeatureStorage + The HDF5FeatureStorage with vector data, as fixture. + + """ + with pytest.raises(RuntimeError, match="not valid data kind"): + read_transform( + storage=vector_storage, # type: ignore + feature_name="BOLD_vector", + transform="bctpy_gonggong", + ) + + +def test_untested_bctpy_function(matrix_storage: HDF5FeatureStorage) -> None: + """Test warning check for untested function of bctpy. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + + """ + # Skip test if import fails + pytest.importorskip("bct") + + with pytest.raises(ValueError): + with pytest.warns(RuntimeWarning, match="You are about to use"): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform="bctpy_distance_bin", + ) + + +def test_incorrect_bctpy_function(matrix_storage: HDF5FeatureStorage) -> None: + """Test error check for incorrect function of bctpy. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + + """ + # Skip test if import fails + pytest.importorskip("bct") + + with pytest.raises(AttributeError, match="has no attribute"): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform="bctpy_haumea", + ) + + +@pytest.mark.parametrize( + "func", + ( + "degrees_und", + "strengths_und", + "clustering_coef_wu", + "eigenvector_centrality_und", + ), +) +def test_bctpy_function( + matrix_storage: HDF5FeatureStorage, + caplog: pytest.LogCaptureFixture, + func: str, +) -> None: + """Test working function of bctpy. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + caplog : pytest.LogCaptureFixture + The pytest.LogCaptureFixture object. + func : str + The function to test. + + """ + # Skip test if import fails + pytest.importorskip("bct") + + with caplog.at_level(logging.DEBUG): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform=f"bctpy_{func}", + ) + assert "Computing" in caplog.text + assert "Generating" in caplog.text diff --git a/pyproject.toml b/pyproject.toml index 214ca524a..9f5e156df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ docs = [ "towncrier==22.12.0", "sphinxcontrib-mermaid==0.8.1", ] +onthefly = ["bctpy==0.6.0"] ################ # Tool configs # diff --git a/tox.ini b/tox.ini index d8999e004..f636cf57c 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ commands = [testenv:coverage] skip_install = false deps = + bctpy==0.6.0 pytest pytest-cov commands = @@ -85,4 +86,4 @@ exclude_lines = if TYPE_CHECKING: # Don't complain if non-runnable code isn't run: if __name__ == .__main__.: -precision = 2 \ No newline at end of file +precision = 2