From c101ea61f5fadc5ca4b92994d134555b2aee314d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:00:58 +0200 Subject: [PATCH 01/11] feature: introduce new onthefly sub-module and read_transform() from it --- junifer/__init__.py | 1 + junifer/onthefly/__init__.py | 6 ++ junifer/onthefly/read_transform.py | 134 +++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 junifer/onthefly/__init__.py create mode 100644 junifer/onthefly/read_transform.py diff --git a/junifer/__init__.py b/junifer/__init__.py index 9ae266782..500fde796 100644 --- a/junifer/__init__.py +++ b/junifer/__init__.py @@ -17,5 +17,6 @@ from . import ( storage, utils, external, + onthefly, ) from ._version import __version__ diff --git a/junifer/onthefly/__init__.py b/junifer/onthefly/__init__.py new file mode 100644 index 000000000..bfaff947f --- /dev/null +++ b/junifer/onthefly/__init__.py @@ -0,0 +1,6 @@ +"""Provide imports for onthefly sub-package.""" + +# Authors: Synchon Mandal +# License: AGPL + +from .read_transform import read_transform diff --git a/junifer/onthefly/read_transform.py b/junifer/onthefly/read_transform.py new file mode 100644 index 000000000..d8f80d8fd --- /dev/null +++ b/junifer/onthefly/read_transform.py @@ -0,0 +1,134 @@ +"""Provide implementation for read-and-transform function.""" + +# Authors: Synchon Mandal +# License: AGPL + + +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Type + +import pandas as pd + +from ..utils import logger, raise_error, warn_with_log + + +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + +def read_transform( + storage: Type["BaseFeatureStorage"], + feature_name: str, + transform: str, + transform_args: Optional[Tuple] = None, + transform_kw_args: Optional[Dict] = None, +) -> pd.DataFrame: + """Read stored feature and transform to specific statistical output. + + Parameters + ---------- + storage : storage-like + The storage class, for example, SQLiteFeatureStorage. + feature_name : str + Name of the feature to read. + transform : str + The kind of transform formatted as ``_``, + for example, ``bctpy_degrees_und``. + transform_args : tuple, optional + The positional arguments for the callable of ``transform`` + (default None). + transform_kw_args : dict, optional + The keyword arguments for the callable of ``transform`` + (default None). + + Returns + ------- + pandas.DataFrame + The transformed feature as a dataframe. + + Notes + ----- + This function has been only tested for: + + * bct.degrees_und + * bct.strengths_und + * bct.clustering_coef_wu + * bct.eigenvector_centrality_und + + Using other functions may fail and require tweaking. + + """ + # Set default values for args and kwargs + transform_args = transform_args or () + transform_kw_args = transform_kw_args or {} + + # Read storage + stored_data = storage.read(feature_name=feature_name) # type: ignore + # Retrieve package and function + package, func_str = transform.split("_", 1) + # Condition for package + if package == "bctpy": + # Check that "matrix" is the feature data kind + if stored_data["kind"] != "matrix": + raise_error( + msg=( + f"'{stored_data['kind']}' is not valid data kind for " + f"'{package}'" + ), + klass=RuntimeError, + ) + + # Check bctpy import + try: + import bct + except ImportError as err: + raise_error(msg=str(err), klass=ImportError) + + # Warning about function usage + if func_str not in [ + "degrees_und", + "strengths_und", + "clustering_coef_wu", + "eigenvector_centrality_und", + ]: + warn_with_log( + f"You are about to use '{package}.{func_str}' which has not " + "been tested to run. In case it fails, you will need to tweak" + " the code yourself." + ) + + # Retrieve callable object + try: + func = getattr(bct, func_str) + except AttributeError as err: + raise_error(msg=str(err), klass=AttributeError) + + # Apply function and store subject-wise + output_list = [] + logger.debug( + f"Computing '{package}.{func_str}' for feature {feature_name} ..." + ) + for subject in range(stored_data["data"].shape[2]): + output = func( + stored_data["data"][:, :, subject], + *transform_args, + **transform_kw_args, + ) + output_list.append(output) + + # Create dataframe for index + idx_df = pd.DataFrame(data=stored_data["element"]) + # Create multiindex from dataframe + logger.debug( + f"Generating pandas.MultiIndex for feature {feature_name} ..." + ) + data_idx = pd.MultiIndex.from_frame(df=idx_df) + + # Create dataframe + df = pd.DataFrame( + data=output_list, + index=data_idx, + columns=stored_data["row_headers"], + ) + return df + else: + raise_error(f"Unknown package: {package}") -- 2.52.0 From b786e3485a0abe65596c0ac69380fc5077f9a033 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:01:37 +0200 Subject: [PATCH 02/11] update: add tests for junifer.onthefly.read_transform() --- junifer/onthefly/tests/test_read_transform.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 junifer/onthefly/tests/test_read_transform.py diff --git a/junifer/onthefly/tests/test_read_transform.py b/junifer/onthefly/tests/test_read_transform.py new file mode 100644 index 000000000..62a289d6a --- /dev/null +++ b/junifer/onthefly/tests/test_read_transform.py @@ -0,0 +1,176 @@ +"""Provide tests for read-and-transform.""" + +# Authors: Synchon Mandal +# License: AGPL + + +import logging +from pathlib import Path + +import numpy as np +import pytest + +from junifer.onthefly import read_transform +from junifer.storage.hdf5 import HDF5FeatureStorage + + +@pytest.fixture +def vector_storage(tmp_path: Path) -> HDF5FeatureStorage: + """Return a HDF5FeatureStorage with vector data. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + storage = HDF5FeatureStorage(tmp_path / "vector_store.hdf5") + storage.store( + kind="vector", + meta={ + "element": {"subject": "test"}, + "dependencies": [], + "marker": {"name": "vector"}, + "type": "BOLD", + }, + data=np.arange(2).reshape(1, 2), + col_names=["f1", "f2"], + ) + return storage + + +@pytest.fixture +def matrix_storage(tmp_path: Path) -> HDF5FeatureStorage: + """Return a HDF5FeatureStorage with matrix data. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + storage = HDF5FeatureStorage(tmp_path / "matrix_store.hdf5") + storage.store( + kind="matrix", + meta={ + "element": {"subject": "test"}, + "dependencies": [], + "marker": {"name": "matrix"}, + "type": "BOLD", + }, + data=np.arange(4).reshape(2, 2), + col_names=["f1", "f2"], + row_names=["g1", "g2"], + ) + return storage + + +def test_incorrect_package(matrix_storage: HDF5FeatureStorage) -> None: + """Test error check for incorrect package name. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + + """ + with pytest.raises(ValueError, match="Unknown package"): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform="godspeed_makemake", + ) + + +def test_incorrect_data_kind(vector_storage: HDF5FeatureStorage) -> None: + """Test error check for incorrect data kind. + + Parameters + ---------- + vector_storage : HDF5FeatureStorage + The HDF5FeatureStorage with vector data, as fixture. + + """ + with pytest.raises(RuntimeError, match="not valid data kind"): + read_transform( + storage=vector_storage, # type: ignore + feature_name="BOLD_vector", + transform="bctpy_gonggong", + ) + + +def test_untested_bctpy_function(matrix_storage: HDF5FeatureStorage) -> None: + """Test warning check for untested function of bctpy. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + + """ + # Skip test if import fails + pytest.importorskip("bct") + + with pytest.raises(ValueError): + with pytest.warns(RuntimeWarning, match="You are about to use"): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform="bctpy_distance_bin", + ) + + +def test_incorrect_bctpy_function(matrix_storage: HDF5FeatureStorage) -> None: + """Test error check for incorrect function of bctpy. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + + """ + # Skip test if import fails + pytest.importorskip("bct") + + with pytest.raises(AttributeError, match="has no attribute"): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform="bctpy_haumea", + ) + + +@pytest.mark.parametrize( + "func", + ( + "degrees_und", + "strengths_und", + "clustering_coef_wu", + "eigenvector_centrality_und", + ), +) +def test_bctpy_function( + matrix_storage: HDF5FeatureStorage, + caplog: pytest.LogCaptureFixture, + func: str, +) -> None: + """Test working function of bctpy. + + Parameters + ---------- + matrix_storage : HDF5FeatureStorage + The HDF5FeatureStorage with matrix data, as fixture. + caplog : pytest.LogCaptureFixture + The pytest.LogCaptureFixture object. + func : str + The function to test. + + """ + with caplog.at_level(logging.DEBUG): + read_transform( + storage=matrix_storage, # type: ignore + feature_name="BOLD_matrix", + transform=f"bctpy_{func}", + ) + assert "Computing" in caplog.text + assert "Generating" in caplog.text -- 2.52.0 From 727d1759e239311599cefab08987f0d4b3e1fe8d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:02:21 +0200 Subject: [PATCH 03/11] update: add new optional dependency block `transform` in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 214ca524a..f5a773b2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ docs = [ "towncrier==22.12.0", "sphinxcontrib-mermaid==0.8.1", ] +transform = ["bctpy==0.6.0"] ################ # Tool configs # -- 2.52.0 From c31248e2e7172b87c48bf6ce791e13a4ca961105 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:03:07 +0200 Subject: [PATCH 04/11] chore: add `onthefly` sub-module in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4808f0659..ef9283814 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The documentation is available at [https://juaml.github.io/junifer](https://juam * `datagrabber`: DataGrabber module. * `datareader`: DataReader module. * `markers`: Markers module. + * `onthefly`: Transformation components (on-the-fly) module. * `pipeline`: Pipeline module. * `preprocess`: Preprocessing module. * `storage`: Storage module. -- 2.52.0 From 656082e99b648828a5d65e10b0406f4e5ab948d5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:03:29 +0200 Subject: [PATCH 05/11] chore: add missing `external` sub-module in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ef9283814..58326dd66 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ The documentation is available at [https://juaml.github.io/junifer](https://juam * `data`: Module that handles data required for the library to work (e.g. parcels, coordinates). * `datagrabber`: DataGrabber module. * `datareader`: DataReader module. + * `external`: Module for external libraries and tools. * `markers`: Markers module. * `onthefly`: Transformation components (on-the-fly) module. * `pipeline`: Pipeline module. -- 2.52.0 From d1e645f2eb3332a0a7e41162107d6860fa0d3654 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:21:33 +0200 Subject: [PATCH 06/11] chore: improve docstring for junifer.onthefly.read_transform() --- junifer/onthefly/read_transform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/junifer/onthefly/read_transform.py b/junifer/onthefly/read_transform.py index d8f80d8fd..c59c70a0b 100644 --- a/junifer/onthefly/read_transform.py +++ b/junifer/onthefly/read_transform.py @@ -49,10 +49,10 @@ def read_transform( ----- This function has been only tested for: - * bct.degrees_und - * bct.strengths_und - * bct.clustering_coef_wu - * bct.eigenvector_centrality_und + * ``bct.degrees_und`` + * ``bct.strengths_und`` + * ``bct.clustering_coef_wu`` + * ``bct.eigenvector_centrality_und`` Using other functions may fail and require tweaking. -- 2.52.0 From e401c2ac152800677c161108b09261dd98cca531 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:21:55 +0200 Subject: [PATCH 07/11] docs: add `onthefly` sub-module --- docs/api/index.rst | 1 + docs/api/onthefly.rst | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 docs/api/onthefly.rst diff --git a/docs/api/index.rst b/docs/api/index.rst index 482837a30..f5812db0d 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -27,6 +27,7 @@ Utilities api utils testing + onthefly Configs ------- diff --git a/docs/api/onthefly.rst b/docs/api/onthefly.rst new file mode 100644 index 000000000..c5599e560 --- /dev/null +++ b/docs/api/onthefly.rst @@ -0,0 +1,8 @@ +On-the-fly +========== + +This package provides functions for quick transform operations on stored data. + +.. automodule:: junifer.onthefly + :members: + :imported-members: -- 2.52.0 From 0ae48b010f67a343095d577bac78cc2a3859c8ae Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 30 Jun 2023 13:28:23 +0200 Subject: [PATCH 08/11] chore: add changelog 237.feature --- docs/changes/newsfragments/237.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/237.feature diff --git a/docs/changes/newsfragments/237.feature b/docs/changes/newsfragments/237.feature new file mode 100644 index 000000000..1a7dbd91d --- /dev/null +++ b/docs/changes/newsfragments/237.feature @@ -0,0 +1 @@ +Introduce :mod:`.onthefly` sub-module and :func:`.read_transform` for quick transform operations on stored data by `Synchon Mandal`_ -- 2.52.0 From 7c38982d254659e91120e1ade107a726367cf4dd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 3 Jul 2023 07:44:45 +0200 Subject: [PATCH 09/11] chore: add missing pytest.importorskip() in junifer.onthefly.read_transform() unit test --- junifer/onthefly/tests/test_read_transform.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/junifer/onthefly/tests/test_read_transform.py b/junifer/onthefly/tests/test_read_transform.py index 62a289d6a..0eec389f5 100644 --- a/junifer/onthefly/tests/test_read_transform.py +++ b/junifer/onthefly/tests/test_read_transform.py @@ -166,6 +166,9 @@ def test_bctpy_function( The function to test. """ + # Skip test if import fails + pytest.importorskip("bct") + with caplog.at_level(logging.DEBUG): read_transform( storage=matrix_storage, # type: ignore -- 2.52.0 From dc6e6320f8532edb8def508f4cf9bada03d74012 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 3 Jul 2023 10:43:23 +0200 Subject: [PATCH 10/11] chore: add bctpy to deps of coverage env in tox.ini --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d8999e004..f636cf57c 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ commands = [testenv:coverage] skip_install = false deps = + bctpy==0.6.0 pytest pytest-cov commands = @@ -85,4 +86,4 @@ exclude_lines = if TYPE_CHECKING: # Don't complain if non-runnable code isn't run: if __name__ == .__main__.: -precision = 2 \ No newline at end of file +precision = 2 -- 2.52.0 From 0e4782a97c300fe8b915b796a6c8d37f102ce474 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 3 Jul 2023 11:57:05 +0200 Subject: [PATCH 11/11] update: rename optional dependency block `transform` to `onthefly` in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f5a773b2a..9f5e156df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ docs = [ "towncrier==22.12.0", "sphinxcontrib-mermaid==0.8.1", ] -transform = ["bctpy==0.6.0"] +onthefly = ["bctpy==0.6.0"] ################ # Tool configs # -- 2.52.0