README.md

@@ -30,7 +30,9 @@ The documentation is available at [https://juaml.github.io/junifer](https://juam
   * `data`: Module that handles data required for the library to work (e.g. parcels, coordinates).
   * `datagrabber`: DataGrabber module.
   * `datareader`: DataReader module.
+  * `external`: Module for external libraries and tools.
   * `markers`: Markers module.
+  * `onthefly`: Transformation components (on-the-fly) module.
   * `pipeline`: Pipeline module.
   * `preprocess`: Preprocessing module.
   * `storage`: Storage module.

docs/api/index.rst

@@ -27,6 +27,7 @@ Utilities
    api
    utils
    testing
+   onthefly
 
 Configs
 -------

docs/api/onthefly.rst

@@ -0,0 +1,8 @@
+On-the-fly
+==========
+
+This package provides functions for quick transform operations on stored data.
+
+.. automodule:: junifer.onthefly
+   :members:
+   :imported-members:

docs/changes/newsfragments/237.feature

@@ -0,0 +1 @@
+Introduce :mod:`.onthefly` sub-module and :func:`.read_transform` for quick transform operations on stored data by `Synchon Mandal`_

junifer/__init__.py

@@ -17,5 +17,6 @@ from . import (
     storage,
     utils,
     external,
+    onthefly,
 )
 from ._version import __version__

junifer/onthefly/__init__.py

@@ -0,0 +1,6 @@
+"""Provide imports for onthefly sub-package."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from .read_transform import read_transform

junifer/onthefly/read_transform.py

@@ -0,0 +1,134 @@
+"""Provide implementation for read-and-transform function."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+
+from typing import TYPE_CHECKING, Dict, Optional, Tuple, Type
+
+import pandas as pd
+
+from ..utils import logger, raise_error, warn_with_log
+
+
+if TYPE_CHECKING:
+    from junifer.storage import BaseFeatureStorage
+
+
+def read_transform(
+    storage: Type["BaseFeatureStorage"],
+    feature_name: str,
+    transform: str,
+    transform_args: Optional[Tuple] = None,
+    transform_kw_args: Optional[Dict] = None,
+) -> pd.DataFrame:
+    """Read stored feature and transform to specific statistical output.
+
+    Parameters
+    ----------
+    storage : storage-like
+        The storage class, for example, SQLiteFeatureStorage.
+    feature_name : str
+        Name of the feature to read.
+    transform : str
+        The kind of transform formatted as ``<package>_<function>``,
+        for example, ``bctpy_degrees_und``.
+    transform_args : tuple, optional
+        The positional arguments for the callable of ``transform``
+        (default None).
+    transform_kw_args : dict, optional
+        The keyword arguments for the callable of ``transform``
+        (default None).
+
+    Returns
+    -------
+    pandas.DataFrame
+        The transformed feature as a dataframe.
+
+    Notes
+    -----
+    This function has been only tested for:
+
+    * ``bct.degrees_und``
+    * ``bct.strengths_und``
+    * ``bct.clustering_coef_wu``
+    * ``bct.eigenvector_centrality_und``
+
+    Using other functions may fail and require tweaking.
+
+    """
+    # Set default values for args and kwargs
+    transform_args = transform_args or ()
+    transform_kw_args = transform_kw_args or {}
+
+    # Read storage
+    stored_data = storage.read(feature_name=feature_name)  # type: ignore
+    # Retrieve package and function
+    package, func_str = transform.split("_", 1)
+    # Condition for package
+    if package == "bctpy":
+        # Check that "matrix" is the feature data kind
+        if stored_data["kind"] != "matrix":
+            raise_error(
+                msg=(
+                    f"'{stored_data['kind']}' is not valid data kind for "
+                    f"'{package}'"
+                ),
+                klass=RuntimeError,
+            )
+
+        # Check bctpy import
+        try:
+            import bct
+        except ImportError as err:
+            raise_error(msg=str(err), klass=ImportError)
+
+        # Warning about function usage
+        if func_str not in [
+            "degrees_und",
+            "strengths_und",
+            "clustering_coef_wu",
+            "eigenvector_centrality_und",
+        ]:
+            warn_with_log(
+                f"You are about to use '{package}.{func_str}' which has not "
+                "been tested to run. In case it fails, you will need to tweak"
+                " the code yourself."
+            )
+
+        # Retrieve callable object
+        try:
+            func = getattr(bct, func_str)
+        except AttributeError as err:
+            raise_error(msg=str(err), klass=AttributeError)
+
+        # Apply function and store subject-wise
+        output_list = []
+        logger.debug(
+            f"Computing '{package}.{func_str}' for feature {feature_name} ..."
+        )
+        for subject in range(stored_data["data"].shape[2]):
+            output = func(
+                stored_data["data"][:, :, subject],
+                *transform_args,
+                **transform_kw_args,
+            )
+            output_list.append(output)
+
+        # Create dataframe for index
+        idx_df = pd.DataFrame(data=stored_data["element"])
+        # Create multiindex from dataframe
+        logger.debug(
+            f"Generating pandas.MultiIndex for feature {feature_name} ..."
+        )
+        data_idx = pd.MultiIndex.from_frame(df=idx_df)
+
+        # Create dataframe
+        df = pd.DataFrame(
+            data=output_list,
+            index=data_idx,
+            columns=stored_data["row_headers"],
+        )
+        return df
+    else:
+        raise_error(f"Unknown package: {package}")

junifer/onthefly/tests/test_read_transform.py

@@ -0,0 +1,179 @@
+"""Provide tests for read-and-transform."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+
+import logging
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from junifer.onthefly import read_transform
+from junifer.storage.hdf5 import HDF5FeatureStorage
+
+
+@pytest.fixture
+def vector_storage(tmp_path: Path) -> HDF5FeatureStorage:
+    """Return a HDF5FeatureStorage with vector data.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    storage = HDF5FeatureStorage(tmp_path / "vector_store.hdf5")
+    storage.store(
+        kind="vector",
+        meta={
+            "element": {"subject": "test"},
+            "dependencies": [],
+            "marker": {"name": "vector"},
+            "type": "BOLD",
+        },
+        data=np.arange(2).reshape(1, 2),
+        col_names=["f1", "f2"],
+    )
+    return storage
+
+
+@pytest.fixture
+def matrix_storage(tmp_path: Path) -> HDF5FeatureStorage:
+    """Return a HDF5FeatureStorage with matrix data.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    storage = HDF5FeatureStorage(tmp_path / "matrix_store.hdf5")
+    storage.store(
+        kind="matrix",
+        meta={
+            "element": {"subject": "test"},
+            "dependencies": [],
+            "marker": {"name": "matrix"},
+            "type": "BOLD",
+        },
+        data=np.arange(4).reshape(2, 2),
+        col_names=["f1", "f2"],
+        row_names=["g1", "g2"],
+    )
+    return storage
+
+
+def test_incorrect_package(matrix_storage: HDF5FeatureStorage) -> None:
+    """Test error check for incorrect package name.
+
+    Parameters
+    ----------
+    matrix_storage : HDF5FeatureStorage
+        The HDF5FeatureStorage with matrix data, as fixture.
+
+    """
+    with pytest.raises(ValueError, match="Unknown package"):
+        read_transform(
+            storage=matrix_storage,  # type: ignore
+            feature_name="BOLD_matrix",
+            transform="godspeed_makemake",
+        )
+
+
+def test_incorrect_data_kind(vector_storage: HDF5FeatureStorage) -> None:
+    """Test error check for incorrect data kind.
+
+    Parameters
+    ----------
+    vector_storage : HDF5FeatureStorage
+        The HDF5FeatureStorage with vector data, as fixture.
+
+    """
+    with pytest.raises(RuntimeError, match="not valid data kind"):
+        read_transform(
+            storage=vector_storage,  # type: ignore
+            feature_name="BOLD_vector",
+            transform="bctpy_gonggong",
+        )
+
+
+def test_untested_bctpy_function(matrix_storage: HDF5FeatureStorage) -> None:
+    """Test warning check for untested function of bctpy.
+
+    Parameters
+    ----------
+    matrix_storage : HDF5FeatureStorage
+        The HDF5FeatureStorage with matrix data, as fixture.
+
+    """
+    # Skip test if import fails
+    pytest.importorskip("bct")
+
+    with pytest.raises(ValueError):
+        with pytest.warns(RuntimeWarning, match="You are about to use"):
+            read_transform(
+                storage=matrix_storage,  # type: ignore
+                feature_name="BOLD_matrix",
+                transform="bctpy_distance_bin",
+            )
+
+
+def test_incorrect_bctpy_function(matrix_storage: HDF5FeatureStorage) -> None:
+    """Test error check for incorrect function of bctpy.
+
+    Parameters
+    ----------
+    matrix_storage : HDF5FeatureStorage
+        The HDF5FeatureStorage with matrix data, as fixture.
+
+    """
+    # Skip test if import fails
+    pytest.importorskip("bct")
+
+    with pytest.raises(AttributeError, match="has no attribute"):
+        read_transform(
+            storage=matrix_storage,  # type: ignore
+            feature_name="BOLD_matrix",
+            transform="bctpy_haumea",
+        )
+
+
+@pytest.mark.parametrize(
+    "func",
+    (
+        "degrees_und",
+        "strengths_und",
+        "clustering_coef_wu",
+        "eigenvector_centrality_und",
+    ),
+)
+def test_bctpy_function(
+    matrix_storage: HDF5FeatureStorage,
+    caplog: pytest.LogCaptureFixture,
+    func: str,
+) -> None:
+    """Test working function of bctpy.
+
+    Parameters
+    ----------
+    matrix_storage : HDF5FeatureStorage
+        The HDF5FeatureStorage with matrix data, as fixture.
+    caplog : pytest.LogCaptureFixture
+        The pytest.LogCaptureFixture object.
+    func : str
+        The function to test.
+
+    """
+    # Skip test if import fails
+    pytest.importorskip("bct")
+
+    with caplog.at_level(logging.DEBUG):
+        read_transform(
+            storage=matrix_storage,  # type: ignore
+            feature_name="BOLD_matrix",
+            transform=f"bctpy_{func}",
+        )
+        assert "Computing" in caplog.text
+        assert "Generating" in caplog.text

pyproject.toml

@@ -73,6 +73,7 @@ docs = [
     "towncrier==22.12.0",
     "sphinxcontrib-mermaid==0.8.1",
 ]
+onthefly = ["bctpy==0.6.0"]
 
 ################
 # Tool configs #

tox.ini

@@ -45,6 +45,7 @@ commands =
 [testenv:coverage]
 skip_install = false
 deps =
+    bctpy==0.6.0
     pytest
     pytest-cov
 commands =