From 4eb808602bcda8d65e2ebff37e907e116b3f2883 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:21:43 +0200 Subject: [PATCH 01/83] update: change validate() to validate_input() in BaseFeatureStorage() and improve docstring --- junifer/storage/base.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index f9bda0886..a13f0e199 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -51,9 +51,8 @@ class BaseFeatureStorage(ABC): } return meta - # TODO: is raising ValueError required? @abstractmethod - def validate(self, input_: List[str]) -> bool: + def validate_input(self, input_: List[str]) -> bool: """Validate the input to the pipeline step. Parameters @@ -66,11 +65,6 @@ class BaseFeatureStorage(ABC): bool Whether the `input` is valid or not. - Raises - ------ - ValueError - If the input does not have the required data. - """ raise_error( msg="Concrete classes need to implement validate_input().", -- 2.52.0 From 8ce79eac3b81c73ea54a8f5af3e88b2ae1ea4360 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:22:36 +0200 Subject: [PATCH 02/83] chore: improve docstring for store_metadata() in BaseFeatureStorage() --- junifer/storage/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index a13f0e199..119517f1b 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -134,7 +134,7 @@ class BaseFeatureStorage(ABC): Returns ------- str - The metadata column + The metadata column. """ raise_error( -- 2.52.0 From d2cbe8a096328ec0598b309898ef85bf2b0c37b8 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:23:40 +0200 Subject: [PATCH 03/83] update: rename, complete type annotation and remove @abstractmethod for store_matrix() in BaseFeatureStorage() --- junifer/storage/base.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 119517f1b..80b67aac7 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -142,22 +142,21 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - # TODO: complete type annotations - @abstractmethod - def store_matrix2d( + def store_matrix( self, - data, + data: Dict, meta: Dict, col_names: Optional[Iterable[str]] = None, row_names: Optional[Iterable[str]] = None, kind: Optional[str] = "full", diagonal: bool = True, ) -> None: - """Store 2D matrix. + """Store matrix. Parameters ---------- - data + data : dict + The matrix data to store. meta : dict The metadata as a dictionary. col_names : list or tuple of str, optional @@ -172,7 +171,7 @@ class BaseFeatureStorage(ABC): diagonal : bool, optional Whether to store the diagonal (default True). If kind == 'full', setting this to false will raise - an error + an error. """ raise_error( -- 2.52.0 From 7bf94397a0be90b5628aa6560d63ea4b54f8a252 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:24:58 +0200 Subject: [PATCH 04/83] update: complete type annotation and remove @abstractmethod for store_table() in BaseFeatureStorage() --- junifer/storage/base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 80b67aac7..b677cb9ad 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -179,11 +179,9 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - # TODO: complete type annotations - @abstractmethod def store_table( self, - data, + data: Dict, meta: Dict, columns: Optional[Iterable[str]] = None, rows_col_name: Optional[str] = None, @@ -192,7 +190,8 @@ class BaseFeatureStorage(ABC): Parameters ---------- - data + data : dict + The table data to store. meta : dict The metadata as a dictionary. columns : list or tuple of str, optional -- 2.52.0 From 83470adf32910332ce5975f928c1737b81e5ca50 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:25:38 +0200 Subject: [PATCH 05/83] update: complete type annotation and remove @abstractmethod in store_timeseries() in BaseFeatureStorage() --- junifer/storage/base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index b677cb9ad..924cc9d74 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -224,14 +224,13 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - # TODO: complete type annotations - @abstractmethod - def store_timeseries(self, data, meta: Dict) -> None: + def store_timeseries(self, data: Dict, meta: Dict) -> None: """Store timeseries. Parameters ---------- - data + data : dict + The timeseries data to store. meta : dict The metadata as a dictionary. -- 2.52.0 From b0d1ec7c7706175fabf3b9b8d0b16fe5af7912f2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:26:35 +0200 Subject: [PATCH 06/83] update: remove @abstractmethod and reorder store_df() in BaseFeatureStorage() --- junifer/storage/base.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 924cc9d74..16ef9c2f8 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -142,6 +142,22 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) + def store_df(self, df: pd.DataFrame, meta: Dict) -> None: + """Store pandas DataFerame. + + Parameters + ---------- + df : pandas.DataFrame + The DataFrame to store. + meta : dict + The metadata as a dictionary. + + """ + raise_error( + msg="Concrete classes need to implement store_df().", + klass=NotImplementedError, + ) + def store_matrix( self, data: Dict, @@ -207,23 +223,6 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - @abstractmethod - def store_df(self, df: pd.DataFrame, meta: Dict) -> None: - """Store pandas DataFerame. - - Parameters - ---------- - df : pandas.DataFrame - The DataFrame to store. - meta : dict - The metadata as a dictionary. - - """ - raise_error( - msg="Concrete classes need to implement store_df().", - klass=NotImplementedError, - ) - def store_timeseries(self, data: Dict, meta: Dict) -> None: """Store timeseries. -- 2.52.0 From f291b9eae2ec2c3cc81fcf9d0892ed740b55b55c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:27:15 +0200 Subject: [PATCH 07/83] update: add store() in BaseFeatureStorage() --- junifer/storage/base.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 16ef9c2f8..1dbeb1039 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -142,6 +142,24 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) + def store(self, kind: str, data: Dict) -> None: + """Store extracted features data. + + Parameters + ---------- + kind : {"matrix", "timeseries", "table"} + The storage kind. + data : dict + The data to store. + + """ + if kind == "matrix": + self.store_matrix() + elif kind == "timeseries": + self.store_timeseries() + elif kind == "table": + self.store_table() + def store_df(self, df: pd.DataFrame, meta: Dict) -> None: """Store pandas DataFerame. -- 2.52.0 From fdaf93d8e3df2aa377dff36f1251dc4f168e2fb4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:28:11 +0200 Subject: [PATCH 08/83] update: rename validate() to validate_input() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 4d0ee310d..54b2f516f 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -238,7 +238,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # Store dataframe self.store_df(df=data_df, meta=meta) - def validate(self, input_: List[str]) -> bool: + def validate_input(self, input_: List[str]) -> bool: """Implement input validation. Parameters -- 2.52.0 From b783d8e29499ebe65250b2f242966c07578e8a2c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:30:57 +0200 Subject: [PATCH 09/83] chore: improve docstring in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 54b2f516f..2cdd1095b 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -59,6 +59,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): **kwargs: str, ) -> None: """Initialize the class.""" + # Check upsert argument value if upsert not in ["update", "ignore"]: raise_error( msg=( @@ -77,7 +78,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): ) uri.parent.mkdir(parents=True, exist_ok=True) super().__init__(uri=uri, single_output=single_output, **kwargs) + # Set upsert self._upsert = upsert + # Define the available storage kinds available self._valid_inputs = ["table", "timeseries", "matrix"] def get_engine(self, meta: Optional[Dict] = None) -> "Engine": @@ -110,8 +113,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): prefix = element_to_prefix(element) # Format URI for engine creation uri = ( - "sqlite:///" f"{self.uri.parent}/{prefix}{self.uri.name}" - ) # type: ignore + f"sqlite:///{self.uri.parent}/{prefix}{self.uri.name}" + ) return create_engine(uri, echo=False) def _save_upsert( @@ -234,7 +237,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # Prepare new dataframe data_df = pd.DataFrame( data, columns=columns, index=idx - ) # type: ignore + ) # Store dataframe self.store_df(df=data_df, meta=meta) @@ -430,7 +433,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): diagonal : bool, optional Whether to store the diagonal (default True). If kind == 'full', setting this to false will raise - an error + an error. + """ if diagonal is False and kind not in ["triu", "tril"]: raise_error( -- 2.52.0 From 8055d63d8699ea9d7b067022d8cbd8ac5303dfdb Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:31:52 +0200 Subject: [PATCH 10/83] update: complete type annotation and rename store_matrix() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 2cdd1095b..6753d65f9 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -402,21 +402,21 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): self._save_upsert(meta_df, "meta", engine) return f"meta_{meta_md5}" - # TODO: complete type annotations - def store_matrix2d( + def store_matrix( self, - data, + data: Dict, meta: Dict, col_names: Optional[List[str]] = None, row_names: Optional[List[str]] = None, kind: Optional[str] = "full", diagonal: bool = True, ) -> None: - """Implement 2D matrix storing. + """Implement matrix storing. Parameters ---------- - data + data : dict + The matrix data to store. meta : dict The metadata as a dictionary. col_names : list or tuple of str, optional -- 2.52.0 From fd5eae4dfab95511c88050eb30ae2f347eb73422 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:33:32 +0200 Subject: [PATCH 11/83] update: complete type annotation for store_table() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 6753d65f9..27d09b491 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -498,10 +498,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # Store dataframe self.store_df(df=data_df, meta=meta) # type: ignore - # TODO: complete type annotations def store_table( self, - data, + data: Dict, meta: Dict, columns: Optional[Iterable[str]] = None, rows_col_name: Optional[str] = None, @@ -510,7 +509,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): Parameters ---------- - data + data : dict + The table data to store. meta : dict The metadata as a dictionary. columns : list or tuple of str, optional -- 2.52.0 From ceadd268b138839d55330c86af415450eeb96db2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:34:45 +0200 Subject: [PATCH 12/83] chore: reorder store_df() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 94 +++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 27d09b491..b37e61120 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -402,6 +402,53 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): self._save_upsert(meta_df, "meta", engine) return f"meta_{meta_md5}" + def store_df(self, df: pd.DataFrame, meta: Dict) -> None: + """Implement pandas DataFrame storing. + + Parameters + ---------- + df : pandas.DataFrame + The DataFrame to store. + meta : dict + The metadata as a dictionary. + + Raises + ------ + ValueError + If the dataframe index has items that are not in the index + generated from the metadata. + + """ + # TODO: Test this function + # Check that the index generated by meta matches the one in + # the dataframe. + idx = element_to_index(meta) + # Given the meta, we might not know if there is an extra column added + # when storing a timeseries or 2d elements. We need to check if the + # extra element is only one. + extra = [x for x in df.index.names if x not in idx.names] + if len(extra) > 1: + raise_error( + "The index of the dataframe has extra items that are not " + "in the index generated from the metadata." + ) + elif len(extra) == 1: + # The df has one extra index item, this should be the new name + # of the missing element in the index + idx = element_to_index(meta, rows_col_name=extra[0]) + + if any(x not in df.index.names for x in idx.names): + raise_error( + "The index of the dataframe is missing index items that are " + "generated from the metadata." + ) + # Get table name + table_name = self.store_metadata(meta) + # Get sqlalchemy engine + engine = self.get_engine(meta) + # Save data + self._save_upsert(df, table_name, engine) + def store_matrix( self, data: Dict, @@ -525,53 +572,6 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): data=data, meta=meta, columns=columns, rows_col_name=rows_col_name ) - def store_df(self, df: pd.DataFrame, meta: Dict) -> None: - """Implement dataframe storing. - - Parameters - ---------- - df : pandas.DataFrame - The DataFrame to store. - meta : dict - The metadata as a dictionary. - - Raises - ------ - ValueError - If the dataframe index has items that are not in the index - generated from the metadata. - - """ - # TODO: Test this function - # Check that the index generated by meta matches the one in - # the dataframe. - idx = element_to_index(meta) - # Given the meta, we might not know if there is an extra column added - # when storing a timeseries or 2d elements. We need to check if the - # extra element is only one. - extra = [x for x in df.index.names if x not in idx.names] - if len(extra) > 1: - raise_error( - "The index of the dataframe has extra items that are not " - "in the index generated from the metadata." - ) - elif len(extra) == 1: - # The df has one extra index item, this should be the new name - # of the missing element in the index - idx = element_to_index(meta, rows_col_name=extra[0]) - - if any(x not in df.index.names for x in idx.names): - raise_error( - "The index of the dataframe is missing index items that are " - "generated from the metadata." - ) - # Get table name - table_name = self.store_metadata(meta) - # Get sqlalchemy engine - engine = self.get_engine(meta) - # Save data - self._save_upsert(df, table_name, engine) - # TODO: complete type annotations def store_timeseries(self, data, meta: Dict) -> None: """Implement timeseries storing. -- 2.52.0 From f6951b3a43d71977864f117ede7d7a75e74459c9 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:35:20 +0200 Subject: [PATCH 13/83] update: complete type annotation and rename _store_2d() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index b37e61120..fb8c434ba 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -206,10 +206,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): msg=f"Invalid option {if_exists} for if_exists." ) - # TODO: complete type annotations - def store_2d( + def _store_2d( self, - data, + data: Dict, meta: Dict, columns: Optional[Iterable[str]] = None, rows_col_name: Optional[str] = None, @@ -218,7 +217,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): Parameters ---------- - data + data : dict + The data to store. meta : dict The metadata as a dictionary. columns : list or tuple of str, optional -- 2.52.0 From 6d702a114c37ae82bda8305684e87013932873d1 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:35:54 +0200 Subject: [PATCH 14/83] fix: use _store_2d() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index fb8c434ba..fc9f44739 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -568,7 +568,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): "index" (default None). """ - self.store_2d( + self._store_2d( data=data, meta=meta, columns=columns, rows_col_name=rows_col_name ) -- 2.52.0 From 31d9cd334c528bc2b0a1d94fd83804ce33d88704 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 12 Oct 2022 20:36:26 +0200 Subject: [PATCH 15/83] update: complete type annotation and implement store_timeseries() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index fc9f44739..315c19f09 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -572,20 +572,19 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): data=data, meta=meta, columns=columns, rows_col_name=rows_col_name ) - # TODO: complete type annotations - def store_timeseries(self, data, meta: Dict) -> None: + def store_timeseries(self, data: Dict, meta: Dict) -> None: """Implement timeseries storing. Parameters ---------- - data + data: dict + The timeseries data to store. meta : dict The metadata as a dictionary. """ - raise_error( - msg="store_timeseries() not implemented.", - klass=NotImplementedError, + self._store_2d( + data=data, meta=meta, columns=None, rows_col_name="timepoint", ) def collect(self) -> None: -- 2.52.0 From bf48c56d4b64d11e4afb976866f220f6c49b9c38 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 12:56:25 +0200 Subject: [PATCH 16/83] update: adjust signatures and docstrings for store_*() methods in BaseFeatureStorage() --- junifer/storage/base.py | 79 ++++++++++------------------------------- 1 file changed, 19 insertions(+), 60 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 1dbeb1039..3a71570c6 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from pathlib import Path -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import pandas as pd @@ -142,33 +142,31 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - def store(self, kind: str, data: Dict) -> None: + def store(self, kind: str, **kwargs) -> None: """Store extracted features data. Parameters ---------- kind : {"matrix", "timeseries", "table"} The storage kind. - data : dict - The data to store. + **kwargs + The keyword arguments. """ if kind == "matrix": - self.store_matrix() + self.store_matrix(**kwargs) elif kind == "timeseries": - self.store_timeseries() + self.store_timeseries(**kwargs) elif kind == "table": - self.store_table() + self.store_table(**kwargs) - def store_df(self, df: pd.DataFrame, meta: Dict) -> None: + def store_df(self, **kwargs) -> None: """Store pandas DataFerame. Parameters ---------- - df : pandas.DataFrame - The DataFrame to store. - meta : dict - The metadata as a dictionary. + **kwargs : dict + The keyword arguments. """ raise_error( @@ -176,36 +174,13 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - def store_matrix( - self, - data: Dict, - meta: Dict, - col_names: Optional[Iterable[str]] = None, - row_names: Optional[Iterable[str]] = None, - kind: Optional[str] = "full", - diagonal: bool = True, - ) -> None: + def store_matrix(self, **kwargs) -> None: """Store matrix. Parameters ---------- - data : dict - The matrix data to store. - meta : dict - The metadata as a dictionary. - col_names : list or tuple of str, optional - The column names (default None). - row_names : list of tuple of str, optional - The row names (default None). - kind : str, optional - The kind of matrix: - - 'triu': store upper triangular only. - - 'tril': store lower triangular. - - 'full': full matrix (default 'full'). - diagonal : bool, optional - Whether to store the diagonal (default True). - If kind == 'full', setting this to false will raise - an error. + **kwargs : dict + The keyword arguments. """ raise_error( @@ -213,27 +188,13 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - def store_table( - self, - data: Dict, - meta: Dict, - columns: Optional[Iterable[str]] = None, - rows_col_name: Optional[str] = None, - ) -> None: + def store_table(self, **kwargs) -> None: """Store table. Parameters ---------- - data : dict - The table data to store. - meta : dict - The metadata as a dictionary. - columns : list or tuple of str, optional - The columns (default None). - rows_col_name : str, optional - The column name to use in case number of rows greater than 1. - If None and number of rows greater than 1, then the name will be - "index" (default None). + **kwargs : dict + The keyword arguments. """ raise_error( @@ -241,15 +202,13 @@ class BaseFeatureStorage(ABC): klass=NotImplementedError, ) - def store_timeseries(self, data: Dict, meta: Dict) -> None: + def store_timeseries(self, **kwargs) -> None: """Store timeseries. Parameters ---------- - data : dict - The timeseries data to store. - meta : dict - The metadata as a dictionary. + **kwargs : dict + The keyword arguments. """ raise_error( -- 2.52.0 From 980504eccbe454c44482f6a5dd8191593436505d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 12:58:27 +0200 Subject: [PATCH 17/83] chore: complete type annotations and docstring for ParcelAggregation() constructor --- junifer/markers/parcel.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/junifer/markers/parcel.py b/junifer/markers/parcel.py index c0f5544c8..14b49667e 100644 --- a/junifer/markers/parcel.py +++ b/junifer/markers/parcel.py @@ -4,7 +4,7 @@ # Synchon Mandal # License: AGPL -from typing import Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import numpy as np from nilearn.image import math_img, resample_to_img @@ -23,16 +23,32 @@ class ParcelAggregation(BaseMarker): Parameters ---------- - atlas - method - method_params - on - name + atlas : str + The name of the atlas. Check valid options by calling + `junifer.data.list_atlases`. + method : str + The method to perform aggregation using. Check valid options in + `junifer.stats.get_aggfunc_by_name`. + method_params : dict, optional + Parameters to pass to the aggregation function. Check valid options in + `junifer.stats.get_aggfunc_by_name`. + on : {"T1w", "BOLD", "VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"} or list + of the options, optional + The kind of data to apply the marker to. If None, will work on all + available data (default None). + name : str, optional + The name of the marker. If None, will use the class name (default + None). """ def __init__( - self, atlas, method, method_params=None, on=None, name=None + self, + atlas: str, + method: str, + method_params: Optional[Dict[str, Any]] = None, + on: Union[List[str], str, None] = None, + name: Optional[str] = None, ) -> None: """Initialize the class.""" self.atlas = atlas -- 2.52.0 From 65e5545294ea642b31d27301f32a1316202b34d0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 12:59:26 +0200 Subject: [PATCH 18/83] chore: fix docstring for get_output_kind() in ParcelAggregation() --- junifer/markers/parcel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/markers/parcel.py b/junifer/markers/parcel.py index 14b49667e..04af4d17c 100644 --- a/junifer/markers/parcel.py +++ b/junifer/markers/parcel.py @@ -68,8 +68,8 @@ class ParcelAggregation(BaseMarker): Returns ------- - str - The kind of output. + list of str + The list of storage kinds. """ outputs = [] -- 2.52.0 From 844e8dc1d5313bd97b51defc2aaf2e50d715ac9b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 13:03:25 +0200 Subject: [PATCH 19/83] chore: complete type annotations and improve code style in ParcelAggregation() --- junifer/markers/parcel.py | 46 +++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/junifer/markers/parcel.py b/junifer/markers/parcel.py index 04af4d17c..4cef64f4b 100644 --- a/junifer/markers/parcel.py +++ b/junifer/markers/parcel.py @@ -17,6 +17,10 @@ from ..utils import logger from .base import BaseMarker +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + @register_marker class ParcelAggregation(BaseMarker): """Class for parcel aggregation. @@ -82,15 +86,22 @@ class ParcelAggregation(BaseMarker): raise ValueError(f"Unknown input kind for {t_input}") return outputs - # TODO: complete type annotations - def store(self, kind: str, out, storage) -> None: + def store( + self, + kind: str, + out: Dict[str, Any], + storage: "BaseFeatureStorage", + ) -> None: """Store. Parameters ---------- - kind - out - storage + kind : {"BOLD", "VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"} + The data kind to store. + out : dict + The computed result as a dictionary to store. + storage : storage-like + The storage class, for example, SQLiteFeatureStorage. """ logger.debug(f"Storing {kind} in {storage}") @@ -99,15 +110,19 @@ class ParcelAggregation(BaseMarker): if kind in ["BOLD"]: storage.store_timeseries(**out) - def compute(self, input: Dict, extra_input: Optional[Dict] = None) -> Dict: + def compute( + self, + input: Dict[str, Any], + extra_input: Optional[Dict] = None, + ) -> Dict: """Compute. Parameters ---------- - input : Dict[str, Dict] + input : dict A single input from the pipeline data object in which to compute the marker. - extra_input : Dict, optional + extra_input : dict, optional The other fields in the pipeline data object. Useful for accessing other data kind that needs to be used in the computation. For example, the functional connectivity markers can make use of the @@ -118,17 +133,24 @@ class ParcelAggregation(BaseMarker): dict The computed result as dictionary. This will be either returned to the user or stored in the storage by calling the store method - with this as a parameter. + with this as a parameter. The dictionary has the following keys: + - data : the actual computed values as a numpy.ndarray + - columns : the column labels for the computed values as a list + - row_names (if more than one row is present in data): 'scan' """ t_input = input["data"] logger.debug(f"Parcel aggregation using {self.method}") agg_func = get_aggfunc_by_name( - self.method, func_params=self.method_params + name=self.method, + func_params=self.method_params, ) # Get the min of the voxels sizes and use it as the resolution - resolution = np.min(t_input.header.get_zooms()[:3]) # type: ignore - t_atlas, t_labels, _ = load_atlas(self.atlas, resolution=resolution) + resolution = np.min(t_input.header.get_zooms()[:3]) + t_atlas, t_labels, _ = load_atlas( + name=self.atlas, + resolution=resolution, + ) atlas_img_res = resample_to_img( t_atlas, t_input, -- 2.52.0 From fdea6494392e8a69be7411bc8cdd98f37144ed74 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 13:04:09 +0200 Subject: [PATCH 20/83] update: replace store_*() methods with store() in ParcelAggregation() --- junifer/markers/parcel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/markers/parcel.py b/junifer/markers/parcel.py index 4cef64f4b..0097828a7 100644 --- a/junifer/markers/parcel.py +++ b/junifer/markers/parcel.py @@ -106,9 +106,9 @@ class ParcelAggregation(BaseMarker): """ logger.debug(f"Storing {kind} in {storage}") if kind in ["VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"]: - storage.store_table(**out) - if kind in ["BOLD"]: - storage.store_timeseries(**out) + storage.store(kind="table", **out) + elif kind in ["BOLD"]: + storage.store(kind="timeseries", **out) def compute( self, -- 2.52.0 From 146855967fd53e8b2d94cd54f99415c2781bf72e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 16:57:29 +0200 Subject: [PATCH 21/83] feature: implement timeseries storage for SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 315c19f09..606fb7f59 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -572,7 +572,13 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): data=data, meta=meta, columns=columns, rows_col_name=rows_col_name ) - def store_timeseries(self, data: Dict, meta: Dict) -> None: + def store_timeseries( + self, + data: Dict, + meta: Dict, + columns: Optional[Iterable[str]] = None, + row_names: str = "timepoint", + ) -> None: """Implement timeseries storing. Parameters @@ -581,10 +587,18 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): The timeseries data to store. meta : dict The metadata as a dictionary. + columns : list or tuple of str, optional + The column labels (default None). + row_names : str, optional + The column name to use in case number of rows greater than 1 + (default "timepoint"). """ self._store_2d( - data=data, meta=meta, columns=None, rows_col_name="timepoint", + data=data, + meta=meta, + columns=columns, + rows_col_name="timepoint", # explicit so as to stop overriding ) def collect(self) -> None: -- 2.52.0 From fba2e4b6dc8a886132e0d365d7e3e858550dfda5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 16:58:25 +0200 Subject: [PATCH 22/83] update: rename read_df() to read_features() --- junifer/storage/base.py | 2 +- junifer/storage/sqlite.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 3a71570c6..6777533ef 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -97,7 +97,7 @@ class BaseFeatureStorage(ABC): ) @abstractmethod - def read_df( + def read_features( self, feature_name: Optional[str] = None, feature_md5: Optional[bool] = None, diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 606fb7f59..b04cf0581 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -291,7 +291,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): out = meta_df.to_dict(orient="index") return out - def read_df( + def read_features( self, feature_name: Optional[str] = None, feature_md5: Optional[str] = None, -- 2.52.0 From 28c9c5e4aab4d54b5d4a1d8cdb8570d0dde85e1d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 16:58:44 +0200 Subject: [PATCH 23/83] chore: spellcheck BaseFeatureStorage() --- junifer/storage/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 6777533ef..c99a5cd2a 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -161,7 +161,7 @@ class BaseFeatureStorage(ABC): self.store_table(**kwargs) def store_df(self, **kwargs) -> None: - """Store pandas DataFerame. + """Store pandas DataFrame. Parameters ---------- -- 2.52.0 From 8ba460a0485b8ee6d7a356adbcce9b3cfe903525 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 16:59:03 +0200 Subject: [PATCH 24/83] chore: black formatting --- junifer/storage/sqlite.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index b04cf0581..94b040c09 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -112,9 +112,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): ) prefix = element_to_prefix(element) # Format URI for engine creation - uri = ( - f"sqlite:///{self.uri.parent}/{prefix}{self.uri.name}" - ) + uri = f"sqlite:///{self.uri.parent}/{prefix}{self.uri.name}" return create_engine(uri, echo=False) def _save_upsert( @@ -235,9 +233,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): meta=meta, n_rows=n_rows, rows_col_name=rows_col_name ) # Prepare new dataframe - data_df = pd.DataFrame( - data, columns=columns, index=idx - ) + data_df = pd.DataFrame(data, columns=columns, index=idx) # Store dataframe self.store_df(df=data_df, meta=meta) -- 2.52.0 From cec8f49efd864e2254016aa5f01b885f7d2eeb73 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 17:19:24 +0200 Subject: [PATCH 25/83] update: make BaseMarker() abstract base class --- junifer/markers/base.py | 43 +++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/junifer/markers/base.py b/junifer/markers/base.py index bb4095bcb..16c515c04 100644 --- a/junifer/markers/base.py +++ b/junifer/markers/base.py @@ -1,23 +1,24 @@ -"""Provide base class for markers.""" +"""Provide abstract base class for markers.""" # Authors: Federico Raimondo # Synchon Mandal # License: AGPL from typing import Dict, List, Optional, Union +from abc import ABC, abstractmethod from ..pipeline import PipelineStepMixin from ..utils import logger, raise_error -class BaseMarker(PipelineStepMixin): - """Base class for all markers. +class BaseMarker(ABC, PipelineStepMixin): + """Abstract base class for all markers. Parameters ---------- - on : list of str + on : str or list of str The kind of data to apply the marker to. By default, will work on all - available data (default None). + available data. name : str, optional The name of the marker. By default, it will use the class name as the name of the marker (default None). @@ -76,6 +77,7 @@ class BaseMarker(PipelineStepMixin): f"\t Required (any of): {self._valid_inputs}" ) + @abstractmethod def get_output_kind(self, input: List[str]) -> List[str]: """Get output kind. @@ -92,9 +94,11 @@ class BaseMarker(PipelineStepMixin): """ raise_error( - msg="get_output_kind() not implemented", klass=NotImplementedError + msg="Concrete classes need to implement get_output_kind().", + klass=NotImplementedError, ) + @abstractmethod def compute(self, input: Dict, extra_input: Optional[Dict] = None) -> Dict: """Compute. @@ -117,19 +121,34 @@ class BaseMarker(PipelineStepMixin): with this as a parameter. """ - raise_error(msg="compute() not implemented", klass=NotImplementedError) + raise_error( + msg="Concrete classes need to implement compute().", + klass=NotImplementedError, + ) - # TODO: complete type annotations - def store(self, kind: str, out: Dict, storage) -> None: + @abstractmethod + def store( + self, + kind: str, + out: Dict[str, Any], + storage: "BaseFeatureStorage", + ) -> None: """Store. Parameters ---------- - input - out + kind : str + The data kind to store. + out : dict + The computed result as a dictionary to store. + storage : storage-like + The storage class. """ - raise_error(msg="store() not implemented", klass=NotImplementedError) + raise_error( + msg="Concrete classes need to implement store().", + klass=NotImplementedError, + ) # TODO: complete type annotations def fit_transform(self, input: Dict[str, Dict], storage=None) -> Dict: -- 2.52.0 From 0c70e97b6e195089d2f9b74702505b8454790d99 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 17:19:56 +0200 Subject: [PATCH 26/83] update: complete type annotations and update docstrings for BaseMarker() --- junifer/markers/base.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/junifer/markers/base.py b/junifer/markers/base.py index 16c515c04..12227ce79 100644 --- a/junifer/markers/base.py +++ b/junifer/markers/base.py @@ -4,13 +4,17 @@ # Synchon Mandal # License: AGPL -from typing import Dict, List, Optional, Union from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from ..pipeline import PipelineStepMixin from ..utils import logger, raise_error +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + class BaseMarker(ABC, PipelineStepMixin): """Abstract base class for all markers. @@ -45,7 +49,7 @@ class BaseMarker(ABC, PipelineStepMixin): Returns ------- dict - The metadata as a dictionary. + The metadata as a dictionary with the only key 'marker'. """ s_meta = super().get_meta() @@ -150,18 +154,25 @@ class BaseMarker(ABC, PipelineStepMixin): klass=NotImplementedError, ) - # TODO: complete type annotations - def fit_transform(self, input: Dict[str, Dict], storage=None) -> Dict: + def fit_transform( + self, + input: Dict[str, Dict], + storage: "BaseFeatureStorage" = None, + ) -> Dict: """Fit and transform. Parameters ---------- - input - storage + input : dict + The Junifer Data object. + storage : storage-like, optional + The storage class, for example, SQLiteFeatureStorage. Returns ------- dict + The processed output as a dictionary. If `storage` is provided, + empty dictionary is returned. """ out = {} @@ -175,11 +186,11 @@ class BaseMarker(ABC, PipelineStepMixin): t_meta = meta.copy() t_meta.update(t_input.get("meta", {})) t_meta.update(self.get_meta(kind)) - t_out = self.compute(t_input, extra_input) + t_out = self.compute(input=t_input, extra_input=extra_input) t_out.update(meta=t_meta) if storage is not None: logger.info(f"Storing in {storage}") - self.store(kind, t_out, storage) + self.store(kind=kind, out=t_out, storage=storage) else: logger.info("No storage specified, returning dictionary") out[kind] = t_out -- 2.52.0 From 13deb5292d9b19b23497d88e7a7cd9bf590fd0f0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 19:32:33 +0200 Subject: [PATCH 27/83] chore: complete docstring for RSSETSMarker() constructor --- junifer/markers/ets_rss.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py index c9c3d24ae..ec1e29045 100644 --- a/junifer/markers/ets_rss.py +++ b/junifer/markers/ets_rss.py @@ -24,9 +24,13 @@ class RSSETSMarker(BaseMarker): Parameters ---------- atlas : str - The name of the atlas. + The name of the atlas. Check valid options by calling + `junifer.data.list_atlases`. aggregation_method : str, optional The aggregation method (default "mean"). + name : str, optional + The name of the marker. If None, will use the class name (default + None). """ -- 2.52.0 From 5f39da72d074178146ad2596728a33ed85e5ade6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 19:33:50 +0200 Subject: [PATCH 28/83] update: improve docstring and type annotations for RSSETSMarker; fix: correct storage interface for RSSETSMarker --- junifer/markers/ets_rss.py | 44 ++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py index ec1e29045..e016301cc 100644 --- a/junifer/markers/ets_rss.py +++ b/junifer/markers/ets_rss.py @@ -6,7 +6,7 @@ # Synchon Mandal # License: AGPL -from typing import Dict, List +from typing import TYPE_CHECKING, Any, Dict, List, Optional import numpy as np @@ -17,6 +17,10 @@ from .parcel import ParcelAggregation from .utils import _ets +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + @register_marker class RSSETSMarker(BaseMarker): """Class for root sum of squares of edgewise timeseries. @@ -62,21 +66,32 @@ class RSSETSMarker(BaseMarker): """ return ["timeseries"] - # TODO: complete type annotations - def store(self, kind: str, out, storage) -> None: + def store( + self, + kind: str, + out: Dict[str, Any], + storage: "BaseFeatureStorage", + ) -> None: """Store. Parameters ---------- - kind - out - storage + kind : {"BOLD"} + The data kind to store. + out : dict + The computed result as a dictionary to store. + storage : storage-like + The storage class, for example, SQLiteFeatureStorage. """ logger.debug(f"Storing BOLD in {storage}") - storage.store_timeseries(**out) + storage.store(kind="timeseries", **out) - def compute(self, input: Dict) -> Dict: + def compute( + self, + input: Dict[str, Any], + extra_input: Optional[Dict] = None, + ) -> Dict: """Compute. Take a timeseries of brain areas, and calculate timeseries for each @@ -85,12 +100,19 @@ class RSSETSMarker(BaseMarker): Parameters ---------- - input : dict of the BOLD data + input : dict + The BOLD data as dictionary. + extra_input : dict, optional + The other fields in the pipeline data object (default None). Returns ------- dict - The computed result as dictionary. + The computed result as dictionary. The dictionary has the following + keys: + - data : the actual computed values as a numpy.ndarray + - columns : the column labels for the computed values as a list + - row_names (if more than one row is present in data): 'scan' References ---------- @@ -107,7 +129,7 @@ class RSSETSMarker(BaseMarker): method=self.aggregation_method, ) # Compute the parcel aggregation - out = parcel_aggregation.compute(input) + out = parcel_aggregation.compute(input=input, extra_input=extra_input) edge_ts = _ets(out["data"]) # Compute the RSS out["data"] = np.sum(edge_ts**2, 1) ** 0.5 -- 2.52.0 From 4aae546f0a5f931d04bf52e77d95e61c8d02d68c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 13 Oct 2022 19:35:11 +0200 Subject: [PATCH 29/83] update: improve and complete testing for RSSETSMarker --- junifer/markers/tests/test_ets_rss.py | 52 +++++++++++++++++---------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/junifer/markers/tests/test_ets_rss.py b/junifer/markers/tests/test_ets_rss.py index 8a38c56e8..0ccc23737 100644 --- a/junifer/markers/tests/test_ets_rss.py +++ b/junifer/markers/tests/test_ets_rss.py @@ -14,26 +14,35 @@ from nilearn.maskers import NiftiLabelsMasker from junifer.data import load_atlas from junifer.markers.ets_rss import RSSETSMarker from junifer.testing.datagrabbers import SPMAuditoryTestingDatagrabber +from junifer.storage import SQLiteFeatureStorage + + +# Set atlas +ATLAS = "Schaefer100x17" def test_compute() -> None: - """Test RSS ETS.""" - atlas = "Schaefer100x17" - test_atlas, _, _ = load_atlas(atlas) - + """Test RSS ETS compute().""" with SPMAuditoryTestingDatagrabber() as dg: + # Fetch element out = dg["sub001"] + # Load BOLD image niimg = image.load_img(str(out["BOLD"]["path"].absolute())) + # Create input data input_dict = {"data": niimg, "path": out["BOLD"]["path"]} # Compute the RSSETSMarker - ets_rss_marker = RSSETSMarker(atlas=atlas) + ets_rss_marker = RSSETSMarker(atlas=ATLAS) new_out = ets_rss_marker.compute(input_dict) + + # Load atlas + test_atlas, _, _ = load_atlas(ATLAS) # Compute the NiftiLabelsMasker test_masker = NiftiLabelsMasker(test_atlas) test_ts = test_masker.fit_transform(niimg) # Assert the dimension of timeseries n_time, _ = test_ts.shape assert n_time == len(new_out["data"]) + # Assert the meta meta = ets_rss_marker.get_meta("BOLD")["marker"] assert meta["atlas"] == "Schaefer100x17" @@ -42,10 +51,8 @@ def test_compute() -> None: def test_get_output_kind() -> None: - """Test get_output_kind.""" - - atlas = "Schaefer100x17" - ets_rss_marker = RSSETSMarker(atlas=atlas) + """Test RSS ETS get_output_kind().""" + ets_rss_marker = RSSETSMarker(atlas=ATLAS) input_list = ["BOLD"] input_list = ets_rss_marker.get_output_kind(input_list) assert len(input_list) == 1 @@ -53,19 +60,26 @@ def test_get_output_kind() -> None: def test_store(tmp_path: Path) -> None: - """Test store.""" + """Test RSS ETS store(). - atlas = "Schaefer100x17" + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ with SPMAuditoryTestingDatagrabber() as dg: + # Fetch element out = dg["sub001"] + # Load BOLD image niimg = image.load_img(str(out["BOLD"]["path"].absolute())) input_dict = {"data": niimg, "path": out["BOLD"]["path"]} # Compute the RSSETSMarker - ets_rss_marker = RSSETSMarker(atlas=atlas) - _ = ets_rss_marker.compute(input_dict) - # TODO: Needs store_timeseries implemented for SQLiteFeatureStorage - # storage = { - # "kind": "SQLiteFeatureStorage", - # "uri": str((tmp_path / "test.db").absolute()), - # } - # ets_rss_marker.store("SQLiteFeatureStorage", new_out, storage) + ets_rss_marker = RSSETSMarker(atlas=ATLAS) + # Create storage + storage = SQLiteFeatureStorage( + uri=str((tmp_path / "test.db").absolute()), + single_output=True, + ) + # Store + ets_rss_marker.fit_transform(input=input_dict, storage=storage) -- 2.52.0 From ffe8d35327e6271911404e2161cf318f28709127 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 07:46:26 +0200 Subject: [PATCH 30/83] fix: set correct column label for RSSETSMarker compute output --- junifer/markers/ets_rss.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py index e016301cc..9a99f5b4f 100644 --- a/junifer/markers/ets_rss.py +++ b/junifer/markers/ets_rss.py @@ -133,5 +133,6 @@ class RSSETSMarker(BaseMarker): edge_ts = _ets(out["data"]) # Compute the RSS out["data"] = np.sum(edge_ts**2, 1) ** 0.5 - + # Set correct column label + out["columns"] = ["root_sum_of_squares_ets"] return out -- 2.52.0 From 8bb2f4450114462b91729f1175cac506e77e6ab1 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 07:47:06 +0200 Subject: [PATCH 31/83] chore: isort test_ets_rss.py --- junifer/markers/tests/test_ets_rss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/markers/tests/test_ets_rss.py b/junifer/markers/tests/test_ets_rss.py index 0ccc23737..2c7b793b9 100644 --- a/junifer/markers/tests/test_ets_rss.py +++ b/junifer/markers/tests/test_ets_rss.py @@ -13,8 +13,8 @@ from nilearn.maskers import NiftiLabelsMasker from junifer.data import load_atlas from junifer.markers.ets_rss import RSSETSMarker -from junifer.testing.datagrabbers import SPMAuditoryTestingDatagrabber from junifer.storage import SQLiteFeatureStorage +from junifer.testing.datagrabbers import SPMAuditoryTestingDatagrabber # Set atlas -- 2.52.0 From 7c30393bd988b310c61be2d1d04310dc10f28a9b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 07:47:31 +0200 Subject: [PATCH 32/83] update: complete example for RSSETSMarker --- examples/run_ets_rss_marker.py | 51 +++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/examples/run_ets_rss_marker.py b/examples/run_ets_rss_marker.py index dc8901bdd..36cc6db07 100644 --- a/examples/run_ets_rss_marker.py +++ b/examples/run_ets_rss_marker.py @@ -15,6 +15,8 @@ License: BSD 3 clause import tempfile import junifer.testing.registry # noqa: F401 +from junifer.api import collect, run +from junifer.storage import SQLiteFeatureStorage from junifer.utils import configure_logging @@ -22,11 +24,15 @@ from junifer.utils import configure_logging # Set the logging level to info to see extra information: configure_logging(level="INFO") +############################################################################## +# Define the datagrabber interface +datagrabber = { + "kind": "SPMAuditoryTestingDatagrabber", +} ############################################################################### -# Define the markers you want: - -marker_dicts = [ +# Define the markers interface +markers = [ { "name": "Schaefer100x17_RSSETS", "kind": "RSSETSMarker", @@ -39,26 +45,27 @@ marker_dicts = [ }, ] - ############################################################################### # Create a temporary directory for junifer feature extraction: # At the end you can read the extracted data into a ``pandas.DataFrame``. with tempfile.TemporaryDirectory() as tmpdir: - - storage = {"kind": "SQLiteFeatureStorage", "uri": f"{tmpdir}/test.db"} - # run the defined junifer feature extraction pipeline - # TODO: needs SQLiteFeatureStorage.store_timeseries() to be - # implemented first - # run( - # workdir="/tmp", - # datagrabber={"kind": "SPMAuditoryTestingDatagrabber"}, - # markers=marker_dicts, - # storage=storage, - # ) - - # read in extracted features and add confounds and targets - # for julearn run cross validation - # This will not run for now as store_timeseries() is not implemented yet - # collect(storage) - # db = SQLiteFeatureStorage(uri=storage["uri"], single_output=True) - # df_vbm = db.read_df(feature_name="Schaefer100x17") + # Define the storage interface + storage = { + "kind": "SQLiteFeatureStorage", + "uri": f"{tmpdir}/test.db", + "single_output": False, + } + # Run the defined junifer feature extraction pipeline + run( + workdir=tmpdir, + datagrabber=datagrabber, + markers=markers, + storage=storage, + elements="sub001", + ) + # Collect extracted features data + collect(storage=storage) + # Create storage object to read in extracted features + db = SQLiteFeatureStorage(uri=storage["uri"], single_output=True) + # Read extracted features + df_vbm = db.read_features(feature_name="BOLD_Schaefer100x17_RSSETS") -- 2.52.0 From a9cb9d43b043972b15c018d00873840936c03125 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 08:28:05 +0200 Subject: [PATCH 33/83] update: improve RSSETSMarker example --- examples/run_ets_rss_marker.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/run_ets_rss_marker.py b/examples/run_ets_rss_marker.py index 36cc6db07..4cc82cb71 100644 --- a/examples/run_ets_rss_marker.py +++ b/examples/run_ets_rss_marker.py @@ -2,7 +2,7 @@ Extracting root sum of squares from edge-wise timeseries. ========================================================= -This example uses a RSSETSMarker to compute root sum of squares +This example uses a ``RSSETSMarker`` to compute root sum of squares of the edge-wise timeseries using the Schaefer atlas (100 rois and 200 rois, 17 Yeo networks) for a 4D nifti BOLD file. @@ -61,11 +61,18 @@ with tempfile.TemporaryDirectory() as tmpdir: datagrabber=datagrabber, markers=markers, storage=storage, - elements="sub001", + elements=["sub001"], # we calculate for one subject only ) # Collect extracted features data collect(storage=storage) # Create storage object to read in extracted features - db = SQLiteFeatureStorage(uri=storage["uri"], single_output=True) + db = SQLiteFeatureStorage( + uri=storage["uri"], + single_output=True, # as we ran collect, we have single output now + ) # Read extracted features df_vbm = db.read_features(feature_name="BOLD_Schaefer100x17_RSSETS") + +############################################################################### +# Now we take a look at the dataframe +df_vbm.head() -- 2.52.0 From 2a4c70258eae362ce3037bad0226e0cb278a2b8a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 08:28:21 +0200 Subject: [PATCH 34/83] fix: docstring fix for proper documentation build --- junifer/markers/parcel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/markers/parcel.py b/junifer/markers/parcel.py index 0097828a7..144ba77fa 100644 --- a/junifer/markers/parcel.py +++ b/junifer/markers/parcel.py @@ -37,7 +37,7 @@ class ParcelAggregation(BaseMarker): Parameters to pass to the aggregation function. Check valid options in `junifer.stats.get_aggfunc_by_name`. on : {"T1w", "BOLD", "VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"} or list - of the options, optional + of the options, optional The kind of data to apply the marker to. If None, will work on all available data (default None). name : str, optional -- 2.52.0 From 9361f8741661dabb5fdc5562f5d99259481f5939 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 08:29:13 +0200 Subject: [PATCH 35/83] fix: replace remaining read_df() usage to read_features() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 94b040c09..dedbe86a0 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -633,7 +633,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # TODO: Fix this, needs that read_feature sets the index # properly table_name = f"meta_{meta_md5}" - t_df = in_storage.read_df(feature_md5=meta_md5) + t_df = in_storage.read_features(feature_md5=meta_md5) # Save data out_storage._save_upsert(t_df, table_name, if_exists="nocheck") -- 2.52.0 From 7a50f7d4ca36c0c9865e485d453324ef4cc8afb0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 08:33:24 +0200 Subject: [PATCH 36/83] chore: improve docstring for ets_rss.py --- junifer/markers/ets_rss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py index 9a99f5b4f..1b1516d6f 100644 --- a/junifer/markers/ets_rss.py +++ b/junifer/markers/ets_rss.py @@ -29,7 +29,7 @@ class RSSETSMarker(BaseMarker): ---------- atlas : str The name of the atlas. Check valid options by calling - `junifer.data.list_atlases`. + :func:`junifer.data.list_atlases`. aggregation_method : str, optional The aggregation method (default "mean"). name : str, optional @@ -112,7 +112,7 @@ class RSSETSMarker(BaseMarker): keys: - data : the actual computed values as a numpy.ndarray - columns : the column labels for the computed values as a list - - row_names (if more than one row is present in data): 'scan' + - row_names (if more than one row is present in data): "scan" References ---------- -- 2.52.0 From 738805e40f9e46ed2a139473cb1ba08e4b7cf59f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 08:33:49 +0200 Subject: [PATCH 37/83] chore: improve docstring for parcel.py --- junifer/markers/parcel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/junifer/markers/parcel.py b/junifer/markers/parcel.py index 144ba77fa..b4c64a150 100644 --- a/junifer/markers/parcel.py +++ b/junifer/markers/parcel.py @@ -29,13 +29,13 @@ class ParcelAggregation(BaseMarker): ---------- atlas : str The name of the atlas. Check valid options by calling - `junifer.data.list_atlases`. + :func:`junifer.data.list_atlases`. method : str The method to perform aggregation using. Check valid options in - `junifer.stats.get_aggfunc_by_name`. + :func:`junifer.stats.get_aggfunc_by_name`. method_params : dict, optional Parameters to pass to the aggregation function. Check valid options in - `junifer.stats.get_aggfunc_by_name`. + :func:`junifer.stats.get_aggfunc_by_name`. on : {"T1w", "BOLD", "VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"} or list of the options, optional The kind of data to apply the marker to. If None, will work on all @@ -136,7 +136,7 @@ class ParcelAggregation(BaseMarker): with this as a parameter. The dictionary has the following keys: - data : the actual computed values as a numpy.ndarray - columns : the column labels for the computed values as a list - - row_names (if more than one row is present in data): 'scan' + - row_names (if more than one row is present in data): "scan" """ t_input = input["data"] -- 2.52.0 From 3dc70ec5ccbf772e069f7f97614a0e53922c6a41 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 09:32:32 +0200 Subject: [PATCH 38/83] update: complete type annotations and improve docstrings in SphereAggregation() --- junifer/markers/sphere_aggregation.py | 43 ++++++++++++++++++++------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py index 6c3797942..7a45e0bb4 100644 --- a/junifer/markers/sphere_aggregation.py +++ b/junifer/markers/sphere_aggregation.py @@ -4,7 +4,7 @@ # Synchon Mandal # License: AGPL -from typing import Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from nilearn.maskers import NiftiSpheresMasker @@ -14,6 +14,10 @@ from ..utils import logger, raise_error from .base import BaseMarker +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + @register_marker class SphereAggregation(BaseMarker): """Class for sphere aggregation. @@ -22,7 +26,7 @@ class SphereAggregation(BaseMarker): ---------- coords: str The name of the coordinates list to use. See - :mod:`junifer.data.coordinates` + :mod:`junifer.data.coordinates` for options. radius: float The radius of the sphere in mm. If None, the signal will be extracted from a single voxel. See :class:`nilearn.maskers.NiftiSpheresMasker` @@ -30,7 +34,7 @@ class SphereAggregation(BaseMarker): method: str The aggregation method to use. See :func:`junifer.stats.get_aggfunc_by_name` for more information. - method_params: Dict, optional + method_params: dict, optional The parameters to pass to the aggregation method. on: list of str, optional The kind of data to apply the marker to. By default, will work on all @@ -92,14 +96,22 @@ class SphereAggregation(BaseMarker): raise ValueError(f"Unknown input kind for {t_input}") return outputs - def store(self, kind: str, out, storage) -> None: + def store( + self, + kind: str, + out: Dict[str, Any], + storage: "BaseFeatureStorage", + ) -> None: """Store. Parameters ---------- - kind - out - storage + kind : {"BOLD", "VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"} + The data kind to store. + out : dict + The computed result as a dictionary to store. + storage : storage-like + The storage class, for example, SQLiteFeatureStorage. """ logger.debug(f"Storing {kind} in {storage}") @@ -108,15 +120,19 @@ class SphereAggregation(BaseMarker): elif kind in ["BOLD"]: storage.store_timeseries(**out) - def compute(self, input: Dict, extra_input: Optional[Dict] = None) -> Dict: + def compute( + self, + input: Dict[str, Any], + extra_input: Optional[Dict] = None, + ) -> Dict: """Compute. Parameters ---------- - input : Dict[str, Dict] + input : dict A single input from the pipeline data object in which to compute the marker. - extra_input : Dict, optional + extra_input : dict, optional The other fields in the pipeline data object. Useful for accessing other data kind that needs to be used in the computation. For example, the functional connectivity markers can make use of the @@ -125,7 +141,12 @@ class SphereAggregation(BaseMarker): Returns ------- dict - The computed result as dictionary. + The computed result as dictionary. This will be either returned + to the user or stored in the storage by calling the store method + with this as a parameter. The dictionary has the following keys: + - data : the actual computed values as a numpy.ndarray + - columns : the column labels for the computed values as a list + - row_names (if more than one row is present in data): "scan" """ t_input = input["data"] -- 2.52.0 From df2bf1e330c9335c26104954ef4129c967e03541 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 09:33:01 +0200 Subject: [PATCH 39/83] update: correct interface for store() in SphereAggregation() --- junifer/markers/sphere_aggregation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py index 7a45e0bb4..d738ab8f6 100644 --- a/junifer/markers/sphere_aggregation.py +++ b/junifer/markers/sphere_aggregation.py @@ -116,9 +116,9 @@ class SphereAggregation(BaseMarker): """ logger.debug(f"Storing {kind} in {storage}") if kind in ["VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"]: - storage.store_table(**out) + storage.store(kind="table", **out) elif kind in ["BOLD"]: - storage.store_timeseries(**out) + storage.store(kind="timeseries", **out) def compute( self, -- 2.52.0 From a69f2d29dfe706e8b9e021cb02c817d6ce131a84 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 09:33:19 +0200 Subject: [PATCH 40/83] update: complete storage test for SphereAggregation() --- .../markers/tests/test_sphere_aggregation.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/junifer/markers/tests/test_sphere_aggregation.py b/junifer/markers/tests/test_sphere_aggregation.py index d7733592b..9b5d1a0f0 100644 --- a/junifer/markers/tests/test_sphere_aggregation.py +++ b/junifer/markers/tests/test_sphere_aggregation.py @@ -138,19 +138,17 @@ def test_SphereAggregation_storage(tmp_path: Path) -> None: marker.fit_transform(input, storage=storage) - # TODO: Needs store_timeseries implemented for SQLiteFeatureStorage + meta = { + "element": "test", + "version": "0.0.1", + "marker": {"name": "BOLD_fcname"}, + } + # Get the SPM auditory data: + subject_data = datasets.fetch_spm_auditory() + fmri_img = concat_imgs(subject_data.func) # type: ignore + input = {"BOLD": {"data": fmri_img}, "meta": meta} + marker = SphereAggregation( + coords="DMNBuckner", method="mean", radius=8, on="BOLD" + ) - # meta = { - # "element": "test", - # "version": "0.0.1", - # "marker": {"name": "BOLD_fcname"}, - # } - # # Get the SPM auditory data: - # subject_data = datasets.fetch_spm_auditory() - # fmri_img = concat_imgs(subject_data.func) # type: ignore - # input = {"BOLD": {"data": fmri_img}, "meta": meta} - # marker = SphereAggregation( - # coords="DMNBuckner", method="mean", radius=8, on="BOLD" - # ) - - # marker.fit_transform(input, storage=storage) + marker.fit_transform(input, storage=storage) -- 2.52.0 From 7316634dce81adc7f3f890071b63b63a951772d4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 09:33:45 +0200 Subject: [PATCH 41/83] fix: replace read_df() with read_features() in run_junifer_julearn.py --- examples/run_junifer_julearn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_junifer_julearn.py b/examples/run_junifer_julearn.py index 1b58bdc30..6ae1298ff 100644 --- a/examples/run_junifer_julearn.py +++ b/examples/run_junifer_julearn.py @@ -84,7 +84,7 @@ with tempfile.TemporaryDirectory() as tmpdir: collect(storage) db = SQLiteFeatureStorage(uri=storage["uri"], single_output=True) - df_vbm = db.read_df(feature_name="VBM_GM_Schaefer200x17_Mean") + df_vbm = db.read_features(feature_name="VBM_GM_Schaefer200x17_Mean") oasis_subjects = [x[0] for x in df_vbm.index] df_vbm.index = oasis_subjects -- 2.52.0 From 08ae7de37bbe477ffe4f40a74dcb19aa50d0e8ab Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:25:19 +0200 Subject: [PATCH 42/83] update: complete type annotations and improve docstring in FunctionalConnectivityAtlas() --- .../markers/functional_connectivity_atlas.py | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/junifer/markers/functional_connectivity_atlas.py b/junifer/markers/functional_connectivity_atlas.py index a0ee6ffcf..ab0d53259 100644 --- a/junifer/markers/functional_connectivity_atlas.py +++ b/junifer/markers/functional_connectivity_atlas.py @@ -4,7 +4,7 @@ # Kaustubh R. Patil # License: AGPL -from typing import Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from nilearn.connectome import ConnectivityMeasure from sklearn.covariance import EmpiricalCovariance @@ -15,18 +15,34 @@ from .base import BaseMarker from .parcel import ParcelAggregation +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + @register_marker class FunctionalConnectivityAtlas(BaseMarker): """Class for functional connectivity. Parameters ---------- - atlas - agg_method - agg_method_params - cor_method - cor_method_params - name + atlas : str + The name of the atlas. Check valid options by calling + :func:`junifer.data.list_atlases`. + agg_method : str, optional + The method to perform aggregation using. Check valid options in + :func:`junifer.stats.get_aggfunc_by_name` (default "mean"). + agg_method_params : dict, optional + Parameters to pass to the aggregation function. Check valid options in + :func:`junifer.stats.get_aggfunc_by_name` (default None). + cor_method : str, optional + The method to perform correlation using. Check valid options in + :func:`nilearn.connectome.ConnectivityMeasure` (default "covariance"). + cor_method_params : dict, optional + Parameters to pass to the correlation function. Check valid options in + :func:`nilearn.connectome.ConnectivityMeasure` (default None). + name : str, optional + The name of the marker. If None, will use the class name (default + None). """ @@ -75,15 +91,19 @@ class FunctionalConnectivityAtlas(BaseMarker): outputs = ["matrix"] return outputs - def compute(self, input: Dict, extra_input: Optional[Dict] = None) -> Dict: + def compute( + self, + input: Dict[str, Any], + extra_input: Optional[Dict] = None, + ) -> Dict: """Compute. Parameters ---------- - input : Dict[str, Dict] + input : dict A single input from the pipeline data object in which to compute the marker. - extra_input : Dict, optional + extra_input : dict, optional The other fields in the pipeline data object. Useful for accessing other data kind that needs to be used in the computation. For example, the functional connectivity markers can make use of the @@ -94,10 +114,11 @@ class FunctionalConnectivityAtlas(BaseMarker): dict The computed result as dictionary. The following data will be included in the dictionary: - - 'data': FC matrix as a 2D numpy array. - - 'row_names': Row names as a list. - - 'col_names': Col names as a list. - - 'kind': The kind of matrix (tril, triu or full) + - data: functional connectivity matrix as a numpy.ndarray. + - row_names: row names as a list + - col_names: column names as a list + - matrix_kind: the kind of matrix (tril, triu or full) + """ pa = ParcelAggregation( atlas=self.atlas, @@ -123,14 +144,22 @@ class FunctionalConnectivityAtlas(BaseMarker): out["kind"] = "tril" return out - # TODO: complete type annotations - def store(self, kind: str, out: Dict, storage) -> None: + def store( + self, + kind: str, + out: Dict[str, Any], + storage: "BaseFeatureStorage", + ) -> None: """Store. Parameters ---------- - input - out + kind : {"BOLD"} + The data kind to store. + out : dict + The computed result as a dictionary to store. + storage : storage-like + The storage class, for example, SQLiteFeatureStorage. """ logger.debug(f"Storing {kind} in {storage}") -- 2.52.0 From 87aea88abf1badbb9ba2d1f51adbe69624ea3312 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:25:48 +0200 Subject: [PATCH 43/83] update: change storage parameter name from kind to matrix_kind to avoid issues --- junifer/markers/functional_connectivity_atlas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/markers/functional_connectivity_atlas.py b/junifer/markers/functional_connectivity_atlas.py index ab0d53259..71adff8c7 100644 --- a/junifer/markers/functional_connectivity_atlas.py +++ b/junifer/markers/functional_connectivity_atlas.py @@ -141,7 +141,7 @@ class FunctionalConnectivityAtlas(BaseMarker): # create column names out["row_names"] = ts["columns"] out["col_names"] = ts["columns"] - out["kind"] = "tril" + out["matrix_kind"] = "tril" return out def store( -- 2.52.0 From 4a2fc6d922162ec2fdc95902857c15f7465aa766 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:26:20 +0200 Subject: [PATCH 44/83] update: use updated interface for FunctionalConnectivityAtlas() storage --- junifer/markers/functional_connectivity_atlas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/markers/functional_connectivity_atlas.py b/junifer/markers/functional_connectivity_atlas.py index 71adff8c7..bca80a2fd 100644 --- a/junifer/markers/functional_connectivity_atlas.py +++ b/junifer/markers/functional_connectivity_atlas.py @@ -163,4 +163,4 @@ class FunctionalConnectivityAtlas(BaseMarker): """ logger.debug(f"Storing {kind} in {storage}") - storage.store_matrix2d(**out) + storage.store(kind="matrix", **out) -- 2.52.0 From 43ef5e78e304ab8d435697f3a8d4d8d1aa1a62e3 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:27:16 +0200 Subject: [PATCH 45/83] update: change kind to matrix_kind for store_matrix() in SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index dedbe86a0..5bbcf6265 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -451,7 +451,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): meta: Dict, col_names: Optional[List[str]] = None, row_names: Optional[List[str]] = None, - kind: Optional[str] = "full", + matrix_kind: Optional[str] = "full", diagonal: bool = True, ) -> None: """Implement matrix storing. @@ -468,43 +468,43 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): The column name to use in case number of rows greater than 1. If None and number of rows greater than 1, then the name will be "index" (default None). - kind : str, optional + matrix_kind : str, optional The kind of matrix: - 'triu: store upper triangular only. - 'tril': store lower triangular. - 'full': full matrix (default 'full'). diagonal : bool, optional - Whether to store the diagonal (default True). - If kind == 'full', setting this to false will raise - an error. + Whether to store the diagonal. If `matrix_kind` is "full", setting + this to False will raise an error (default True).. """ - if diagonal is False and kind not in ["triu", "tril"]: + if diagonal is False and matrix_kind not in ["triu", "tril"]: raise_error( msg="Diagonal cannot be False if kind is not full", klass=ValueError, ) - if kind in ["triu", "tril"]: + if matrix_kind in ["triu", "tril"]: if data.shape[0] != data.shape[1]: raise_error( "Cannot store a non-square matrix as a triangular matrix", klass=ValueError, ) - if kind == "triu": + if matrix_kind == "triu": k = 0 if diagonal is True else 1 data_idx = np.triu_indices(data.shape[0], k=k) - elif kind == "tril": + elif matrix_kind == "tril": k = 0 if diagonal is True else -1 data_idx = np.tril_indices(data.shape[0], k=k) - elif kind == "full": + elif matrix_kind == "full": data_idx = ( np.repeat(np.arange(data.shape[0]), data.shape[1]), np.tile(np.arange(data.shape[1]), data.shape[0]), ) else: - raise_error(msg=f"Invalid kind {kind}", klass=ValueError) + raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError) + if row_names is None: row_names = [f"r{i}" for i in range(data.shape[0])] elif len(row_names) != data.shape[0]: -- 2.52.0 From a168eed809c3fd9d091371bc8864de6323b0d0ef Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:27:42 +0200 Subject: [PATCH 46/83] chore: improve docstring for SQLiteFeatureStorage() --- junifer/storage/sqlite.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 5bbcf6265..f4a76c318 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -470,9 +470,10 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): "index" (default None). matrix_kind : str, optional The kind of matrix: - - 'triu: store upper triangular only. - - 'tril': store lower triangular. - - 'full': full matrix (default 'full'). + - "triu" : store upper triangular only + - "tril" : store lower triangular + - "full" : full matrix + (default "full"). diagonal : bool, optional Whether to store the diagonal. If `matrix_kind` is "full", setting this to False will raise an error (default True).. @@ -538,8 +539,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): new_names = [x for x in data_df.index.names[:-1]] new_names.append("pair") data_df.index.names = new_names + # Store dataframe - self.store_df(df=data_df, meta=meta) # type: ignore + self.store_df(df=data_df, meta=meta) def store_table( self, -- 2.52.0 From cc71a0b72e211156ef5172a1932d84d3f47d7b1e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:28:02 +0200 Subject: [PATCH 47/83] chore: improve docstring for RSSETSMarker() --- junifer/markers/ets_rss.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py index 1b1516d6f..668773201 100644 --- a/junifer/markers/ets_rss.py +++ b/junifer/markers/ets_rss.py @@ -31,7 +31,8 @@ class RSSETSMarker(BaseMarker): The name of the atlas. Check valid options by calling :func:`junifer.data.list_atlases`. aggregation_method : str, optional - The aggregation method (default "mean"). + The method to perform aggregation using. Check valid options in + :func:`junifer.stats.get_aggfunc_by_name` (default "mean"). name : str, optional The name of the marker. If None, will use the class name (default None). -- 2.52.0 From 517597025479d351832246d60ee676df13c62aa9 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:28:22 +0200 Subject: [PATCH 48/83] chore: improve docstring in test_functional_connectivity_atlas.py --- .../tests/test_functional_connectivity_atlas.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/junifer/markers/tests/test_functional_connectivity_atlas.py b/junifer/markers/tests/test_functional_connectivity_atlas.py index 7aa435183..4b14b17c1 100644 --- a/junifer/markers/tests/test_functional_connectivity_atlas.py +++ b/junifer/markers/tests/test_functional_connectivity_atlas.py @@ -1,4 +1,4 @@ -"""Provide test for parcel aggregation.""" +"""Provide tests for functional connectivity atlas.""" # Authors: Amir Omidvarnia # Kaustubh R. Patil @@ -19,8 +19,14 @@ from junifer.storage import SQLiteFeatureStorage def test_FunctionalConnectivityAtlas(tmp_path: Path) -> None: - """Test FunctionalConnectivityAtlas.""" + """Test FunctionalConnectivityAtlas. + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ # get a dataset ni_data = datasets.fetch_spm_auditory(subject_id="sub001") fmri_img = image.concat_imgs(ni_data.func) # type: ignore -- 2.52.0 From 1dd35367e7997b6fb04c0b76b36064ebb24b2f83 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:28:45 +0200 Subject: [PATCH 49/83] update: expose FunctionalConnectivityAtlas in __init__.py --- junifer/markers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/junifer/markers/__init__.py b/junifer/markers/__init__.py index 0fe3ca032..58638f5d0 100644 --- a/junifer/markers/__init__.py +++ b/junifer/markers/__init__.py @@ -7,5 +7,6 @@ from .base import BaseMarker from .collection import MarkerCollection from .ets_rss import RSSETSMarker +from .functional_connectivity_atlas import FunctionalConnectivityAtlas from .parcel import ParcelAggregation from .sphere_aggregation import SphereAggregation -- 2.52.0 From e3173d415ae39d76664cd15b48789a3675050285 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:43:18 +0200 Subject: [PATCH 50/83] chore: complete type annotations in FunctionalConnectivityAtlas() --- junifer/markers/functional_connectivity_atlas.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/junifer/markers/functional_connectivity_atlas.py b/junifer/markers/functional_connectivity_atlas.py index bca80a2fd..4a84f215c 100644 --- a/junifer/markers/functional_connectivity_atlas.py +++ b/junifer/markers/functional_connectivity_atlas.py @@ -48,12 +48,12 @@ class FunctionalConnectivityAtlas(BaseMarker): def __init__( self, - atlas, - agg_method="mean", - agg_method_params=None, - cor_method="covariance", - cor_method_params=None, - name=None, + atlas: str, + agg_method: str = "mean", + agg_method_params: Optional[Dict] = None, + cor_method: str = "covariance", + cor_method_params: Optional[Dict] = None, + name: Optional[str] = None, ) -> None: """Initialize the class.""" self.atlas = atlas -- 2.52.0 From 2dfe172029de53d478b7a2b7b20a747f43a2f663 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:47:41 +0200 Subject: [PATCH 51/83] update: improve docstring and code quality in SphereAggregation() --- junifer/markers/sphere_aggregation.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py index d738ab8f6..e25478c52 100644 --- a/junifer/markers/sphere_aggregation.py +++ b/junifer/markers/sphere_aggregation.py @@ -27,15 +27,16 @@ class SphereAggregation(BaseMarker): coords: str The name of the coordinates list to use. See :mod:`junifer.data.coordinates` for options. - radius: float + radius: float, optional The radius of the sphere in mm. If None, the signal will be extracted from a single voxel. See :class:`nilearn.maskers.NiftiSpheresMasker` - for more information. - method: str + for more information (default None). + method: str, optional The aggregation method to use. - See :func:`junifer.stats.get_aggfunc_by_name` for more information. + See :func:`junifer.stats.get_aggfunc_by_name` for more information + (default "mean"). method_params: dict, optional - The parameters to pass to the aggregation method. + The parameters to pass to the aggregation method (default None). on: list of str, optional The kind of data to apply the marker to. By default, will work on all available data (default None). @@ -48,8 +49,8 @@ class SphereAggregation(BaseMarker): def __init__( self, coords: str, - radius: float, - method: str, + radius: Optional[float] = None, + method: str = "mean", method_params: Optional[Dict] = None, on: Optional[List[str]] = None, name: Optional[str] = None, @@ -156,8 +157,8 @@ class SphereAggregation(BaseMarker): # ) coords, out_labels = load_coordinates(self.coords) masker = NiftiSpheresMasker( - coords, - self.radius, + seeds=coords, + radius=self.radius, mask_img=None, # TODO: support this (needs #79) ) -- 2.52.0 From af32dac5925a1febf4049e1308e1aab015fae6d7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:49:00 +0200 Subject: [PATCH 52/83] chore: improve type annotations and docstring in FunctionalConnectivitySpheres() --- .../functional_connectivity_spheres.py | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/junifer/markers/functional_connectivity_spheres.py b/junifer/markers/functional_connectivity_spheres.py index 36c401e25..f98ffc39a 100644 --- a/junifer/markers/functional_connectivity_spheres.py +++ b/junifer/markers/functional_connectivity_spheres.py @@ -4,7 +4,7 @@ # Kaustubh R. Patil # License: AGPL -from typing import Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from nilearn.connectome import ConnectivityMeasure from sklearn.covariance import EmpiricalCovariance @@ -15,6 +15,10 @@ from .base import BaseMarker from .sphere_aggregation import SphereAggregation +if TYPE_CHECKING: + from junifer.storage import BaseFeatureStorage + + @register_marker class FunctionalConnectivitySpheres(BaseMarker): """Class for functional connectivity using coordinates (spheres). @@ -23,16 +27,23 @@ class FunctionalConnectivitySpheres(BaseMarker): ---------- coords : str The name of the coordinates list to use. See - :mod:`junifer.data.coordinates` - radius : float + :mod:`junifer.data.coordinates` for options. + radius : float, optional The radius of the sphere in mm. If None, the signal will be extracted from a single voxel. See :class:`nilearn.maskers.NiftiSpheresMasker` - for more information. - agg_method : str + for more information (default None). + agg_method : str, optional The aggregation method to use. - See :func:`junifer.stats.get_aggfunc_by_name` for more information. + See :func:`junifer.stats.get_aggfunc_by_name` for more information + (default None). agg_method_params : dict, optional - The parameters to pass to the aggregation method. + The parameters to pass to the aggregation method (default None). + cor_method : str, optional + The method to perform correlation using. Check valid options in + :func:`nilearn.connectome.ConnectivityMeasure` (default "covariance"). + cor_method_params : dict, optional + Parameters to pass to the correlation function. Check valid options in + :func:`nilearn.connectome.ConnectivityMeasure` (default None). name : str, optional The name of the marker. By default, it will use KIND_FunctionalConnectivitySpheres where KIND is the kind of data it @@ -43,7 +54,7 @@ class FunctionalConnectivitySpheres(BaseMarker): def __init__( self, coords: str, - radius: float, + radius: Optional[float] = None, agg_method: str = "mean", agg_method_params: Optional[Dict] = None, cor_method: str = "covariance", @@ -88,12 +99,16 @@ class FunctionalConnectivitySpheres(BaseMarker): outputs = ["matrix"] return outputs - def compute(self, input: Dict, extra_input: Optional[Dict] = None) -> Dict: + def compute( + self, + input: Dict[str, Any], + extra_input: Optional[Dict] = None, + ) -> Dict: """Compute. Parameters ---------- - input : dict[str, dict] + input : dict A single input from the pipeline data object in which to compute the marker. extra_input : dict, optional @@ -107,10 +122,10 @@ class FunctionalConnectivitySpheres(BaseMarker): dict The computed result as dictionary. The following keys will be included in the dictionary: - - 'data': FC matrix as a 2D numpy array. - - 'row_names': Row names as a list. - - 'col_names': Col names as a list. - 'kind': The kind of matrix (tril, triu or full) + - data: functional connectivity matrix as a numpy.ndarray. + - row_names: row names as a list + - col_names: column names as a list """ sa = SphereAggregation( @@ -138,14 +153,22 @@ class FunctionalConnectivitySpheres(BaseMarker): out["kind"] = "tril" return out - # TODO: complete type annotations - def store(self, kind: str, out: Dict, storage) -> None: + def store( + self, + kind: str, + out: Dict[str, Any], + storage: "BaseFeatureStorage", + ) -> None: """Store. Parameters ---------- - input - out + kind : {"BOLD"} + The data kind to store. + out : dict + The computed result as a dictionary to store. + storage : storage-like + The storage class, for example, SQLiteFeatureStorage. """ logger.debug(f"Storing {kind} in {storage}") -- 2.52.0 From 9a242ce31b80b95fe4827833e1b6cc63b3a70168 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:50:46 +0200 Subject: [PATCH 53/83] update: replace kind with matrix_kind for FunctionalConnectivitySpheres() compute output --- junifer/markers/functional_connectivity_spheres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/markers/functional_connectivity_spheres.py b/junifer/markers/functional_connectivity_spheres.py index f98ffc39a..54a983d0b 100644 --- a/junifer/markers/functional_connectivity_spheres.py +++ b/junifer/markers/functional_connectivity_spheres.py @@ -122,10 +122,10 @@ class FunctionalConnectivitySpheres(BaseMarker): dict The computed result as dictionary. The following keys will be included in the dictionary: - - 'kind': The kind of matrix (tril, triu or full) - data: functional connectivity matrix as a numpy.ndarray. - row_names: row names as a list - col_names: column names as a list + - matrix_kind: the kind of matrix (tril, triu or full) """ sa = SphereAggregation( @@ -150,7 +150,7 @@ class FunctionalConnectivitySpheres(BaseMarker): # create column names out["row_names"] = ts["columns"] out["col_names"] = ts["columns"] - out["kind"] = "tril" + out["matrix_kind"] = "tril" return out def store( -- 2.52.0 From 8d558a25f63c9ab115525e60b1875982b8b60679 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:52:18 +0200 Subject: [PATCH 54/83] update: correct storage interface for FunctionalConnectivitySpheres() --- junifer/markers/functional_connectivity_spheres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/markers/functional_connectivity_spheres.py b/junifer/markers/functional_connectivity_spheres.py index 54a983d0b..6398bc5b1 100644 --- a/junifer/markers/functional_connectivity_spheres.py +++ b/junifer/markers/functional_connectivity_spheres.py @@ -172,4 +172,4 @@ class FunctionalConnectivitySpheres(BaseMarker): """ logger.debug(f"Storing {kind} in {storage}") - storage.store_matrix2d(**out) + storage.store(kind="matrix", **out) -- 2.52.0 From 2b6787d91cc8b5ba931dc7fc50517d83212b3fcb Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:57:05 +0200 Subject: [PATCH 55/83] chore: black formatting --- junifer/markers/functional_connectivity_spheres.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/junifer/markers/functional_connectivity_spheres.py b/junifer/markers/functional_connectivity_spheres.py index 6398bc5b1..e65a6f10b 100644 --- a/junifer/markers/functional_connectivity_spheres.py +++ b/junifer/markers/functional_connectivity_spheres.py @@ -100,9 +100,9 @@ class FunctionalConnectivitySpheres(BaseMarker): return outputs def compute( - self, - input: Dict[str, Any], - extra_input: Optional[Dict] = None, + self, + input: Dict[str, Any], + extra_input: Optional[Dict] = None, ) -> Dict: """Compute. @@ -154,10 +154,10 @@ class FunctionalConnectivitySpheres(BaseMarker): return out def store( - self, - kind: str, - out: Dict[str, Any], - storage: "BaseFeatureStorage", + self, + kind: str, + out: Dict[str, Any], + storage: "BaseFeatureStorage", ) -> None: """Store. -- 2.52.0 From 18568bcf998804db21924d800fcfdf1ddb01dcee Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:57:22 +0200 Subject: [PATCH 56/83] chore: improve docstring and code style for FunctionalConnectivitySpheres() tests --- .../tests/test_functional_connectivity_spheres.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/junifer/markers/tests/test_functional_connectivity_spheres.py b/junifer/markers/tests/test_functional_connectivity_spheres.py index a58b53927..96a41dfd7 100644 --- a/junifer/markers/tests/test_functional_connectivity_spheres.py +++ b/junifer/markers/tests/test_functional_connectivity_spheres.py @@ -1,18 +1,17 @@ -"""Provide test for functional connectivity spheres.""" +"""Provide tests for functional connectivity spheres.""" # Authors: Amir Omidvarnia # Kaustubh R. Patil # Federico Raimondo # License: AGPL -import pytest - from pathlib import Path -from numpy.testing import assert_array_almost_equal -from sklearn.covariance import EmpiricalCovariance +import pytest from nilearn import datasets, image from nilearn.connectome import ConnectivityMeasure +from numpy.testing import assert_array_almost_equal +from sklearn.covariance import EmpiricalCovariance from junifer.markers.functional_connectivity_spheres import ( FunctionalConnectivitySpheres, @@ -30,7 +29,6 @@ def test_FunctionalConnectivitySpheres(tmp_path: Path) -> None: The path to the test directory. """ - # get a dataset ni_data = datasets.fetch_spm_auditory(subject_id="sub001") fmri_img = image.concat_imgs(ni_data.func) # type: ignore @@ -118,8 +116,7 @@ def test_FunctionalConnectivitySpheres_empirical(tmp_path: Path) -> None: # Check that FC are almost equal when using nileran cm = ConnectivityMeasure( - cov_estimator=EmpiricalCovariance(), # type: ignore - kind="correlation" + cov_estimator=EmpiricalCovariance(), kind="correlation" # type: ignore ) out_ni = cm.fit_transform([ts["data"]])[0] assert_array_almost_equal(out_ni, out["data"], decimal=3) -- 2.52.0 From 3ea41d5be3de30e7f87559c6d7205815cd1bbc38 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 10:57:57 +0200 Subject: [PATCH 57/83] update: expose FunctionalConnectivitySpheres in __init__.py --- junifer/markers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/junifer/markers/__init__.py b/junifer/markers/__init__.py index 58638f5d0..6fcc780b9 100644 --- a/junifer/markers/__init__.py +++ b/junifer/markers/__init__.py @@ -8,5 +8,6 @@ from .base import BaseMarker from .collection import MarkerCollection from .ets_rss import RSSETSMarker from .functional_connectivity_atlas import FunctionalConnectivityAtlas +from .functional_connectivity_spheres import FunctionalConnectivitySpheres from .parcel import ParcelAggregation from .sphere_aggregation import SphereAggregation -- 2.52.0 From f401f4e070b0ccba5831a250812b07e8c34fa46e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 11:51:42 +0200 Subject: [PATCH 58/83] fix: rewrite tests for BaseMarker --- junifer/markers/tests/test_markers_base.py | 123 +++++++-------------- 1 file changed, 43 insertions(+), 80 deletions(-) diff --git a/junifer/markers/tests/test_markers_base.py b/junifer/markers/tests/test_markers_base.py index c86c074d0..12d0b583e 100644 --- a/junifer/markers/tests/test_markers_base.py +++ b/junifer/markers/tests/test_markers_base.py @@ -4,94 +4,57 @@ # Synchon Mandal # License: AGPL -from typing import List, Optional - import pytest from junifer.markers.base import BaseMarker -@pytest.mark.parametrize( - "on, name, kind, expected_class, expected_name", - [ - (["bold", "dwi"], None, "bold", "BaseMarker", "bold_BaseMarker"), - (["bold", "dwi"], "mymarker", "dwi", "BaseMarker", "dwi_mymarker"), - ], -) -def test_base_marker_meta( - on: List[str], - name: Optional[str], - kind: str, - expected_class: str, - expected_name: str, -) -> None: - """Test metadata for BaseMarker. - - Parameters - ---------- - on : list of str - The parametrized kind of data to work on. - name : str or None - The parametrized name of the marker. - kind : str - The parametrized kind of data to get metadata for. - expected_class : str - The paramtrized expected class of the marker. - expected_name : str - The parametrized expected name of the marker. - - """ - base = BaseMarker(on=on, name=name) - t_meta = base.get_meta(kind=kind) - assert t_meta["marker"]["class"] == expected_class - assert t_meta["marker"]["name"] == expected_name +def test_base_marker_abstractness() -> None: + """Test BaseMarker is abstract base class.""" + with pytest.raises(TypeError, match=r"abstract"): + BaseMarker(on=["BOLD"]) -def test_compute_parameters() -> None: - """Test compute parameters.""" - base = BaseMarker(on=["bold", "dwi"], name="mymarker") - base.compute = lambda x, y: { # type: ignore - "data": x.keys(), - "extra": y.keys(), +def test_base_datagrabber_subclassing() -> None: + """Test proper subclassing of BaseMarker.""" + # Create concrete class + class MyBaseMarker(BaseMarker): + def get_output_kind(self, input): + return ["timeseries"] + + def compute(self, input, extra_input): + return { + "data": "data", + "columns": "columns", + "row_names": "row_names", + } + + def store(self, kind, out, storage): + return super().store(kind=kind, out=out, storage=storage) + + # Create input for marker + input_ = { + "meta": {"datagrabber": "dg", "element": "elem", "datareader": "dr"}, + "BOLD": { + "path": ".", + "data": "data", + }, } - input_ = {"bold": {"path": "test"}, "t2": {"path": "test"}} - out = base.fit_transform( - input_, - ) - assert list(out["bold"]["data"]) == ["path"] - assert list(out["bold"]["extra"]) == ["t2"] - - -def test_BaseMarker() -> None: - """Test base class.""" - base = BaseMarker(on=["bold", "dwi"], name="mymarker") - input_ = {"bold": {"path": "test"}, "t2": {"path": "test"}} - base.validate_input(list(input_.keys())) - - wrong_input = {"t2": {"path": "test"}} - with pytest.raises(ValueError): - base.validate_input(list(wrong_input.keys())) + marker = MyBaseMarker(on=["BOLD"]) + output = marker.fit_transform(input=input_) # process + # Check output + assert "BOLD" in output + assert "data" in output["BOLD"] + assert "columns" in output["BOLD"] + assert "row_names" in output["BOLD"] + assert "meta" in output["BOLD"] + assert "datagrabber" in output["BOLD"]["meta"] + assert "element" in output["BOLD"]["meta"] + assert "datareader" in output["BOLD"]["meta"] + # Check no implementation check with pytest.raises(NotImplementedError): - base.get_output_kind(list(wrong_input.keys())) + marker.store(kind="kind", out="out", storage="storage") - with pytest.raises(NotImplementedError): - base.fit_transform(input_) - - with pytest.raises(NotImplementedError): - base.store("bold", {}, None) - - base.compute = lambda x, y: {"data": 1} # type: ignore - - out = base.fit_transform(input_) - assert out["bold"]["data"] == 1 - assert out["bold"]["meta"]["marker"]["name"] == "bold_mymarker" - assert out["bold"]["meta"]["marker"]["class"] == "BaseMarker" - assert "dwi" not in out - - base2 = BaseMarker(on="bold", name="mymarker") - base2.compute = lambda x, y: {"data": 1} # type: ignore - out2 = base2.fit_transform(input_) - assert out2["bold"]["data"] == 1 - assert out2["bold"]["meta"]["marker"]["name"] == "bold_mymarker" - assert out2["bold"]["meta"]["marker"]["class"] == "BaseMarker" + # Check attributes + assert marker.name == "MyBaseMarker" -- 2.52.0 From 47665be38ea410f67cc00e4cbc163f2ea62751ec Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:07:59 +0200 Subject: [PATCH 59/83] fix: replace read_df() with read_features() and kind with matrix_kind in test_sqlite.py --- junifer/storage/tests/test_sqlite.py | 73 ++++++++++++++++------------ 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index 0d3850f12..bec4a7a57 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -263,7 +263,7 @@ def test_upsert_invalid_option(tmp_path: Path) -> None: # TODO: can the tests be separated? -def test_store_df_and_read_df(tmp_path: Path) -> None: +def test_store_df_and_read_features(tmp_path: Path) -> None: """Test storing dataframe and reading of stored table into dataframe. Parameters @@ -307,20 +307,20 @@ def test_store_df_and_read_df(tmp_path: Path) -> None: assert table_name.replace("meta_", "") in features # Check for missing feature with pytest.raises(ValueError, match="not found"): - storage.read_df("wrong_md5") + storage.read_features("wrong_md5") # Check for missing feature to fetch with pytest.raises(ValueError, match="least one"): - storage.read_df() + storage.read_features() # Check for multiple features to fetch with pytest.raises(ValueError, match="Only one"): - storage.read_df("wrong_md5", "wrong_name") + storage.read_features("wrong_md5", "wrong_name") # Get MD5 hash of features feature_md5 = list(features.keys())[0] # Check for key assert "fcname" == features[feature_md5]["name"] # Read into dataframes - read_df1 = storage.read_df(feature_md5=feature_md5) - read_df2 = storage.read_df(feature_name="fcname") + read_df1 = storage.read_features(feature_md5=feature_md5) + read_df2 = storage.read_features(feature_name="fcname") # Check if dataframes are equal assert_frame_equal(read_df1, read_df2) assert_frame_equal(read_df1, to_store) @@ -406,8 +406,8 @@ def test_store_table(tmp_path: Path) -> None: assert_frame_equal(df_new, c_df_new) -def test_store_matrix2d(tmp_path: Path) -> None: - """Test 2D Matrix store. +def test_store_matrix(tmp_path: Path) -> None: + """Test matrix store. Parameters ---------- @@ -428,8 +428,8 @@ def test_store_matrix2d(tmp_path: Path) -> None: col_names = ["col1", "col2", "col3"] # Store table - storage.store_matrix2d( - data, meta, row_names=row_names, col_names=col_names + storage.store_matrix( + data=data, meta=meta, row_names=row_names, col_names=col_names, ) stored_names = [f"{i}~{j}" for i in row_names for j in col_names] @@ -438,14 +438,14 @@ def test_store_matrix2d(tmp_path: Path) -> None: feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_features(feature_md5=feature_md5) assert read_df.shape == (1, 12) assert_array_equal(read_df.values[0], data.flatten()) assert list(read_df.columns) == stored_names # Store without row and column names uri = tmp_path / "test_store_table_nonames.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) - storage.store_matrix2d(data, meta) + storage.store_matrix(data=data, meta=meta) stored_names = [ f"r{i}~c{j}" for i in range(data.shape[0]) @@ -454,17 +454,22 @@ def test_store_matrix2d(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_features(feature_md5=feature_md5) assert list(read_df.columns) == stored_names with pytest.raises(ValueError, match="Invalid kind"): - storage.store_matrix2d(data, meta, kind="wrong") + storage.store_matrix(data=data, meta=meta, matrix_kind="wrong") with pytest.raises(ValueError, match="non-square"): - storage.store_matrix2d(data, meta, kind="triu") + storage.store_matrix(data=data, meta=meta, matrix_kind="triu") with pytest.raises(ValueError, match="cannot be False"): - storage.store_matrix2d(data, meta, kind="full", diagonal=False) + storage.store_matrix( + data=data, + meta=meta, + matrix_kind="full", + diagonal=False, + ) # Store upper triangular matrix data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]]) @@ -472,8 +477,12 @@ def test_store_matrix2d(tmp_path: Path) -> None: col_names = ["col1", "col2", "col3"] uri = tmp_path / "test_store_table_triu.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) - storage.store_matrix2d( - data, meta, kind="triu", row_names=row_names, col_names=col_names + storage.store_matrix( + data=data, + meta=meta, + matrix_kind="triu", + row_names=row_names, + col_names=col_names, ) stored_names = [ @@ -488,7 +497,7 @@ def test_store_matrix2d(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_features(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.triu_indices(n=data.shape[0])][None, :] @@ -497,10 +506,10 @@ def test_store_matrix2d(tmp_path: Path) -> None: # Store upper triangular matrix without diagonal uri = tmp_path / "test_store_table_triu_nodiagonal.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) - storage.store_matrix2d( - data, - meta, - kind="triu", + storage.store_matrix( + data=data, + meta=meta, + matrix_kind="triu", row_names=row_names, col_names=col_names, diagonal=False, @@ -515,7 +524,7 @@ def test_store_matrix2d(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_features(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.triu_indices(n=data.shape[0], k=1)][None, :] @@ -527,8 +536,12 @@ def test_store_matrix2d(tmp_path: Path) -> None: col_names = ["col1", "col2", "col3"] uri = tmp_path / "test_store_table_tril.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) - storage.store_matrix2d( - data, meta, kind="tril", row_names=row_names, col_names=col_names + storage.store_matrix( + data=data, + meta=meta, + matrix_kind="tril", + row_names=row_names, + col_names=col_names, ) stored_names = [ @@ -543,7 +556,7 @@ def test_store_matrix2d(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_features(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.tril_indices(n=data.shape[0])][None, :] @@ -552,10 +565,10 @@ def test_store_matrix2d(tmp_path: Path) -> None: # Store lower triangular matrix without diagonal uri = tmp_path / "test_store_table_tril_nodiagonal.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) - storage.store_matrix2d( + storage.store_matrix( data, meta, - kind="tril", + matrix_kind="tril", row_names=row_names, col_names=col_names, diagonal=False, @@ -570,7 +583,7 @@ def test_store_matrix2d(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_features(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.tril_indices(n=data.shape[0], k=-1)][None, :] -- 2.52.0 From bccf98d35ef81e9128c796db53285f86a75edb90 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:13:13 +0200 Subject: [PATCH 60/83] fix: make test_storage_base.py work --- junifer/storage/tests/test_storage_base.py | 39 +++++----------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index c9af9db04..8c4c484ee 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -22,30 +22,21 @@ def test_BaseFeatureStorage() -> None: def __init__(self, uri, single_output=False): super().__init__(uri, single_output=single_output) - def validate(self, input): - super().validate(input) + def validate_input(self, input): + super().validate_input(input) def list_features(self): super().list_features() - def read_df(self, feature_name=None, feature_md5=None): - super().read_df(feature_name=feature_name, feature_md5=feature_md5) + def read_features(self, feature_name=None, feature_md5=None): + super().read_features( + feature_name=feature_name, + feature_md5=feature_md5, + ) def store_metadata(self, metadata): super().store_metadata(metadata) - def store_matrix2d(self, matrix, meta): - super().store_matrix2d(matrix, meta) - - def store_table(self, table, meta): - super().store_table(table, meta) - - def store_df(self, df, meta): - super().store_df(df, meta) - - def store_timeseries(self, timeseries, meta): - super().store_timeseries(timeseries, meta) - def collect(self): return super().collect() @@ -56,29 +47,17 @@ def test_BaseFeatureStorage() -> None: assert st.single_output is True with pytest.raises(NotImplementedError): - st.validate(None) + st.validate_input(None) with pytest.raises(NotImplementedError): st.list_features() with pytest.raises(NotImplementedError): - st.read_df(None) + st.read_features(None) with pytest.raises(NotImplementedError): st.store_metadata(None) - with pytest.raises(NotImplementedError): - st.store_matrix2d(None, None) - - with pytest.raises(NotImplementedError): - st.store_table(None, None) - - with pytest.raises(NotImplementedError): - st.store_df(None, None) # type: ignore - - with pytest.raises(NotImplementedError): - st.store_timeseries(None, None) - with pytest.raises(NotImplementedError): st.collect() -- 2.52.0 From 5c0333248933134d62290206c596283c488b76a6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:30:42 +0200 Subject: [PATCH 61/83] chore: complete type annotations and improve docstring in MarkerCollection() --- junifer/markers/collection.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/junifer/markers/collection.py b/junifer/markers/collection.py index 1897783f7..a0f220cef 100644 --- a/junifer/markers/collection.py +++ b/junifer/markers/collection.py @@ -5,7 +5,7 @@ # License: AGPL from collections import Counter -from typing import Dict, List, Optional +from typing import TYPE_CHECKING, Dict, List, Optional from ..datareader.default import DefaultDataReader from ..markers.base import BaseMarker @@ -14,15 +14,23 @@ from ..storage.base import BaseFeatureStorage from ..utils import logger +if TYPE_CHECKING: + from junifer.datagrabber import BaseDataGrabber + + class MarkerCollection: """Class for marker collection. Parameters ---------- - markers - datareader - preprocessing - storage + markers : list of marker-like + The markers to compute. + datareader : datareader-like, optional + The datareader to use (default None). + preprocessing : preprocessing-like, optional + The preprocessing steps to apply. + storage : storage-like, optional + The storage to use (default None). """ @@ -60,17 +68,23 @@ class MarkerCollection: Returns ------- - output : dict or None + dict or None The output of the pipeline. Each key represents a marker name and the values are the computer marker values. If the pipeline has a storage configured, then the output will be None. """ logger.info("Fitting pipeline") + + # Fetch actual data using datareader data = self._datareader.fit_transform(input) + + # Apply preprocessing steps if self._preprocessing is not None: logger.info("Preprocessing data") data = self._preprocessing.fit_transform(data) + + # Compute markers out = {} for marker in self._markers: logger.info(f"Fitting marker {marker.name}") @@ -78,20 +92,21 @@ class MarkerCollection: if self._storage is None: out[marker.name] = m_value logger.info("Marker collection fitting done") + return None if self._storage else out - # TODO: complete type annotations - def validate(self, datagrabber) -> None: + def validate(self, datagrabber: "BaseDataGrabber") -> None: """Validate the pipeline. - Without doing any computation, check if the Marker Collection can + Without doing any computation, check if the marker collection can be fit without problems. That is, the data required for each marker is present and streamed down the steps. Also, if a storage is configured, check that the storage can handle the markers output. Parameters ---------- - datagrabber + datagrabber : datagrabber-like + The datagrabber to validate. """ logger.info("Validating Marker Collection") -- 2.52.0 From 68798cba3c247db2c5d2ce392af564a2c5e2ec7d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:31:25 +0200 Subject: [PATCH 62/83] fix: replace validate() with validate_input() in MarkerCollection() --- junifer/markers/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/markers/collection.py b/junifer/markers/collection.py index a0f220cef..3ff8746fc 100644 --- a/junifer/markers/collection.py +++ b/junifer/markers/collection.py @@ -119,8 +119,8 @@ class MarkerCollection: for marker in self._markers: logger.info(f"Validating Marker: {marker.name}") - m_data = marker.validate(t_data) + m_data = marker.validate_input(t_data) logger.info(f"Marker output type: {m_data}") if self._storage is not None: logger.info(f"Validating storage for {marker.name}") - self._storage.validate(m_data) + self._storage.validate_input(m_data) -- 2.52.0 From 128eefa6bb1d41673c640eff8db38c4646d24204 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:31:53 +0200 Subject: [PATCH 63/83] chore: complete type annotations in MarkerCollection() tests --- junifer/markers/tests/test_collection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/junifer/markers/tests/test_collection.py b/junifer/markers/tests/test_collection.py index 8ef21777d..054137eed 100644 --- a/junifer/markers/tests/test_collection.py +++ b/junifer/markers/tests/test_collection.py @@ -4,6 +4,8 @@ # Synchon Mandal # License: AGPL +from pathlib import Path + import pytest from numpy.testing import assert_array_equal @@ -94,7 +96,7 @@ def test_marker_collection(): ) -def test_MarkerCollection_storage(tmp_path) -> None: +def test_marker_collection_storage(tmp_path: Path) -> None: """Test marker collection with storage. Parameters -- 2.52.0 From da07944148ad8e1ec6580e9fe394099fdcc77908 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:32:20 +0200 Subject: [PATCH 64/83] fix: replace read_df() with read_features() in MarkerCollection() tests --- junifer/markers/tests/test_collection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/junifer/markers/tests/test_collection.py b/junifer/markers/tests/test_collection.py index 054137eed..8105f13a8 100644 --- a/junifer/markers/tests/test_collection.py +++ b/junifer/markers/tests/test_collection.py @@ -125,7 +125,7 @@ def test_marker_collection_storage(tmp_path: Path) -> None: uri = tmp_path / "test_marker_collection_storage.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) mc = MarkerCollection( - markers=markers, storage=storage, datareader=DefaultDataReader() + markers=markers, storage=storage, datareader=DefaultDataReader(), ) mc.validate(dg) assert mc._storage is not None @@ -146,21 +146,21 @@ def test_marker_collection_storage(tmp_path: Path) -> None: features = storage.list_features() assert len(features) == 3 feature_md5 = list(features.keys())[0] - t_feature = storage.read_df(feature_md5=feature_md5) + t_feature = storage.read_features(feature_md5=feature_md5) fname = "gmd_schaefer100x7_mean" t_data = out[fname]["VBM_GM"]["data"] # type: ignore cols = out[fname]["VBM_GM"]["columns"] # type: ignore assert_array_equal(t_feature[cols].values, t_data) # type: ignore feature_md5 = list(features.keys())[1] - t_feature = storage.read_df(feature_md5=feature_md5) + t_feature = storage.read_features(feature_md5=feature_md5) fname = "gmd_schaefer100x7_std" t_data = out[fname]["VBM_GM"]["data"] # type: ignore cols = out[fname]["VBM_GM"]["columns"] # type: ignore assert_array_equal(t_feature[cols].values, t_data) # type: ignore feature_md5 = list(features.keys())[2] - t_feature = storage.read_df(feature_md5=feature_md5) + t_feature = storage.read_features(feature_md5=feature_md5) fname = "gmd_schaefer100x7_trim_mean90" t_data = out[fname]["VBM_GM"]["data"] # type: ignore cols = out[fname]["VBM_GM"]["columns"] # type: ignore -- 2.52.0 From 199fe20856ba20068ccf48b5e5e14eace6b1b8ff Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:33:14 +0200 Subject: [PATCH 65/83] chore: black formatting --- junifer/markers/tests/test_collection.py | 4 +++- junifer/storage/tests/test_sqlite.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/junifer/markers/tests/test_collection.py b/junifer/markers/tests/test_collection.py index 8105f13a8..22dc3c9e7 100644 --- a/junifer/markers/tests/test_collection.py +++ b/junifer/markers/tests/test_collection.py @@ -125,7 +125,9 @@ def test_marker_collection_storage(tmp_path: Path) -> None: uri = tmp_path / "test_marker_collection_storage.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) mc = MarkerCollection( - markers=markers, storage=storage, datareader=DefaultDataReader(), + markers=markers, + storage=storage, + datareader=DefaultDataReader(), ) mc.validate(dg) assert mc._storage is not None diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index bec4a7a57..aefd85141 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -429,7 +429,10 @@ def test_store_matrix(tmp_path: Path) -> None: # Store table storage.store_matrix( - data=data, meta=meta, row_names=row_names, col_names=col_names, + data=data, + meta=meta, + row_names=row_names, + col_names=col_names, ) stored_names = [f"{i}~{j}" for i in row_names for j in col_names] -- 2.52.0 From 9dcd8006fd70c416e327bae6bb939c8310a95934 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 12:45:38 +0200 Subject: [PATCH 66/83] chore: update latest.inc --- docs/changes/latest.inc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changes/latest.inc b/docs/changes/latest.inc index b418273f5..4d83e2d8b 100644 --- a/docs/changes/latest.inc +++ b/docs/changes/latest.inc @@ -51,13 +51,17 @@ Enhancements - Implement SphereAggregation marker (by `Fede Raimondo`_). -- Implement PIOP1 and PIOP2 AOMIC datasets and refactor AOMICID1000 slightly +- Implement PIOP1 and PIOP2 AOMIC datasets and refactor AOMICID1000 slightly (:gh: `94` by `Leonard Sasse`_) - Implement a JuselessDataladCamCANVBM datagrabber class (:gh: `99` by `Leonard Sasse`_) - Implement IXI CAT output datagrabber for juseless (:gh: `48` by `Leonard Sasse`_). +- Upgrade storage interface for storage-like objects (:gh: `84` by `Synchon Mandal`_). + +- Add missing type annotations (:gh: `74` by `Synchon Mandal`_). + Bugs ~~~~ -- 2.52.0 From ecaa7ff07f91c3a93bee28ffc87f71f66e6cf27f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 14:55:54 +0200 Subject: [PATCH 67/83] update: replace validate_input with validate for MarkerCollection() marker check --- junifer/markers/collection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/junifer/markers/collection.py b/junifer/markers/collection.py index 3ff8746fc..88c9304c8 100644 --- a/junifer/markers/collection.py +++ b/junifer/markers/collection.py @@ -119,7 +119,8 @@ class MarkerCollection: for marker in self._markers: logger.info(f"Validating Marker: {marker.name}") - m_data = marker.validate_input(t_data) + # Validate marker + m_data = marker.validate(input=t_data) logger.info(f"Marker output type: {m_data}") if self._storage is not None: logger.info(f"Validating storage for {marker.name}") -- 2.52.0 From 5942a624aa18be98ca989d89d6079752aa57e348 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 14 Oct 2022 15:11:30 +0200 Subject: [PATCH 68/83] update: revert renaming of read_df() to read_features() in storage --- junifer/storage/base.py | 4 ++-- junifer/storage/sqlite.py | 6 +++--- junifer/storage/tests/test_sqlite.py | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index c99a5cd2a..2c98c1c60 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -97,12 +97,12 @@ class BaseFeatureStorage(ABC): ) @abstractmethod - def read_features( + def read_df( self, feature_name: Optional[str] = None, feature_md5: Optional[bool] = None, ) -> pd.DataFrame: - """Read feature from the storage. + """Read feature into a pandas DataFrame. Parameters ---------- diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index f4a76c318..6bd469ff1 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -287,12 +287,12 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): out = meta_df.to_dict(orient="index") return out - def read_features( + def read_df( self, feature_name: Optional[str] = None, feature_md5: Optional[str] = None, ) -> pd.DataFrame: - """Implement feature reading from the storage. + """Implement feature reading into a pandas DataFrame. Either one of `feature_name` or `feature_md5` needs to be specified. @@ -635,7 +635,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # TODO: Fix this, needs that read_feature sets the index # properly table_name = f"meta_{meta_md5}" - t_df = in_storage.read_features(feature_md5=meta_md5) + t_df = in_storage.read_df(feature_md5=meta_md5) # Save data out_storage._save_upsert(t_df, table_name, if_exists="nocheck") diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index aefd85141..c9eb73c34 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -263,7 +263,7 @@ def test_upsert_invalid_option(tmp_path: Path) -> None: # TODO: can the tests be separated? -def test_store_df_and_read_features(tmp_path: Path) -> None: +def test_store_df_and_read_df(tmp_path: Path) -> None: """Test storing dataframe and reading of stored table into dataframe. Parameters @@ -307,20 +307,20 @@ def test_store_df_and_read_features(tmp_path: Path) -> None: assert table_name.replace("meta_", "") in features # Check for missing feature with pytest.raises(ValueError, match="not found"): - storage.read_features("wrong_md5") + storage.read_df("wrong_md5") # Check for missing feature to fetch with pytest.raises(ValueError, match="least one"): - storage.read_features() + storage.read_df() # Check for multiple features to fetch with pytest.raises(ValueError, match="Only one"): - storage.read_features("wrong_md5", "wrong_name") + storage.read_df("wrong_md5", "wrong_name") # Get MD5 hash of features feature_md5 = list(features.keys())[0] # Check for key assert "fcname" == features[feature_md5]["name"] # Read into dataframes - read_df1 = storage.read_features(feature_md5=feature_md5) - read_df2 = storage.read_features(feature_name="fcname") + read_df1 = storage.read_df(feature_md5=feature_md5) + read_df2 = storage.read_df(feature_name="fcname") # Check if dataframes are equal assert_frame_equal(read_df1, read_df2) assert_frame_equal(read_df1, to_store) @@ -441,7 +441,7 @@ def test_store_matrix(tmp_path: Path) -> None: feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_features(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=feature_md5) assert read_df.shape == (1, 12) assert_array_equal(read_df.values[0], data.flatten()) assert list(read_df.columns) == stored_names @@ -457,7 +457,7 @@ def test_store_matrix(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_features(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=feature_md5) assert list(read_df.columns) == stored_names with pytest.raises(ValueError, match="Invalid kind"): @@ -500,7 +500,7 @@ def test_store_matrix(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_features(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.triu_indices(n=data.shape[0])][None, :] @@ -527,7 +527,7 @@ def test_store_matrix(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_features(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.triu_indices(n=data.shape[0], k=1)][None, :] @@ -559,7 +559,7 @@ def test_store_matrix(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_features(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.tril_indices(n=data.shape[0])][None, :] @@ -586,7 +586,7 @@ def test_store_matrix(tmp_path: Path) -> None: features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] - read_df = storage.read_features(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=feature_md5) assert list(read_df.columns) == stored_names assert_array_equal( read_df.values, data[np.tril_indices(n=data.shape[0], k=-1)][None, :] -- 2.52.0 From d0b8d902b58647ba1dc40911acff7a79ac990437 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 10:49:14 +0200 Subject: [PATCH 69/83] update: make BaseFeaturestorage and SQLiteFeatureStorage validate() raise error --- junifer/storage/base.py | 12 ++++++------ junifer/storage/sqlite.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 2c98c1c60..028908046 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -52,7 +52,7 @@ class BaseFeatureStorage(ABC): return meta @abstractmethod - def validate_input(self, input_: List[str]) -> bool: + def validate(self, input_: List[str]) -> None: """Validate the input to the pipeline step. Parameters @@ -60,14 +60,14 @@ class BaseFeatureStorage(ABC): input_ : list The input to the pipeline step. - Returns - ------- - bool - Whether the `input` is valid or not. + Raises + ------ + ValueError + If the `input_` is invalid. """ raise_error( - msg="Concrete classes need to implement validate_input().", + msg="Concrete classes need to implement validate().", klass=NotImplementedError, ) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 6bd469ff1..6788a2121 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -237,7 +237,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # Store dataframe self.store_df(df=data_df, meta=meta) - def validate_input(self, input_: List[str]) -> bool: + def validate(self, input_: List[str]) -> None: """Implement input validation. Parameters @@ -245,17 +245,23 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): input_ : list of str The input to the pipeline step. - Returns - ------- - bool - Whether the `input` is valid or not. + Raises + ------ + ValueError + If the validation fails. """ # Convert input to list if not isinstance(input_, list): input_ = [input_] - return all(x in self._valid_inputs for x in input_) + if not all(x in self._valid_inputs for x in input_): + raise_error( + msg=( + "At least one of the input is not present in " + f"{self._valid_inputs}", + ) + ) def list_features( self, return_df: bool = False -- 2.52.0 From 45fbeb567de61afc03b4663e094e1401a3f44cbe Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 10:53:47 +0200 Subject: [PATCH 70/83] update: rename validate_input() to validate() for storage-like objects in collection.py --- junifer/markers/collection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/junifer/markers/collection.py b/junifer/markers/collection.py index 88c9304c8..210c5aead 100644 --- a/junifer/markers/collection.py +++ b/junifer/markers/collection.py @@ -122,6 +122,8 @@ class MarkerCollection: # Validate marker m_data = marker.validate(input=t_data) logger.info(f"Marker output type: {m_data}") + # Check storage for the marker if self._storage is not None: logger.info(f"Validating storage for {marker.name}") - self._storage.validate_input(m_data) + # Validate storage + self._storage.validate(input=m_data) -- 2.52.0 From 3406ebff6c4dd50c69e0552a4a9a1396dd3a8942 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 11:03:29 +0200 Subject: [PATCH 71/83] update: revert renaming of read_df() to read_features() in test_storage_base.py --- junifer/storage/tests/test_storage_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index 8c4c484ee..3194523c3 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -28,8 +28,8 @@ def test_BaseFeatureStorage() -> None: def list_features(self): super().list_features() - def read_features(self, feature_name=None, feature_md5=None): - super().read_features( + def read_df(self, feature_name=None, feature_md5=None): + super().read_df( feature_name=feature_name, feature_md5=feature_md5, ) @@ -53,7 +53,7 @@ def test_BaseFeatureStorage() -> None: st.list_features() with pytest.raises(NotImplementedError): - st.read_features(None) + st.read_df(None) with pytest.raises(NotImplementedError): st.store_metadata(None) -- 2.52.0 From 13000c43761745fc0632dea37a757cf33ee3ec77 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 11:03:44 +0200 Subject: [PATCH 72/83] update: revert renaming of read_df() to read_features() in test_collection.py --- junifer/markers/tests/test_collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/markers/tests/test_collection.py b/junifer/markers/tests/test_collection.py index 22dc3c9e7..ea3cd9f76 100644 --- a/junifer/markers/tests/test_collection.py +++ b/junifer/markers/tests/test_collection.py @@ -148,21 +148,21 @@ def test_marker_collection_storage(tmp_path: Path) -> None: features = storage.list_features() assert len(features) == 3 feature_md5 = list(features.keys())[0] - t_feature = storage.read_features(feature_md5=feature_md5) + t_feature = storage.read_df(feature_md5=feature_md5) fname = "gmd_schaefer100x7_mean" t_data = out[fname]["VBM_GM"]["data"] # type: ignore cols = out[fname]["VBM_GM"]["columns"] # type: ignore assert_array_equal(t_feature[cols].values, t_data) # type: ignore feature_md5 = list(features.keys())[1] - t_feature = storage.read_features(feature_md5=feature_md5) + t_feature = storage.read_df(feature_md5=feature_md5) fname = "gmd_schaefer100x7_std" t_data = out[fname]["VBM_GM"]["data"] # type: ignore cols = out[fname]["VBM_GM"]["columns"] # type: ignore assert_array_equal(t_feature[cols].values, t_data) # type: ignore feature_md5 = list(features.keys())[2] - t_feature = storage.read_features(feature_md5=feature_md5) + t_feature = storage.read_df(feature_md5=feature_md5) fname = "gmd_schaefer100x7_trim_mean90" t_data = out[fname]["VBM_GM"]["data"] # type: ignore cols = out[fname]["VBM_GM"]["columns"] # type: ignore -- 2.52.0 From b9d6ebfaf2d3a9cb8564db03613f16267b4f668b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 11:04:09 +0200 Subject: [PATCH 73/83] update: revert renaming of read_df() to read_features() in run_junifer_julearn.py --- examples/run_junifer_julearn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_junifer_julearn.py b/examples/run_junifer_julearn.py index 6ae1298ff..1b58bdc30 100644 --- a/examples/run_junifer_julearn.py +++ b/examples/run_junifer_julearn.py @@ -84,7 +84,7 @@ with tempfile.TemporaryDirectory() as tmpdir: collect(storage) db = SQLiteFeatureStorage(uri=storage["uri"], single_output=True) - df_vbm = db.read_features(feature_name="VBM_GM_Schaefer200x17_Mean") + df_vbm = db.read_df(feature_name="VBM_GM_Schaefer200x17_Mean") oasis_subjects = [x[0] for x in df_vbm.index] df_vbm.index = oasis_subjects -- 2.52.0 From 4a81a3cdfd0879a7849f1d09f7d06ef539892e4b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 11:04:26 +0200 Subject: [PATCH 74/83] update: revert renaming of read_df() to read_features() in run_ets_rss_marker.py --- examples/run_ets_rss_marker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_ets_rss_marker.py b/examples/run_ets_rss_marker.py index 4cc82cb71..79b2b1006 100644 --- a/examples/run_ets_rss_marker.py +++ b/examples/run_ets_rss_marker.py @@ -71,7 +71,7 @@ with tempfile.TemporaryDirectory() as tmpdir: single_output=True, # as we ran collect, we have single output now ) # Read extracted features - df_vbm = db.read_features(feature_name="BOLD_Schaefer100x17_RSSETS") + df_vbm = db.read_df(feature_name="BOLD_Schaefer100x17_RSSETS") ############################################################################### # Now we take a look at the dataframe -- 2.52.0 From e3905c84182982401aea39bcce371f9264ba7f1a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 11:14:54 +0200 Subject: [PATCH 75/83] update: revert renaming of validate() to validate_input() in test_storage_base.py --- junifer/storage/tests/test_storage_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index 3194523c3..5ba86c46b 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -22,8 +22,8 @@ def test_BaseFeatureStorage() -> None: def __init__(self, uri, single_output=False): super().__init__(uri, single_output=single_output) - def validate_input(self, input): - super().validate_input(input) + def validate(self, input): + super().validate(input) def list_features(self): super().list_features() -- 2.52.0 From 2ff9094ecdcedbf914fd6d2f77d486ec345c8e85 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 11:15:25 +0200 Subject: [PATCH 76/83] fix: correct argument name for storage.validate() --- junifer/markers/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/markers/collection.py b/junifer/markers/collection.py index 210c5aead..4fe2a0f33 100644 --- a/junifer/markers/collection.py +++ b/junifer/markers/collection.py @@ -126,4 +126,4 @@ class MarkerCollection: if self._storage is not None: logger.info(f"Validating storage for {marker.name}") # Validate storage - self._storage.validate(input=m_data) + self._storage.validate(input_=m_data) -- 2.52.0 From 76dccd7008ae96aea0a8b8ba5cb71841c411eb6d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 11:34:59 +0200 Subject: [PATCH 77/83] fix: rename validate_input() to validate() in test_storage_base.py --- junifer/storage/tests/test_storage_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index 5ba86c46b..ecab7d72e 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -47,7 +47,7 @@ def test_BaseFeatureStorage() -> None: assert st.single_output is True with pytest.raises(NotImplementedError): - st.validate_input(None) + st.validate(None) with pytest.raises(NotImplementedError): st.list_features() -- 2.52.0 From d7e5e298c5542d465b9f024b01fb014600fff40d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 15:37:25 +0200 Subject: [PATCH 78/83] update: add case for handling invalid argument values to storage.store() --- junifer/storage/base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 028908046..32ef2d655 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -152,6 +152,11 @@ class BaseFeatureStorage(ABC): **kwargs The keyword arguments. + Raises + ------ + ValueError + If `kind` is invalid. + """ if kind == "matrix": self.store_matrix(**kwargs) @@ -159,6 +164,8 @@ class BaseFeatureStorage(ABC): self.store_timeseries(**kwargs) elif kind == "table": self.store_table(**kwargs) + else: + raise ValueError(f"I don't know how to store {kind}") def store_df(self, **kwargs) -> None: """Store pandas DataFrame. -- 2.52.0 From 9b91c3bc5841cf477324863e70e4c9b369af028b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 15:39:17 +0200 Subject: [PATCH 79/83] update: add tests for storage.store() --- junifer/storage/tests/test_storage_base.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index ecab7d72e..c7efe5995 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -61,4 +61,16 @@ def test_BaseFeatureStorage() -> None: with pytest.raises(NotImplementedError): st.collect() + with pytest.raises(NotImplementedError): + st.store(kind="matrix") + + with pytest.raises(NotImplementedError): + st.store(kind="timeseries") + + with pytest.raises(NotImplementedError): + st.store(kind="table") + + with pytest.raises(ValueError): + st.store(kind="lego") + assert st.uri == "/tmp" -- 2.52.0 From b73889c1bcd41d02632ba81dd34eb96979316a16 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 16:34:30 +0200 Subject: [PATCH 80/83] update: add new argument storage_types for BaseFeatureStorage and adapt its tests --- junifer/storage/base.py | 10 +++++++++- junifer/storage/tests/test_storage_base.py | 11 ++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 32ef2d655..a50a372be 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -24,16 +24,24 @@ class BaseFeatureStorage(ABC): ---------- uri : str or pathlib.Path The path to the storage. + storage_types : str or list of str + The available storage types for the class. single_output : bool, optional Whether to have single output (default False). """ def __init__( - self, uri: Union[str, Path], single_output: bool = False + self, + uri: Union[str, Path], + storage_types: Union[List[str], str], + single_output: bool = False, ) -> None: """Initialize the class.""" self.uri = uri + if not isinstance(storage_types, list): + storage_types = [storage_types] + self._valid_inputs = storage_types self.single_output = single_output def get_meta(self) -> Dict: diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index c7efe5995..34bd0a643 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -12,18 +12,23 @@ from junifer.storage.base import BaseFeatureStorage def test_BaseFeatureStorage_abstractness() -> None: """Test BaseFeatureStorage is abstract base class.""" with pytest.raises(TypeError, match=r"abstract"): - BaseFeatureStorage(uri="/tmp") # type: ignore + BaseFeatureStorage(uri="/tmp", storage_types=["matrix"]) def test_BaseFeatureStorage() -> None: """Test BaseFeatureStorage.""" # Create concrete class class MyFeatureStorage(BaseFeatureStorage): - def __init__(self, uri, single_output=False): - super().__init__(uri, single_output=single_output) def validate(self, input): super().validate(input) + def __init__(self, uri, single_output=False): + storage_types = ["matrix"] + super().__init__( + uri=uri, + storage_types=storage_types, + single_output=single_output, + ) def list_features(self): super().list_features() -- 2.52.0 From aa1d90b2c79b2765d45803ff3e4d8c5a7a1e8e1a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 16:35:49 +0200 Subject: [PATCH 81/83] update: move storage.validate() logic to BaseFeatureStorage --- junifer/storage/base.py | 13 +++++++------ junifer/storage/tests/test_storage_base.py | 9 +++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index a50a372be..16513ba7c 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -59,13 +59,12 @@ class BaseFeatureStorage(ABC): } return meta - @abstractmethod def validate(self, input_: List[str]) -> None: """Validate the input to the pipeline step. Parameters ---------- - input_ : list + input_ : list of str The input to the pipeline step. Raises @@ -74,10 +73,12 @@ class BaseFeatureStorage(ABC): If the `input_` is invalid. """ - raise_error( - msg="Concrete classes need to implement validate().", - klass=NotImplementedError, - ) + if not any(x in input_ for x in self._valid_inputs): + raise_error( + "Input does not have the required data." + f"\t Input: {input}" + f"\t Required (any of): {self._valid_inputs}" + ) @abstractmethod def list_features( diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index 34bd0a643..d3c0d4a38 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -20,8 +20,6 @@ def test_BaseFeatureStorage() -> None: # Create concrete class class MyFeatureStorage(BaseFeatureStorage): - def validate(self, input): - super().validate(input) def __init__(self, uri, single_output=False): storage_types = ["matrix"] super().__init__( @@ -51,8 +49,11 @@ def test_BaseFeatureStorage() -> None: st = MyFeatureStorage(uri="/tmp", single_output=True) assert st.single_output is True - with pytest.raises(NotImplementedError): - st.validate(None) + # Check validate with valid argument + st.validate(input_=["matrix"]) + # Check validate with invalid argument + with pytest.raises(ValueError): + st.validate(input_=["table"]) with pytest.raises(NotImplementedError): st.list_features() -- 2.52.0 From 36485a2d57dcb1bca02ac1d6b3ef56daf92719e1 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 16:36:32 +0200 Subject: [PATCH 82/83] update: adapt SQLiteFeatureStorage to new changes of BaseFeatureStorage --- junifer/storage/sqlite.py | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 6788a2121..80a3fb1cd 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -77,11 +77,16 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): "does not exist, creating now." ) uri.parent.mkdir(parents=True, exist_ok=True) - super().__init__(uri=uri, single_output=single_output, **kwargs) + # Available storage kinds + storage_types = ["table", "timeseries", "matrix"] + super().__init__( + uri=uri, + storage_types=storage_types, + single_output=single_output, + **kwargs, + ) # Set upsert self._upsert = upsert - # Define the available storage kinds available - self._valid_inputs = ["table", "timeseries", "matrix"] def get_engine(self, meta: Optional[Dict] = None) -> "Engine": """Get engine. @@ -237,32 +242,6 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # Store dataframe self.store_df(df=data_df, meta=meta) - def validate(self, input_: List[str]) -> None: - """Implement input validation. - - Parameters - ---------- - input_ : list of str - The input to the pipeline step. - - Raises - ------ - ValueError - If the validation fails. - - """ - # Convert input to list - if not isinstance(input_, list): - input_ = [input_] - - if not all(x in self._valid_inputs for x in input_): - raise_error( - msg=( - "At least one of the input is not present in " - f"{self._valid_inputs}", - ) - ) - def list_features( self, return_df: bool = False ) -> Union[Dict[str, Dict], pd.DataFrame]: -- 2.52.0 From d4e8990f09dbde5978ec2b121a8539fb373dae85 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 17 Oct 2022 16:37:06 +0200 Subject: [PATCH 83/83] chore: add comments and docstrings to test_storage_base.py --- junifer/storage/tests/test_storage_base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py index d3c0d4a38..cfe9a0068 100644 --- a/junifer/storage/tests/test_storage_base.py +++ b/junifer/storage/tests/test_storage_base.py @@ -16,9 +16,10 @@ def test_BaseFeatureStorage_abstractness() -> None: def test_BaseFeatureStorage() -> None: - """Test BaseFeatureStorage.""" + """Test proper subclassing of BaseFeatureStorage.""" # Create concrete class class MyFeatureStorage(BaseFeatureStorage): + """Implement concrete class.""" def __init__(self, uri, single_output=False): storage_types = ["matrix"] @@ -43,9 +44,10 @@ def test_BaseFeatureStorage() -> None: def collect(self): return super().collect() + # Check single_output is False st = MyFeatureStorage(uri="/tmp") assert st.single_output is False - + # Check single_output is True st = MyFeatureStorage(uri="/tmp", single_output=True) assert st.single_output is True -- 2.52.0