From 00bd37034997aa4c001de94bbf20a8e918ffd842 Mon Sep 17 00:00:00 2001 From: LeSasse Date: Tue, 13 Sep 2022 18:02:41 +0200 Subject: [PATCH 01/14] example: how to junifer and julearn all in one go --- examples/run_junifer_julearn.py | 91 +++++++++++++++++++++++++++++++++ pyproject.toml | 3 +- 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 examples/run_junifer_julearn.py diff --git a/examples/run_junifer_julearn.py b/examples/run_junifer_julearn.py new file mode 100644 index 000000000..3b7cb54a4 --- /dev/null +++ b/examples/run_junifer_julearn.py @@ -0,0 +1,91 @@ +""" +Run junifer and julearn. +======================== + +This example uses a ParcelAggregation marker to compute the mean of each parcel +using the Schaefer atlas (100 rois, 7 Yeo networks) for a 3D nifti to extract +some features for machine learning using julearn to predict some other data. + +Authors: Leonard Sasse, Sami Hamdan, Nicolas Nieto, Synchon Mandal + +License: BSD 3 clause +""" + +import nilearn + +import junifer.testing.registry # noqa +import pandas as pd +from julearn import run_cross_validation +import tempfile +from junifer.api import run, collect +from junifer.storage.sqlite import SQLiteFeatureStorage +from junifer.utils import configure_logging + +############################################################################### +# Set the logging level to info to see extra information +configure_logging(level="INFO") + + +############################################################################### +# Define the markers you want + +# register(step='datagrabber', name='Oasis', klass=OasisVBMTestingDatagrabber) +marker_dicts = [ + { + "name": "Schaefer100x17_TrimMean80", + "kind": "ParcelAggregation", + "atlas": "Schaefer100x17", + "method": "trim_mean", + "method_params": {"proportiontocut": 0.2} + }, + { + "name": "Schaefer200x17_Mean", + "kind": "ParcelAggregation", + "atlas": "Schaefer200x17", + "method": "mean" + } +] + +y = 'age' +confound = 'sex' +oasis_dataset = nilearn.datasets.fetch_oasis_vbm() +age = oasis_dataset.ext_vars[y][:10] +sex = ( + pd.Series(oasis_dataset.ext_vars['mf'][:10]) + .map(lambda x: 1 if x == 'F' else 0) + .values +) +with tempfile.TemporaryDirectory() as tmpdir: + + storage = {'kind': 'SQLiteFeatureStorage', + 'uri': f'{tmpdir}/test.db' + } + run( + workdir="/tmp", + datagrabber={'kind': 'OasisVBMTestingDatagrabber'}, + markers=marker_dicts, + storage=storage, + ) + + collect(storage) + + db = SQLiteFeatureStorage(uri=storage['uri'], single_output=True) + + df_vbm = db.read_df(feature_name='VBM_GM_Schaefer200x17_Mean') + oasis_subjects = [x[0] for x in df_vbm.index] + df_vbm.index = oasis_subjects + X = list(df_vbm.columns) + df_vbm[y] = age + df_vbm[confound] = sex + + scores = run_cross_validation( + X=X, confounds=confound, y=y, + data=df_vbm, problem_type='regression', + model='ridge', cv=3, + preprocess_X=['zscore', 'remove_confound'] + ) + print(scores) + + +############################################################################### +# Check the results diff --git a/pyproject.toml b/pyproject.toml index 38d20190e..e2303bd7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ docs = [ "sphinx-rtd-theme>=1.0.0,<1.1", "sphinx-multiversion>=0.2.4,<0.3", "numpydoc>=1.4.0,<1.5", + "julearn=0.2.5" ] ################ @@ -82,4 +83,4 @@ write_to = "junifer/_version.py" [tool.black] line-length = 79 -target-version = ["py38"] \ No newline at end of file +target-version = ["py38"] -- 2.52.0 From d706e9b0b9f7e5d741a7f7db92174b5d476f9c7d Mon Sep 17 00:00:00 2001 From: LeSasse Date: Tue, 13 Sep 2022 18:13:05 +0200 Subject: [PATCH 02/14] example: how to junifer and julearn all in one go --- examples/run_junifer_julearn.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/examples/run_junifer_julearn.py b/examples/run_junifer_julearn.py index 3b7cb54a4..e3e279a31 100644 --- a/examples/run_junifer_julearn.py +++ b/examples/run_junifer_julearn.py @@ -46,8 +46,16 @@ marker_dicts = [ } ] + +############################################################################### +# define target and confounds for julearn machine learning y = 'age' confound = 'sex' + + +############################################################################### +# Load the VBM phenotype data for machine learning data: +# - Fetch the Oasis dataset oasis_dataset = nilearn.datasets.fetch_oasis_vbm() age = oasis_dataset.ext_vars[y][:10] sex = ( @@ -55,11 +63,16 @@ sex = ( .map(lambda x: 1 if x == 'F' else 0) .values ) + + +############################################################################### +# create a temporary directory for junifer feature extraction with tempfile.TemporaryDirectory() as tmpdir: storage = {'kind': 'SQLiteFeatureStorage', 'uri': f'{tmpdir}/test.db' } + # run the defined junifer feature extraction pipeline run( workdir="/tmp", datagrabber={'kind': 'OasisVBMTestingDatagrabber'}, @@ -67,8 +80,9 @@ with tempfile.TemporaryDirectory() as tmpdir: storage=storage, ) + # read in extracted features and add confounds and targets + # for julearn run cross validation collect(storage) - db = SQLiteFeatureStorage(uri=storage['uri'], single_output=True) df_vbm = db.read_df(feature_name='VBM_GM_Schaefer200x17_Mean') -- 2.52.0 From 4440a4c0475c9faf3b66ec80a65f1331febffb50 Mon Sep 17 00:00:00 2001 From: LeSasse Date: Tue, 13 Sep 2022 18:17:04 +0200 Subject: [PATCH 03/14] update: added entry to latest --- docs/changes/latest.inc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changes/latest.inc b/docs/changes/latest.inc index 855a8a6d5..420a7862e 100644 --- a/docs/changes/latest.inc +++ b/docs/changes/latest.inc @@ -24,6 +24,9 @@ Enhancements - Created a the repository based on the mockup by by `Fede Raimondo`_. - Added comments to datalad grabber and changed to use datalad-clone instead of datalad-install (:gh: `55` by `Benjamin Poldrack`_). +- Added an example how to use junifer and julearn (:gh: `40` by `Leonard Sasse_`, + `Nicolas Nieto_`, and `Sami Hamdan_`) in one pipeline to extract features + and do machine learning Bugs ~~~~ -- 2.52.0 From 66cfd4bb864b7103065b9aaa36a404d6de15f1b7 Mon Sep 17 00:00:00 2001 From: Sami Hamdan Date: Wed, 14 Sep 2022 10:12:33 +0200 Subject: [PATCH 04/14] update: fix julearn dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e2303bd7e..e425e4e5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ docs = [ "sphinx-rtd-theme>=1.0.0,<1.1", "sphinx-multiversion>=0.2.4,<0.3", "numpydoc>=1.4.0,<1.5", - "julearn=0.2.5" + "julearn==0.2.5" ] ################ -- 2.52.0 From 6d24dc2a3419e10aef8073668c15b8d9827f18b3 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 10:18:30 +0200 Subject: [PATCH 05/14] Testing SQLite matrix2d storage WIP --- junifer/storage/base.py | 11 ++++ junifer/storage/sqlite.py | 84 +++++++++++++++++++++++++--- junifer/storage/tests/test_sqlite.py | 58 +++++++++++++++++++ 3 files changed, 145 insertions(+), 8 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 46a2f0e40..9984ef47d 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -156,6 +156,8 @@ class BaseFeatureStorage(ABC): meta: Dict, col_names: Optional[Iterable[str]] = None, row_names: Optional[Iterable[str]] = None, + kind: Optional[str] = 'full', + diagonal: bool = True ) -> None: """Store 2D matrix. @@ -168,6 +170,15 @@ class BaseFeatureStorage(ABC): The column names (default None). row_names : list of tuple of str, optional The row names (default None). + kind : str, optional + The kind of matrix: + - 'triu: store upper triangular only. + - 'tril': store lower triangular. + - 'full': full matrix (default 'full'). + diagonal : bool, optional + Whether to store the diagonal (default True). + If kind == 'full', setting this to false will raise + an error """ raise_error( diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 9522713b0..7840fb22c 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -4,9 +4,11 @@ # Synchon Mandal # License: AGPL +from itertools import product from pathlib import Path from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union +import numpy as np import pandas as pd from pandas.core.base import NoNewAttributesMixin from pandas.io.sql import pandasSQL_builder @@ -77,7 +79,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): uri.parent.mkdir(parents=True, exist_ok=True) super().__init__(uri=uri, single_output=single_output, **kwargs) self._upsert = upsert - self._valid_inputs = ["table", "timeseries"] + self._valid_inputs = ["table", "timeseries", "matrix"] def get_engine(self, meta: Optional[Dict] = None) -> "Engine": """Get engine. @@ -403,8 +405,10 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): self, data, meta: Dict, - col_names: Optional[Iterable[str]] = None, - rows_col_name: Optional[str] = None, + col_names: Optional[List[str]] = None, + row_names: Optional[List[str]] = None, + kind: Optional[str] = 'full', + diagonal: bool = True ) -> None: """Implement 2D matrix storing. @@ -415,17 +419,81 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): The metadata as a dictionary. col_names : list or tuple of str, optional The column names (default None). - rows_col_name : str, optional + row_names : str, optional The column name to use in case number of rows greater than 1. If None and number of rows greater than 1, then the name will be "index" (default None). - + kind : str, optional + The kind of matrix: + - 'triu: store upper triangular only. + - 'tril': store lower triangular. + - 'full': full matrix (default 'full'). + diagonal : bool, optional + Whether to store the diagonal (default True). + If kind == 'full', setting this to false will raise + an error """ - # Same as store_2d, but order is important - raise_error( - msg="store_matrix2d() not implemented", klass=NotImplementedError + if diagonal is False and kind not in ['triu', 'tril']: + raise_error( + msg='Diagonal cannot be False if kind is not full', + klass=ValueError + ) + + if kind in ['triu', 'tril']: + if data.shape[0] != data.shape[1]: + raise_error( + 'Cannot store a non-square matrix as a triangular matrix', + klass=ValueError + ) + + n_rows = 1 + # Convert element metadata to index + idx = element_to_index( + meta=meta, n_rows=n_rows, rows_col_name=None ) + if kind == 'triu': + k = 0 if diagonal is True else 1 + data_idx = np.triu_indices(data.shape[0], k=k) + elif kind == 'tril': + k = 0 if diagonal is True else -1 + data_idx = np.tril(data, k=k) + elif kind == 'full': + data_idx = ( + np.repeat(np.arange(data.shape[0]), data.shape[1]), + np.tile(np.arange(data.shape[1]), data.shape[0]) + ) + else: + raise_error( + msg=f'Invalid kind {kind}', + klass=ValueError + ) + if row_names is None: + row_names = [f"r{i}" for i in range(data.shape[0])] + elif len(row_names) != data.shape[0]: + raise_error( + msg='Number of row names does not match number of rows', + klass=ValueError + ) + + if col_names is None: + col_names = [f"c{i}" for i in range(data.shape[1])] + elif len(col_names) != data.shape[1]: + raise_error( + msg='Number of column names does not match number of columns', + klass=ValueError + ) + + flat_data = data[data_idx] + columns = [f"{row_names[i]}~{col_names[j]}" + for i, j in zip(data_idx[0], data_idx[1])] + # Prepare new dataframe + data_df = pd.DataFrame( + flat_data[None, :], columns=columns, index=idx + ) # type: ignore + # Store dataframe + self.store_df(df=data_df, meta=meta) + # TODO: complete type annotations def store_table( self, diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index af5c4a3e0..155fefcee 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import List, Union import numpy as np +from numpy.testing import assert_array_equal import pandas as pd import pytest from pandas.testing import assert_frame_equal @@ -405,6 +406,63 @@ def test_store_table(tmp_path: Path) -> None: assert_frame_equal(df_new, c_df_new) +def test_store_matrix2d(tmp_path: Path) -> None: + uri = tmp_path / "test_store_table.db" + storage = SQLiteFeatureStorage(uri=uri, single_output=True) + # Metadata to store + meta = {"element": "test", "version": "0.0.1", "marker": {"name": "fc"}} + + # Store 4 x 3 full matrix + data = np.array( + [[1, 2, 3], [11, 22, 33], [111, 222, 333], [1111, 2222, 3333]] + ) + row_names = ["row1", "row2", "row3", "row4"] + col_names = ["col1", "col2", "col3"] + + # Store table + storage.store_matrix2d( + data, meta, row_names=row_names, col_names=col_names + ) + + stored_names = [f"{i}~{j}" for i in row_names for j in col_names] + + features = storage.list_features() + feature_md5 = list(features.keys())[0] + assert "fc" == features[feature_md5]["name"] + + read_df = storage.read_df(feature_md5=feature_md5) + assert read_df.shape == (1, 12) + assert_array_equal(read_df.values[0], data.flatten()) + assert list(read_df.columns) == stored_names + # Store without row and column names + uri = tmp_path / "test_store_table_nonames.db" + storage = SQLiteFeatureStorage(uri=uri, single_output=True) + storage.store_matrix2d(data, meta) + stored_names = [f"r{i}~c{j}" for i in range(data.shape[0]) + for j in range(data.shape[1])] + features = storage.list_features() + feature_md5 = list(features.keys())[0] + assert "fc" == features[feature_md5]["name"] + read_df = storage.read_df(feature_md5=feature_md5) + assert list(read_df.columns) == stored_names + + with pytest.raises(ValueError, match='Invalid kind'): + storage.store_matrix2d(data, meta, kind='wrong') + + with pytest.raises(ValueError, match='non-square'): + storage.store_matrix2d(data, meta, kind='triu') + + with pytest.raises(ValueError, match='cannot be False'): + storage.store_matrix2d(data, meta, kind='full', diagonal=False) + + # Store upper triangular matrix + data = np.array( + [[1, 2, 3], [11, 22, 33], [111, 222, 333]] + ) + row_names = ["row1", "row2", "row3"] + col_names = ["col1", "col2", "col3"] + + # TODO: can the test be parametrized? def test_store_multiple_output(tmp_path: Path): """Test storing using single_output=False. -- 2.52.0 From 4a9b9e9e00a3bacdda397b836fcd6c791bae5387 Mon Sep 17 00:00:00 2001 From: Sami Hamdan Date: Wed, 14 Sep 2022 10:36:34 +0200 Subject: [PATCH 06/14] update: formatting --- examples/run_junifer_julearn.py | 63 ++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/examples/run_junifer_julearn.py b/examples/run_junifer_julearn.py index e3e279a31..b326a539d 100644 --- a/examples/run_junifer_julearn.py +++ b/examples/run_junifer_julearn.py @@ -11,16 +11,18 @@ Authors: Leonard Sasse, Sami Hamdan, Nicolas Nieto, Synchon Mandal License: BSD 3 clause """ -import nilearn +import tempfile -import junifer.testing.registry # noqa +import nilearn import pandas as pd from julearn import run_cross_validation -import tempfile -from junifer.api import run, collect + +import junifer.testing.registry # noqa +from junifer.api import collect, run from junifer.storage.sqlite import SQLiteFeatureStorage from junifer.utils import configure_logging + ############################################################################### # Set the logging level to info to see extra information configure_logging(level="INFO") @@ -36,21 +38,21 @@ marker_dicts = [ "kind": "ParcelAggregation", "atlas": "Schaefer100x17", "method": "trim_mean", - "method_params": {"proportiontocut": 0.2} + "method_params": {"proportiontocut": 0.2}, }, { "name": "Schaefer200x17_Mean", "kind": "ParcelAggregation", "atlas": "Schaefer200x17", - "method": "mean" - } + "method": "mean", + }, ] ############################################################################### # define target and confounds for julearn machine learning -y = 'age' -confound = 'sex' +y = "age" +confound = "sex" ############################################################################### @@ -59,8 +61,8 @@ confound = 'sex' oasis_dataset = nilearn.datasets.fetch_oasis_vbm() age = oasis_dataset.ext_vars[y][:10] sex = ( - pd.Series(oasis_dataset.ext_vars['mf'][:10]) - .map(lambda x: 1 if x == 'F' else 0) + pd.Series(oasis_dataset.ext_vars["mf"][:10]) + .map(lambda x: 1 if x == "F" else 0) .values ) @@ -69,13 +71,11 @@ sex = ( # create a temporary directory for junifer feature extraction with tempfile.TemporaryDirectory() as tmpdir: - storage = {'kind': 'SQLiteFeatureStorage', - 'uri': f'{tmpdir}/test.db' - } + storage = {"kind": "SQLiteFeatureStorage", "uri": f"{tmpdir}/test.db"} # run the defined junifer feature extraction pipeline run( workdir="/tmp", - datagrabber={'kind': 'OasisVBMTestingDatagrabber'}, + datagrabber={"kind": "OasisVBMTestingDatagrabber"}, markers=marker_dicts, storage=storage, ) @@ -83,23 +83,28 @@ with tempfile.TemporaryDirectory() as tmpdir: # read in extracted features and add confounds and targets # for julearn run cross validation collect(storage) - db = SQLiteFeatureStorage(uri=storage['uri'], single_output=True) + db = SQLiteFeatureStorage(uri=storage["uri"], single_output=True) - df_vbm = db.read_df(feature_name='VBM_GM_Schaefer200x17_Mean') + df_vbm = db.read_df(feature_name="VBM_GM_Schaefer200x17_Mean") oasis_subjects = [x[0] for x in df_vbm.index] df_vbm.index = oasis_subjects - X = list(df_vbm.columns) - df_vbm[y] = age - df_vbm[confound] = sex - - scores = run_cross_validation( - X=X, confounds=confound, y=y, - data=df_vbm, problem_type='regression', - model='ridge', cv=3, - preprocess_X=['zscore', 'remove_confound'] - ) - print(scores) ############################################################################### -# Check the results +# using julearn for machine learning +# we predict the age given our vbm features and sex as a confound +X = list(df_vbm.columns) +df_vbm[y] = age +df_vbm[confound] = sex + +scores = run_cross_validation( + X=X, + confounds=confound, + y=y, + data=df_vbm, + problem_type="regression", + model="ridge", + cv=3, + preprocess_X=["zscore", "remove_confound"], +) +print(scores) -- 2.52.0 From 0f0275698b98e88b8fef42cb583fb1f2b5effbdb Mon Sep 17 00:00:00 2001 From: Sami Hamdan Date: Wed, 14 Sep 2022 10:56:42 +0200 Subject: [PATCH 07/14] update: text --- examples/run_junifer_julearn.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/run_junifer_julearn.py b/examples/run_junifer_julearn.py index b326a539d..1b58bdc30 100644 --- a/examples/run_junifer_julearn.py +++ b/examples/run_junifer_julearn.py @@ -24,14 +24,13 @@ from junifer.utils import configure_logging ############################################################################### -# Set the logging level to info to see extra information +# Set the logging level to info to see extra information: configure_logging(level="INFO") ############################################################################### -# Define the markers you want +# Define the markers you want: -# register(step='datagrabber', name='Oasis', klass=OasisVBMTestingDatagrabber) marker_dicts = [ { "name": "Schaefer100x17_TrimMean80", @@ -50,7 +49,7 @@ marker_dicts = [ ############################################################################### -# define target and confounds for julearn machine learning +# Define target and confounds for julearn machine learning: y = "age" confound = "sex" @@ -68,7 +67,7 @@ sex = ( ############################################################################### -# create a temporary directory for junifer feature extraction +# Create a temporary directory for junifer feature extraction: with tempfile.TemporaryDirectory() as tmpdir: storage = {"kind": "SQLiteFeatureStorage", "uri": f"{tmpdir}/test.db"} @@ -91,8 +90,8 @@ with tempfile.TemporaryDirectory() as tmpdir: ############################################################################### -# using julearn for machine learning -# we predict the age given our vbm features and sex as a confound +# Using julearn for machine learning: +# We predict the age given our vbm features and sex as a confound. X = list(df_vbm.columns) df_vbm[y] = age df_vbm[confound] = sex @@ -108,3 +107,9 @@ scores = run_cross_validation( preprocess_X=["zscore", "remove_confound"], ) print(scores) + +############################################################################### +# Interpretation of results: +# Doing machine learning with only 10 datapoints is not meaningful. +# This explains the big variation in scores +# for different cross-validation folds. -- 2.52.0 From b0e01970b3834f46d8154e3a93d2270782d1b390 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 11:09:01 +0200 Subject: [PATCH 08/14] SQLIte Matrix 2d storage done --- junifer/storage/sqlite.py | 2 +- junifer/storage/tests/test_sqlite.py | 128 ++++++++++++++++++++++++--- 2 files changed, 118 insertions(+), 12 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 7840fb22c..f0b92bb8e 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -457,7 +457,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): data_idx = np.triu_indices(data.shape[0], k=k) elif kind == 'tril': k = 0 if diagonal is True else -1 - data_idx = np.tril(data, k=k) + data_idx = np.tril_indices(data.shape[0], k=k) elif kind == 'full': data_idx = ( np.repeat(np.arange(data.shape[0]), data.shape[1]), diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index 155fefcee..03a6f4729 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -438,29 +438,135 @@ def test_store_matrix2d(tmp_path: Path) -> None: uri = tmp_path / "test_store_table_nonames.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) storage.store_matrix2d(data, meta) - stored_names = [f"r{i}~c{j}" for i in range(data.shape[0]) - for j in range(data.shape[1])] + stored_names = [ + f"r{i}~c{j}" + for i in range(data.shape[0]) + for j in range(data.shape[1]) + ] features = storage.list_features() feature_md5 = list(features.keys())[0] assert "fc" == features[feature_md5]["name"] read_df = storage.read_df(feature_md5=feature_md5) assert list(read_df.columns) == stored_names - with pytest.raises(ValueError, match='Invalid kind'): - storage.store_matrix2d(data, meta, kind='wrong') + with pytest.raises(ValueError, match="Invalid kind"): + storage.store_matrix2d(data, meta, kind="wrong") - with pytest.raises(ValueError, match='non-square'): - storage.store_matrix2d(data, meta, kind='triu') + with pytest.raises(ValueError, match="non-square"): + storage.store_matrix2d(data, meta, kind="triu") - with pytest.raises(ValueError, match='cannot be False'): - storage.store_matrix2d(data, meta, kind='full', diagonal=False) + with pytest.raises(ValueError, match="cannot be False"): + storage.store_matrix2d(data, meta, kind="full", diagonal=False) # Store upper triangular matrix - data = np.array( - [[1, 2, 3], [11, 22, 33], [111, 222, 333]] - ) + data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]]) row_names = ["row1", "row2", "row3"] col_names = ["col1", "col2", "col3"] + uri = tmp_path / "test_store_table_triu.db" + storage = SQLiteFeatureStorage(uri=uri, single_output=True) + storage.store_matrix2d( + data, meta, kind="triu", row_names=row_names, col_names=col_names + ) + + stored_names = [ + "row1~col1", + "row1~col2", + "row1~col3", + "row2~col2", + "row2~col3", + "row3~col3", + ] + + features = storage.list_features() + feature_md5 = list(features.keys())[0] + assert "fc" == features[feature_md5]["name"] + read_df = storage.read_df(feature_md5=feature_md5) + assert list(read_df.columns) == stored_names + assert_array_equal( + read_df.values, data[np.triu_indices(n=data.shape[0])][None, :] + ) + + # Store upper triangular matrix without diagnoal + uri = tmp_path / "test_store_table_triu_nodiagonal.db" + storage = SQLiteFeatureStorage(uri=uri, single_output=True) + storage.store_matrix2d( + data, + meta, + kind="triu", + row_names=row_names, + col_names=col_names, + diagonal=False, + ) + + stored_names = [ + "row1~col2", + "row1~col3", + "row2~col3", + ] + + features = storage.list_features() + feature_md5 = list(features.keys())[0] + assert "fc" == features[feature_md5]["name"] + read_df = storage.read_df(feature_md5=feature_md5) + assert list(read_df.columns) == stored_names + assert_array_equal( + read_df.values, data[np.triu_indices(n=data.shape[0], k=1)][None, :] + ) + + # Store lower triangular matrix + data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]]) + row_names = ["row1", "row2", "row3"] + col_names = ["col1", "col2", "col3"] + uri = tmp_path / "test_store_table_tril.db" + storage = SQLiteFeatureStorage(uri=uri, single_output=True) + storage.store_matrix2d( + data, meta, kind="tril", row_names=row_names, col_names=col_names + ) + + stored_names = [ + "row1~col1", + "row2~col1", + "row2~col2", + "row3~col1", + "row3~col2", + "row3~col3", + ] + + features = storage.list_features() + feature_md5 = list(features.keys())[0] + assert "fc" == features[feature_md5]["name"] + read_df = storage.read_df(feature_md5=feature_md5) + assert list(read_df.columns) == stored_names + assert_array_equal( + read_df.values, data[np.tril_indices(n=data.shape[0])][None, :] + ) + + # Store lower triangular matrix without diagnoal + uri = tmp_path / "test_store_table_tril_nodiagonal.db" + storage = SQLiteFeatureStorage(uri=uri, single_output=True) + storage.store_matrix2d( + data, + meta, + kind="tril", + row_names=row_names, + col_names=col_names, + diagonal=False, + ) + + stored_names = [ + "row2~col1", + "row3~col1", + "row3~col2", + ] + + features = storage.list_features() + feature_md5 = list(features.keys())[0] + assert "fc" == features[feature_md5]["name"] + read_df = storage.read_df(feature_md5=feature_md5) + assert list(read_df.columns) == stored_names + assert_array_equal( + read_df.values, data[np.tril_indices(n=data.shape[0], k=-1)][None, :] + ) # TODO: can the test be parametrized? -- 2.52.0 From dbb302144a7e70f5a8049614a3d18b11900fa26c Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 11:10:32 +0200 Subject: [PATCH 09/14] Changes --- docs/changes/latest.inc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changes/latest.inc b/docs/changes/latest.inc index 855a8a6d5..3bc473eea 100644 --- a/docs/changes/latest.inc +++ b/docs/changes/latest.inc @@ -22,9 +22,12 @@ Enhancements - Implemented SPM Auditory testing datagrabber X (:gh:`52` by `Fede Raimondo`_). - Created a the repository based on the mockup by by `Fede Raimondo`_. + - Added comments to datalad grabber and changed to use datalad-clone instead of datalad-install (:gh: `55` by `Benjamin Poldrack`_). +- Implement matrix storage in SQliteFeatureStorage (:gh:`42` by `Fede Raimondo`_). + Bugs ~~~~ -- 2.52.0 From d5e5c3264a985d2aca05a1040d18bf3481f13df1 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 11:11:42 +0200 Subject: [PATCH 10/14] damn flake --- junifer/storage/sqlite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index f0b92bb8e..d783ccb7b 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -4,7 +4,6 @@ # Synchon Mandal # License: AGPL -from itertools import product from pathlib import Path from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union -- 2.52.0 From 58d2f70ff29300215af547a01437a9bd48caacd8 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 11:14:20 +0200 Subject: [PATCH 11/14] damn flake again --- junifer/storage/tests/test_sqlite.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index 03a6f4729..80ca456d3 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -407,6 +407,14 @@ def test_store_table(tmp_path: Path) -> None: def test_store_matrix2d(tmp_path: Path) -> None: + """Test 2D Matrix store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ uri = tmp_path / "test_store_table.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) # Metadata to store -- 2.52.0 From 790d2df17f74993207daf0efb238d38c04c99d4c Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 11:16:18 +0200 Subject: [PATCH 12/14] damn flake again and again --- junifer/storage/tests/test_sqlite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index 80ca456d3..d7eb6d786 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -494,7 +494,7 @@ def test_store_matrix2d(tmp_path: Path) -> None: read_df.values, data[np.triu_indices(n=data.shape[0])][None, :] ) - # Store upper triangular matrix without diagnoal + # Store upper triangular matrix without diagonal uri = tmp_path / "test_store_table_triu_nodiagonal.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) storage.store_matrix2d( @@ -549,7 +549,7 @@ def test_store_matrix2d(tmp_path: Path) -> None: read_df.values, data[np.tril_indices(n=data.shape[0])][None, :] ) - # Store lower triangular matrix without diagnoal + # Store lower triangular matrix without diagonal uri = tmp_path / "test_store_table_tril_nodiagonal.db" storage = SQLiteFeatureStorage(uri=uri, single_output=True) storage.store_matrix2d( -- 2.52.0 From 11235ed4e6cc3dfbd2125eee3403858bca8414a7 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 11:18:30 +0200 Subject: [PATCH 13/14] Black formatting --- junifer/storage/base.py | 4 ++-- junifer/storage/sqlite.py | 47 ++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 9984ef47d..a78e12d1c 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -156,8 +156,8 @@ class BaseFeatureStorage(ABC): meta: Dict, col_names: Optional[Iterable[str]] = None, row_names: Optional[Iterable[str]] = None, - kind: Optional[str] = 'full', - diagonal: bool = True + kind: Optional[str] = "full", + diagonal: bool = True, ) -> None: """Store 2D matrix. diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index d783ccb7b..2d4ee7626 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -406,8 +406,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): meta: Dict, col_names: Optional[List[str]] = None, row_names: Optional[List[str]] = None, - kind: Optional[str] = 'full', - diagonal: bool = True + kind: Optional[str] = "full", + diagonal: bool = True, ) -> None: """Implement 2D matrix storing. @@ -432,60 +432,57 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): If kind == 'full', setting this to false will raise an error """ - if diagonal is False and kind not in ['triu', 'tril']: + if diagonal is False and kind not in ["triu", "tril"]: raise_error( - msg='Diagonal cannot be False if kind is not full', - klass=ValueError + msg="Diagonal cannot be False if kind is not full", + klass=ValueError, ) - if kind in ['triu', 'tril']: + if kind in ["triu", "tril"]: if data.shape[0] != data.shape[1]: raise_error( - 'Cannot store a non-square matrix as a triangular matrix', - klass=ValueError + "Cannot store a non-square matrix as a triangular matrix", + klass=ValueError, ) n_rows = 1 # Convert element metadata to index - idx = element_to_index( - meta=meta, n_rows=n_rows, rows_col_name=None - ) + idx = element_to_index(meta=meta, n_rows=n_rows, rows_col_name=None) - if kind == 'triu': + if kind == "triu": k = 0 if diagonal is True else 1 data_idx = np.triu_indices(data.shape[0], k=k) - elif kind == 'tril': + elif kind == "tril": k = 0 if diagonal is True else -1 data_idx = np.tril_indices(data.shape[0], k=k) - elif kind == 'full': + elif kind == "full": data_idx = ( np.repeat(np.arange(data.shape[0]), data.shape[1]), - np.tile(np.arange(data.shape[1]), data.shape[0]) + np.tile(np.arange(data.shape[1]), data.shape[0]), ) else: - raise_error( - msg=f'Invalid kind {kind}', - klass=ValueError - ) + raise_error(msg=f"Invalid kind {kind}", klass=ValueError) if row_names is None: row_names = [f"r{i}" for i in range(data.shape[0])] elif len(row_names) != data.shape[0]: raise_error( - msg='Number of row names does not match number of rows', - klass=ValueError + msg="Number of row names does not match number of rows", + klass=ValueError, ) if col_names is None: col_names = [f"c{i}" for i in range(data.shape[1])] elif len(col_names) != data.shape[1]: raise_error( - msg='Number of column names does not match number of columns', - klass=ValueError + msg="Number of column names does not match number of columns", + klass=ValueError, ) flat_data = data[data_idx] - columns = [f"{row_names[i]}~{col_names[j]}" - for i, j in zip(data_idx[0], data_idx[1])] + columns = [ + f"{row_names[i]}~{col_names[j]}" + for i, j in zip(data_idx[0], data_idx[1]) + ] # Prepare new dataframe data_df = pd.DataFrame( flat_data[None, :], columns=columns, index=idx -- 2.52.0 From c91b7eee817a7a0e229b40534842859c68477d23 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 14 Sep 2022 11:21:13 +0200 Subject: [PATCH 14/14] missing docstring --- junifer/storage/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index a78e12d1c..f9bda0886 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -172,7 +172,7 @@ class BaseFeatureStorage(ABC): The row names (default None). kind : str, optional The kind of matrix: - - 'triu: store upper triangular only. + - 'triu': store upper triangular only. - 'tril': store lower triangular. - 'full': full matrix (default 'full'). diagonal : bool, optional -- 2.52.0