From bf5418bc3f117607ab7ad3e09b2a8d750c88361c Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:19:11 +0100
Subject: [PATCH 01/51] refactor: rename table to vector in BaseFeatureStorage

---
 junifer/storage/base.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/junifer/storage/base.py b/junifer/storage/base.py
index 38b2f5a5b..9df6eb356 100644
--- a/junifer/storage/base.py
+++ b/junifer/storage/base.py
@@ -152,7 +152,7 @@ class BaseFeatureStorage(ABC):
 
         Parameters
         ----------
-        kind : {"matrix", "timeseries", "table"}
+        kind : {"matrix", "timeseries", "vector"}
             The storage kind.
         **kwargs
             The keyword arguments.
@@ -179,8 +179,8 @@ class BaseFeatureStorage(ABC):
             self.store_timeseries(
                 meta_md5=meta_md5, element=t_element, **kwargs
             )
-        elif kind == "table":
-            self.store_table(meta_md5=meta_md5, element=t_element, **kwargs)
+        elif kind == "vector":
+            self.store_vector(meta_md5=meta_md5, element=t_element, **kwargs)
 
     def store_matrix(
         self,
@@ -225,7 +225,7 @@ class BaseFeatureStorage(ABC):
             klass=NotImplementedError,
         )
 
-    def store_table(
+    def store_vector(
         self,
         meta_md5: str,
         element: Dict,
@@ -233,7 +233,7 @@ class BaseFeatureStorage(ABC):
         columns: Optional[Iterable[str]] = None,
         rows_col_name: Optional[str] = None,
     ) -> None:
-        """Store table.
+        """Store vector.
 
         Parameters
         ----------
@@ -242,7 +242,7 @@ class BaseFeatureStorage(ABC):
         element : dict
             The element as a dictionary.
         data : numpy.ndarray or list
-            The table data to store.
+            The vector data to store.
         columns : list or tuple of str, optional
             The columns (default None).
         rows_col_name : str, optional
@@ -251,7 +251,7 @@ class BaseFeatureStorage(ABC):
             "index" (default None).
         """
         raise_error(
-            msg="Concrete classes need to implement store_table().",
+            msg="Concrete classes need to implement store_vector().",
             klass=NotImplementedError,
         )
 
-- 
2.52.0


From 21374e5c3b4002b47dc800fa7b6156a44381b707 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:20:13 +0100
Subject: [PATCH 02/51] refactor: rename table to vector in BaseFeatureStorage
 tests

---
 junifer/storage/tests/test_storage_base.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py
index d59b27086..a7b1b75ac 100644
--- a/junifer/storage/tests/test_storage_base.py
+++ b/junifer/storage/tests/test_storage_base.py
@@ -21,8 +21,8 @@ def test_BaseFeatureStorage() -> None:
     class MyFeatureStorage(BaseFeatureStorage):
         """Implement concrete class."""
 
-        def __init__(self, uri, single_output=False):
-            storage_types = ["matrix", "table", "timeseries"]
+        def __init__(self, uri, single_output=True):
+            storage_types = ["matrix", "vector", "timeseries"]
             super().__init__(
                 uri=uri,
                 storage_types=storage_types,
@@ -30,7 +30,7 @@ def test_BaseFeatureStorage() -> None:
             )
 
         def get_valid_inputs(self):
-            return ["matrix", "table", "timeseries"]
+            return ["matrix", "vector", "timeseries"]
 
         def list_features(self):
             super().list_features()
@@ -88,7 +88,7 @@ def test_BaseFeatureStorage() -> None:
         st.store(kind="timeseries", meta=meta)
 
     with pytest.raises(NotImplementedError):
-        st.store(kind="table", meta=meta)
+        st.store(kind="vector", meta=meta)
 
     with pytest.raises(ValueError):
         st.store(kind="lego", meta=meta)
-- 
2.52.0


From b4d6ef4ae4fb0c0d90c8e539264b20eb72139706 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:21:43 +0100
Subject: [PATCH 03/51] chore: improve docstrings for BaseFeatureStorage

---
 junifer/storage/base.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/junifer/storage/base.py b/junifer/storage/base.py
index 9df6eb356..a8baa9fdf 100644
--- a/junifer/storage/base.py
+++ b/junifer/storage/base.py
@@ -57,8 +57,9 @@ class BaseFeatureStorage(ABC):
         Returns
         -------
         list of str
-            The list of storage types that can be used as input for this "
-            "storage.
+            The list of storage types that can be used as input for this
+            storage interface.
+
         """
         raise_error(
             msg="Concrete classes need to implement get_valid_inputs().",
@@ -93,9 +94,9 @@ class BaseFeatureStorage(ABC):
         Returns
         -------
         dict
-            List of features in the storage. The keys are the feature names to
-            be used in read_features() and the values are the metadata of each
-            feature.
+            List of features in the storage. The keys are the feature MD5 to
+            be used in :meth:`junifer.storage.BaseFeatureStorage.read_df`
+            and the values are the metadata of each feature.
 
         """
         raise_error(
@@ -207,7 +208,7 @@ class BaseFeatureStorage(ABC):
         row_names : str, optional
             The column name to use in case number of rows greater than 1.
             If None and number of rows greater than 1, then the name will be
-            "index" (default None).
+            "idx" (default None).
         matrix_kind : str, optional
             The kind of matrix:
 
@@ -248,7 +249,7 @@ class BaseFeatureStorage(ABC):
         rows_col_name : str, optional
             The column name to use in case number of rows greater than 1.
             If None and number of rows greater than 1, then the name will be
-            "index" (default None).
+            "idx" (default None).
         """
         raise_error(
             msg="Concrete classes need to implement store_vector().",
@@ -262,7 +263,7 @@ class BaseFeatureStorage(ABC):
         data: np.ndarray,
         columns: Optional[Iterable[str]] = None,
     ) -> None:
-        """Implement timeseries storing.
+        """Store timeseries.
 
         Parameters
         ----------
-- 
2.52.0


From 02d23830b29c4b901a09af78b948cbd1205083cf Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:22:12 +0100
Subject: [PATCH 04/51] chore: improve type annotations for BaseFeatureStorage

---
 junifer/storage/base.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/junifer/storage/base.py b/junifer/storage/base.py
index a8baa9fdf..94134bcc6 100644
--- a/junifer/storage/base.py
+++ b/junifer/storage/base.py
@@ -6,7 +6,7 @@
 
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Dict, Iterable, List, Optional, Union
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -88,7 +88,7 @@ class BaseFeatureStorage(ABC):
             )
 
     @abstractmethod
-    def list_features(self) -> Dict:
+    def list_features(self) -> Dict[str, Dict[str, Any]]:
         """List the features in the storage.
 
         Returns
@@ -108,7 +108,7 @@ class BaseFeatureStorage(ABC):
     def read_df(
         self,
         feature_name: Optional[str] = None,
-        feature_md5: Optional[bool] = None,
+        feature_md5: Optional[str] = None,
     ) -> pd.DataFrame:
         """Read feature into a pandas DataFrame.
 
-- 
2.52.0


From aad7c3394ac7b2c757c1d345a89c359b6f7d72fc Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:22:35 +0100
Subject: [PATCH 05/51] chore: improve tests for BaseFeatureStorage

---
 junifer/storage/tests/test_storage_base.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/junifer/storage/tests/test_storage_base.py b/junifer/storage/tests/test_storage_base.py
index a7b1b75ac..d4c7c5093 100644
--- a/junifer/storage/tests/test_storage_base.py
+++ b/junifer/storage/tests/test_storage_base.py
@@ -12,7 +12,9 @@ from junifer.storage.base import BaseFeatureStorage
 def test_BaseFeatureStorage_abstractness() -> None:
     """Test BaseFeatureStorage is abstract base class."""
     with pytest.raises(TypeError, match=r"abstract"):
-        BaseFeatureStorage(uri="/tm", storage_types=["matrix"])  # type: ignore
+        BaseFeatureStorage(
+            uri="/tmp", storage_types=["matrix"]  # type: ignore
+        )
 
 
 def test_BaseFeatureStorage() -> None:
@@ -48,7 +50,7 @@ def test_BaseFeatureStorage() -> None:
             return super().collect()
 
     # Check single_output is False
-    st = MyFeatureStorage(uri="/tmp")
+    st = MyFeatureStorage(uri="/tmp", single_output=False)
     assert st.single_output is False
     # Check single_output is True
     st = MyFeatureStorage(uri="/tmp", single_output=True)
@@ -66,11 +68,9 @@ def test_BaseFeatureStorage() -> None:
     with pytest.raises(NotImplementedError):
         st.read_df(None)
 
-    element = {"subject": "test"}
-    dependencies = ["numpy"]
     meta = {
-        "element": element,
-        "dependencies": dependencies,
+        "element": {"subject": "test"},
+        "dependencies": ["numpy"],
         "marker": {"name": "fc"},
         "type": "BOLD",
     }
-- 
2.52.0


From 29d25a38fac22f0f03e51a7c2d39d3aecb146e01 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:23:19 +0100
Subject: [PATCH 06/51] docs: rename table to vector in
 understanding/storage.rst

---
 docs/understanding/storage.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/understanding/storage.rst b/docs/understanding/storage.rst
index 0547b8bee..5c8d68495 100644
--- a/docs/understanding/storage.rst
+++ b/docs/understanding/storage.rst
@@ -15,10 +15,10 @@ that object else they are kept in memory.
 Storage is meant to be used inside the datagrabber context but you can operate on them outside the context as long
 as the processed data is in the memory and the Python runtime has not garbage-collected it.
 
-The :ref:`Markers <marker>` are responsible for defining what *storage kind* (``matrix``, ``table``, ``timeseries``)
+The :ref:`Markers <marker>` are responsible for defining what *storage kind* (``matrix``, ``vector``, ``timeseries``)
 they support for which :ref:`data type <data_types>` by overriding its ``store`` method. The storage object in turn
 declares and provides implementation for specific *storage kind*. For example, :class:`junifer.storage.SQLiteFeatureStorage`
-supports saving ``matrix``, ``table`` and ``timeseries`` via ``store_matrix``, ``store_table`` and ``store_timeseries``
+supports saving ``matrix``, ``vector`` and ``timeseries`` via ``store_matrix``, ``store_vector`` and ``store_timeseries``
 methods respectively.
 
 For storage interfaces not supported by junifer yet, you can either make your own ``Storage`` by providing a concrete
@@ -42,15 +42,15 @@ Currently supported storage types
      - A 2D matrix with row and column names
      -  ``col_names``, ``row_names``, ``matrix_kind``, ``diagonal``
      -  :meth:`junifer.storage.BaseFeatureStorage.store_matrix`
-   * - ``table``
+   * - ``vector``
      - A vector of values with column names
      - ``columns``, ``row_names``
-     -  :meth:`junifer.storage.BaseFeatureStorage.store_table`
+     -  :meth:`junifer.storage.BaseFeatureStorage.store_vector`
    * - ``timeseries``
      - A 2D matrix of values with column names
      - ``columns``, ``row_names``
      -  :meth:`junifer.storage.BaseFeatureStorage.store_timeseries`
-  
+
 .. _storage_interfaces:
 
 Currently supported storage interfaces
-- 
2.52.0


From 18a5b8d6af8ca7e921fb1e36dffcfcf41ede451f Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:26:29 +0100
Subject: [PATCH 07/51] chore: improve docstrings for PandasBaseFeatureStorage

---
 junifer/storage/pandas_base.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index bc6b5983c..821e1eb49 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -46,8 +46,9 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         Returns
         -------
         list of str
-            The list of storage types that can be used as input for this "
-            "storage.
+            The list of storage types that can be used as input for this
+            storage interface.
+
         """
         return ["matrix", "table", "timeseries"]
 
@@ -86,8 +87,8 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         n_rows : int, optional
             Number of rows to create (default 1).
         rows_col_name: str, optional
-            The column name to use in case `n_rows` > 1. If None and
-            n_rows > 1, the name will be "idx" (default None).
+            The column name to use in case ``n_rows`` > 1. If None and
+            ``n_rows`` > 1, the name will be "idx" (default None).
 
         Returns
         -------
@@ -157,7 +158,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         rows_col_name : str, optional
             The column name to use in case number of rows greater than 1.
             If None and number of rows greater than 1, then the name will be
-            "index" (default None).
+            "idx" (default None).
 
         """
         n_rows = len(data)
-- 
2.52.0


From 194714f9472d806c71123afdc35fe1c6f45ecdea Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:27:02 +0100
Subject: [PATCH 08/51] update: use raise_error() in
 PandasBaseFeatureStorage.store_df()

---
 junifer/storage/pandas_base.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 821e1eb49..2f7180e37 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -11,6 +11,7 @@ from typing import Any, Dict, Iterable, List, Optional, Union
 import numpy as np
 import pandas as pd
 
+from ..utils import raise_error
 from .base import BaseFeatureStorage
 
 
@@ -133,7 +134,10 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             generated from the metadata.
 
         """
-        raise NotImplementedError("Implement in subclass.")
+        raise_error(
+            msg="Concrete classes need to implement store_df().",
+            klass=NotImplementedError,
+        )
 
     def _store_2d(
         self,
-- 
2.52.0


From c84dc7e62fdd70b390f4e053eb5fe9f12639afff Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:27:52 +0100
Subject: [PATCH 09/51] chore: proper usage of `type: ignore` in
 PandasBaseFeatureStorage

---
 junifer/storage/pandas_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 2f7180e37..b6b7f87b7 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -171,7 +171,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             element=element, n_rows=n_rows, rows_col_name=rows_col_name
         )
         # Prepare new dataframe
-        data_df = pd.DataFrame(  # type: ignore
+        data_df = pd.DataFrame(
             data, columns=columns, index=idx  # type: ignore
         )
         # Store dataframe
-- 
2.52.0


From a9833d1892ec214e852d2cfa41bb4fda395c4635 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:30:48 +0100
Subject: [PATCH 10/51] chore: improve docstrings for SQLiteFeatureStorage

---
 junifer/storage/sqlite.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py
index 99428da28..cd3c76a93 100644
--- a/junifer/storage/sqlite.py
+++ b/junifer/storage/sqlite.py
@@ -34,12 +34,12 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
     uri : str or pathlib.Path
         The path to the file to be used.
     single_output : bool, optional
-        If False, will create one file per element. The name
+        If False, will create one SQLite file per element. The name
         of the file will be prefixed with the respective element.
-        If True, will create only one file as specified in the `uri` and
-        store all the elements in the same file. This behaviour is only
-        suitable for non-parallel executions. SQLite does not support
-        concurrency (default True).
+        If True, will create only one SQLite file as specified in the
+        ``uri`` and store all the elements in the same file. This behaviour
+        is only suitable for non-parallel executions. SQLite does not
+        support concurrency (default True).
     upsert : {"ignore", "update"}, optional
         Upsert mode. If "ignore" is used, the existing elements are ignored.
         If "update", the existing elements are updated (default "update").
@@ -93,8 +93,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
 
         Parameters
         ----------
-        meta : dict, optional
-            The metadata as dictionary (default None).
+        element : dict, optional
+            The element as dictionary (default None).
 
         Returns
         -------
@@ -211,9 +211,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
         Returns
         -------
         dict
-            List of features in the storage. The keys are the feature names to
-            be used in read_features() and the values are the metadata of each
-            feature.
+            List of features in the storage. The keys are the feature MD5 to
+            be used in :meth:`junifer.storage.SQLiteFeatureStorage.read_df`
+            and the values are the metadata of each feature.
 
         """
         meta_df = pd.read_sql(
-- 
2.52.0


From f23b1642f7652645e607685e9d25f1a03a843413 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:36:07 +0100
Subject: [PATCH 11/51] update: make SQLiteFeatureStorage.collect() on single
 output error an IOError

---
 junifer/storage/sqlite.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py
index cd3c76a93..4a1b68bf6 100644
--- a/junifer/storage/sqlite.py
+++ b/junifer/storage/sqlite.py
@@ -510,7 +510,10 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
 
         """
         if self.single_output is True:
-            raise_error(msg="collect() is not implemented for single output.")
+            raise_error(
+                msg="collect() is not implemented for single output.",
+                klass=IOError,
+            )
         logger.info(
             "Collecting data from "
             f"{self.uri.parent}/*{self.uri.name}"  # type: ignore
-- 
2.52.0


From 70f13326a48eae0fa88c891e21426d86a1b68784 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:37:18 +0100
Subject: [PATCH 12/51] chore: update module docstring and test file names for
 SQLiteFeatureStorage.store_matrix() tests

---
 junifer/storage/tests/test_sqlite.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py
index 65b3e6ffc..d5a006b3b 100644
--- a/junifer/storage/tests/test_sqlite.py
+++ b/junifer/storage/tests/test_sqlite.py
@@ -1,4 +1,4 @@
-"""Provide tests for sqlite."""
+"""Provide tests for SQLite storage interface."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -453,7 +453,7 @@ def test_store_matrix(tmp_path: Path) -> None:
         The path to the test directory.
 
     """
-    uri = tmp_path / "test_store_table.sqlite"
+    uri = tmp_path / "test_store_matrix.sqlite"
     storage = SQLiteFeatureStorage(uri=uri)
     # Metadata to store
     element = {"subject": "test"}
@@ -478,7 +478,7 @@ def test_store_matrix(tmp_path: Path) -> None:
     row_names = ["row1", "row2", "row3", "row4"]
     col_names = ["col1", "col2", "col3"]
 
-    # Store table
+    # Store matrix
     storage.store_matrix(
         meta_md5=meta_md5,
         element=element_to_store,
@@ -497,13 +497,13 @@ def test_store_matrix(tmp_path: Path) -> None:
     assert_array_equal(read_df.values[0], data.flatten())
     assert list(read_df.columns) == stored_names
     # Store without row and column names
-    uri = tmp_path / "test_store_table_nonames.sqlite"
+    uri = tmp_path / "test_store_matrix_nonames.sqlite"
     storage = SQLiteFeatureStorage(uri=uri)
     # Store metadata
     storage.store_metadata(
         meta_md5=meta_md5, element=element_to_store, meta=meta_to_store
     )
-    # Store table
+    # Store matrix
     storage.store_matrix(
         meta_md5=meta_md5, element=element_to_store, data=data
     )
@@ -553,7 +553,7 @@ def test_store_matrix(tmp_path: Path) -> None:
     data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]])
     row_names = ["row1", "row2", "row3"]
     col_names = ["col1", "col2", "col3"]
-    uri = tmp_path / "test_store_table_triu.sqlite"
+    uri = tmp_path / "test_store_matrix_triu.sqlite"
     storage = SQLiteFeatureStorage(uri=uri)
     # Store metadata
     storage.store_metadata(
@@ -587,7 +587,7 @@ def test_store_matrix(tmp_path: Path) -> None:
     )
 
     # Store upper triangular matrix without diagonal
-    uri = tmp_path / "test_store_table_triu_nodiagonal.sqlite"
+    uri = tmp_path / "test_store_matrix_triu_nodiagonal.sqlite"
     storage = SQLiteFeatureStorage(uri=uri)
     # Store metadata
     storage.store_metadata(
@@ -622,7 +622,7 @@ def test_store_matrix(tmp_path: Path) -> None:
     data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]])
     row_names = ["row1", "row2", "row3"]
     col_names = ["col1", "col2", "col3"]
-    uri = tmp_path / "test_store_table_tril.sqlite"
+    uri = tmp_path / "test_store_matrix_tril.sqlite"
     storage = SQLiteFeatureStorage(uri=uri)
     # Store metadata
     storage.store_metadata(
@@ -656,7 +656,7 @@ def test_store_matrix(tmp_path: Path) -> None:
     )
 
     # Store lower triangular matrix without diagonal
-    uri = tmp_path / "test_store_table_tril_nodiagonal.sqlite"
+    uri = tmp_path / "test_store_matrix_tril_nodiagonal.sqlite"
     storage = SQLiteFeatureStorage(uri=uri)
     # Store metadata
     storage.store_metadata(
-- 
2.52.0


From d8ff471978c6e83ac83cc5f49e1b53f448307309 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:39:11 +0100
Subject: [PATCH 13/51] chore: update docstrings for storage.utils

---
 junifer/storage/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py
index 462d93bb6..dc3ad46a7 100644
--- a/junifer/storage/utils.py
+++ b/junifer/storage/utils.py
@@ -84,13 +84,13 @@ def process_meta(meta: Dict) -> Tuple[str, Dict, Dict]:
         The MD5 hash of the metadata.
     dict
         The processed metadata for storage.
-    tuple
+    dict
         The element.
 
     Raises
     ------
     ValueError
-        If `meta` is None or if it does not contain the key "element".
+        If ``meta`` is None or if it does not contain the key "element".
 
     """
     if meta is None:
-- 
2.52.0


From 0f161b15afef1aefd8c466c5e664ddec940bdb10 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:39:32 +0100
Subject: [PATCH 14/51] chore: improve type annotations for storage.utils

---
 junifer/storage/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py
index dc3ad46a7..05de657ba 100644
--- a/junifer/storage/utils.py
+++ b/junifer/storage/utils.py
@@ -98,7 +98,7 @@ def process_meta(meta: Dict) -> Tuple[str, Dict, Dict]:
     # Copy the metadata
     t_meta = meta.copy()
     # Remove key "element"
-    element = t_meta.pop("element", None)
+    element: Dict = t_meta.pop("element", None)
     if element is None:
         raise_error(msg="`meta` must contain the key 'element'")
     if "marker" not in t_meta:
-- 
2.52.0


From f6ec6b6a9c4018ea481aaedd6f872cca1d4c7c4a Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:39:50 +0100
Subject: [PATCH 15/51] update: make element_to_index() an instance method for
 PandasBaseFeatureStorage

---
 junifer/storage/pandas_base.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index b6b7f87b7..d9aeed690 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -75,9 +75,11 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         df.index.name = "meta_md5"
         return df
 
-    @staticmethod
     def element_to_index(
-        element: Dict, n_rows: int = 1, rows_col_name: Optional[str] = None
+        self,
+        element: Dict,
+        n_rows: int = 1,
+        rows_col_name: Optional[str] = None,
     ) -> pd.MultiIndex:
         """Convert the element metadata to index.
 
-- 
2.52.0


From d0a4b1f75f97eaccfb2d487395b6ea996e952621 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Tue, 31 Jan 2023 15:46:57 +0100
Subject: [PATCH 16/51] Revert "update: make element_to_index() an instance
 method for PandasBaseFeatureStorage"

This reverts commit f6ec6b6a9c4018ea481aaedd6f872cca1d4c7c4a.
---
 junifer/storage/pandas_base.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index d9aeed690..b6b7f87b7 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -75,11 +75,9 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         df.index.name = "meta_md5"
         return df
 
+    @staticmethod
     def element_to_index(
-        self,
-        element: Dict,
-        n_rows: int = 1,
-        rows_col_name: Optional[str] = None,
+        element: Dict, n_rows: int = 1, rows_col_name: Optional[str] = None
     ) -> pd.MultiIndex:
         """Convert the element metadata to index.
 
-- 
2.52.0


From 1a4012f988290851962203b08062843c79a9b523 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:18:28 +0100
Subject: [PATCH 17/51] update: improve extra index column addition for
 PandasBaseFeatureStorage.element_to_index()

---
 junifer/storage/pandas_base.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index b6b7f87b7..24df1da5d 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -96,19 +96,20 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         pandas.MultiIndex
             The index of the dataframe to store.
 
-        Raises
-        ------
-        ValueError
-            If `meta` does not contain the key "element".
-
         """
-        # Check rows_col_name
-        if rows_col_name is None:
-            rows_col_name = "idx"
-        elem_idx: Dict[Any, Any] = {
+        # Make mapping between element access keys and values
+        elem_idx: Dict[str, Iterable[str]] = {
             k: [v] * n_rows for k, v in element.items()
         }
-        elem_idx[rows_col_name] = np.arange(n_rows)
+
+        # Set rows_col_name if n_rows > 1
+        if n_rows > 1:
+            # Set rows_col_name if None
+            if rows_col_name is None:
+                rows_col_name = "idx"
+            # Set extra column for variable number of rows per element
+            elem_idx[rows_col_name] = np.arange(n_rows)
+
         # Create index
         index = pd.MultiIndex.from_frame(
             pd.DataFrame(elem_idx, index=range(n_rows))
-- 
2.52.0


From 4acb42c160dbfc8594dff4551129cff55d16cdfc Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:19:35 +0100
Subject: [PATCH 18/51] update: fix tests for
 PandasBaseFeatureStorage.element_to_index()

---
 junifer/storage/tests/test_pandas_base.py | 127 ++++++++++++++--------
 1 file changed, 80 insertions(+), 47 deletions(-)

diff --git a/junifer/storage/tests/test_pandas_base.py b/junifer/storage/tests/test_pandas_base.py
index f81dd4247..85a6e5999 100644
--- a/junifer/storage/tests/test_pandas_base.py
+++ b/junifer/storage/tests/test_pandas_base.py
@@ -9,62 +9,95 @@ from junifer.storage.pandas_base import PandasBaseFeatureStorage
 
 def test_element_to_index() -> None:
     """Test element to index."""
+    # First element
     element = {"foo": "bar"}
-    index = PandasBaseFeatureStorage.element_to_index(element)
-    assert index.names == ["foo", "idx"]
-    assert index.levels[0].name == "foo"
-    assert index.levels[0].values[0] == "bar"
-    assert all(x == "bar" for x in index.levels[0].values)
-    assert index.levels[0].values.shape == (1,)
-    assert index.levels[1].name == "idx"
-    assert all(x == i for i, x in enumerate(index.levels[1].values))
-    assert index.levels[1].values.shape == (1,)
 
-    index = PandasBaseFeatureStorage.element_to_index(element, n_rows=10)
-    assert index.names == ["foo", "idx"]
-    assert index.levels[0].name == "foo"
-    assert all(x == "bar" for x in index.levels[0].values)
-    assert index.levels[0].values.shape == (1,)
-
-    assert index.levels[1].name == "idx"
-    assert all(x == i for i, x in enumerate(index.levels[1].values))
-    assert index.levels[1].values.shape == (10,)
+    # First test; no extra column
+    index = PandasBaseFeatureStorage.element_to_index(element=element)
+    # Check index names
+    assert index.names == ["foo"]
+    # Check first index level name
+    assert index.levels[0].name == "foo"  # type: ignore
+    # Check first index level values
+    assert index.levels[0].values[0] == "bar"  # type: ignore
+    assert all(x == "bar" for x in index.levels[0].values)  # type: ignore
+    # Check first index level values shape
+    assert index.levels[0].values.shape == (1,)  # type: ignore
 
+    # Second test; add extra column
     index = PandasBaseFeatureStorage.element_to_index(
-        element, n_rows=1, rows_col_name="scan"
+        element=element, n_rows=10
     )
-    assert index.names == ["foo", "scan"]
-    assert index.levels[0].name == "foo"
-    assert index.levels[0].values[0] == "bar"
-    assert all(x == "bar" for x in index.levels[0].values)
-    assert index.levels[0].values.shape == (1,)
-    assert index.levels[1].name == "scan"
-    assert all(x == i for i, x in enumerate(index.levels[1].values))
-    assert index.levels[1].values.shape == (1,)
+    # Check index names
+    assert index.names == ["foo", "idx"]
+    # Check first index level name
+    assert index.levels[0].name == "foo"  # type: ignore
+    # Check first index level values
+    assert all(x == "bar" for x in index.levels[0].values)  # type: ignore
+    # Check first index level values shape
+    assert index.levels[0].values.shape == (1,)  # type: ignore
+    # Check second index level name
+    assert index.levels[1].name == "idx"  # type: ignore
+    # Check second index level values
+    assert all(x == i for i, x in enumerate(index.levels[1].values))  # type: ignore
+    # Check second index level values shape
+    assert index.levels[1].values.shape == (10,)  # type: ignore
 
+    # Third test; custom extra column name has no effect
     index = PandasBaseFeatureStorage.element_to_index(
-        element, n_rows=7, rows_col_name="scan"
+        element=element, n_rows=1, rows_col_name="scan"
     )
+    # Check index names
+    assert index.names == ["foo"]
+    # Check first index level name
+    assert index.levels[0].name == "foo"  # type: ignore
+    # Check first index level values
+    assert index.levels[0].values[0] == "bar"  # type: ignore
+    assert all(x == "bar" for x in index.levels[0].values)  # type: ignore
+    # Check first index level values shape
+    assert index.levels[0].values.shape == (1,)  # type: ignore
+
+    # Fourth test; custom extra column name has effect
+    index = PandasBaseFeatureStorage.element_to_index(
+        element=element, n_rows=7, rows_col_name="scan"
+    )
+    # Check index names
     assert index.names == ["foo", "scan"]
-    assert index.levels[0].name == "foo"
-    assert all(x == "bar" for x in index.levels[0].values)
-    assert index.levels[0].values.shape == (1,)
-
-    assert index.levels[1].name == "scan"
-    assert all(x == i for i, x in enumerate(index.levels[1].values))
-    assert index.levels[1].values.shape == (7,)
+    # Check first index level name
+    assert index.levels[0].name == "foo"  # type: ignore
+    # Check first index level values
+    assert all(x == "bar" for x in index.levels[0].values)  # type: ignore
+    # Check first index level values shape
+    assert index.levels[0].values.shape == (1,)  # type: ignore
+    # Check second index level name
+    assert index.levels[1].name == "scan"  # type: ignore
+    # Check second index level values
+    assert all(x == i for i, x in enumerate(index.levels[1].values))  # type: ignore
+    # Check second index level values shape
+    assert index.levels[1].values.shape == (7,)  # type: ignore
 
+    # Second element
     element = {"subject": "sub-01", "session": "ses-01"}
-    index = PandasBaseFeatureStorage.element_to_index(element, n_rows=10)
 
-    assert index.levels[0].name == "subject"
-    assert all(x == "sub-01" for x in index.levels[0].values)
-    assert index.levels[0].values.shape == (1,)
-
-    assert index.levels[1].name == "session"
-    assert all(x == "ses-01" for x in index.levels[1].values)
-    assert index.levels[1].values.shape == (1,)
-
-    assert index.levels[2].name == "idx"
-    assert all(x == i for i, x in enumerate(index.levels[2].values))
-    assert index.levels[2].values.shape == (10,)
+    # Fifth test; default name for extra column and multi-level element access
+    index = PandasBaseFeatureStorage.element_to_index(
+        element=element, n_rows=10
+    )
+    # Check first index level name
+    assert index.levels[0].name == "subject"  # type: ignore
+    # Check first index level values
+    assert all(x == "sub-01" for x in index.levels[0].values)  # type: ignore
+    # Check first index level values shape
+    assert index.levels[0].values.shape == (1,)  # type: ignore
+    # Check second index level name
+    assert index.levels[1].name == "session"  # type: ignore
+    # Check second index level values
+    assert all(x == "ses-01" for x in index.levels[1].values)  # type: ignore
+    # Check second index level values shape
+    assert index.levels[1].values.shape == (1,)  # type: ignore
+    # Check third index level name
+    assert index.levels[2].name == "idx"  # type: ignore
+    # Check third index level values
+    assert all(x == i for i, x in enumerate(index.levels[2].values))  # type: ignore
+    # Check third index level values shape
+    assert index.levels[2].values.shape == (10,)  # type: ignore
-- 
2.52.0


From 3bfbd7626c176cf0d46cc76935d08b879bb315c0 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:32:58 +0100
Subject: [PATCH 19/51] update: replace columns and rows_col_names parameters
 with col_names for BaseFeatureStorage.store_vector()

---
 junifer/storage/base.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/junifer/storage/base.py b/junifer/storage/base.py
index 94134bcc6..ea4c58e62 100644
--- a/junifer/storage/base.py
+++ b/junifer/storage/base.py
@@ -231,8 +231,7 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: Dict,
         data: Union[np.ndarray, List],
-        columns: Optional[Iterable[str]] = None,
-        rows_col_name: Optional[str] = None,
+        col_names: Optional[Iterable[str]] = None,
     ) -> None:
         """Store vector.
 
@@ -244,12 +243,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray or list
             The vector data to store.
-        columns : list or tuple of str, optional
-            The columns (default None).
-        rows_col_name : str, optional
-            The column name to use in case number of rows greater than 1.
-            If None and number of rows greater than 1, then the name will be
-            "idx" (default None).
+        col_names : list or tuple of str, optional
+            The column labels (default None).
+
         """
         raise_error(
             msg="Concrete classes need to implement store_vector().",
-- 
2.52.0


From 2a925df8bdec551d95546504c16d89e265726ecf Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:33:57 +0100
Subject: [PATCH 20/51] update: rename columns parameter to col_names in
 BaseFeatureStorage.store_timeseries()

---
 junifer/storage/base.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/junifer/storage/base.py b/junifer/storage/base.py
index ea4c58e62..03eb5b829 100644
--- a/junifer/storage/base.py
+++ b/junifer/storage/base.py
@@ -257,7 +257,7 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: Dict,
         data: np.ndarray,
-        columns: Optional[Iterable[str]] = None,
+        col_names: Optional[Iterable[str]] = None,
     ) -> None:
         """Store timeseries.
 
@@ -269,8 +269,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        columns : list or tuple of str, optional
+        col_names : list or tuple of str, optional
             The column labels (default None).
+
         """
         raise_error(
             msg="Concrete classes need to implement store_timeseries().",
-- 
2.52.0


From 86ebefdd5999e1223b3a8858b7aabd5f4c4976c8 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:36:28 +0100
Subject: [PATCH 21/51] update: rename store_table() to store_vector() in
 PandasBaseFeatureStorage

---
 junifer/storage/pandas_base.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 24df1da5d..94473fea9 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -178,7 +178,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         # Store dataframe
         self.store_df(meta_md5=meta_md5, element=element, df=data_df)
 
-    def store_table(
+    def store_vector(
         self,
         meta_md5: str,
         element: Dict,
@@ -186,7 +186,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         columns: Optional[Iterable[str]] = None,
         rows_col_name: Optional[str] = None,
     ) -> None:
-        """Implement table storing.
+        """Store vector.
 
         Parameters
         ----------
@@ -194,14 +194,14 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             The metadata MD5 hash.
         element : dict
             The element as a dictionary.
-        data : numpy.ndarray or List
-            The table data to store.
         columns : list or tuple of str, optional
             The columns (default None).
         rows_col_name : str, optional
             The column name to use in case number of rows greater than 1.
             If None and number of rows greater than 1, then the name will be
             "index" (default None).
+        data : numpy.ndarray or list
+            The vector data to store.
         """
         self._store_2d(
             meta_md5=meta_md5,
-- 
2.52.0


From 2c5a90503af093b36fcff4e4201108a1dc5612d2 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:38:51 +0100
Subject: [PATCH 22/51] update: replace columns and rows_col_names with
 col_names for PandasBaseFeatureStorage.store_vector()

---
 junifer/storage/pandas_base.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 94473fea9..9584a07da 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -183,8 +183,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         meta_md5: str,
         element: Dict,
         data: Union[np.ndarray, List],
-        columns: Optional[Iterable[str]] = None,
-        rows_col_name: Optional[str] = None,
+        col_names: Optional[Iterable[str]] = None,
     ) -> None:
         """Store vector.
 
@@ -194,14 +193,11 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             The metadata MD5 hash.
         element : dict
             The element as a dictionary.
-        columns : list or tuple of str, optional
-            The columns (default None).
-        rows_col_name : str, optional
-            The column name to use in case number of rows greater than 1.
-            If None and number of rows greater than 1, then the name will be
-            "index" (default None).
         data : numpy.ndarray or list
             The vector data to store.
+        col_names : list or tuple of str, optional
+            The column labels (default None).
+
         """
         self._store_2d(
             meta_md5=meta_md5,
-- 
2.52.0


From e8960b49f6b4be4edea0b00d6e54536bbffc2245 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:40:08 +0100
Subject: [PATCH 23/51] update: rename columns parameter to col_names in
 PandasBaseFeatureStorage.store_timeseries()

---
 junifer/storage/pandas_base.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 9584a07da..fbf209b17 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -212,7 +212,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         meta_md5: str,
         element: Dict,
         data: np.ndarray,
-        columns: Optional[Iterable[str]] = None,
+        col_names: Optional[Iterable[str]] = None,
     ) -> None:
         """Implement timeseries storing.
 
@@ -224,8 +224,9 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        columns : list or tuple of str, optional
+        col_names : list or tuple of str, optional
             The column labels (default None).
+
         """
         self._store_2d(
             meta_md5=meta_md5,
-- 
2.52.0


From 3b9d50cc92c40e16516547a0310cad55f8ff4efd Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:40:30 +0100
Subject: [PATCH 24/51] chore: update docstring for
 PandasBaseFeatureStorage.store_timeseries()

---
 junifer/storage/pandas_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index fbf209b17..e52241b71 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -214,7 +214,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         data: np.ndarray,
         col_names: Optional[Iterable[str]] = None,
     ) -> None:
-        """Implement timeseries storing.
+        """Store timeseries.
 
         Parameters
         ----------
-- 
2.52.0


From d794ab2a861870b5e6c9cb39e1b8b7704259add5 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:46:42 +0100
Subject: [PATCH 25/51] update: rename columns to col_names in
 PandasBaseFeatureStorage._store_2d()

---
 junifer/storage/pandas_base.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index e52241b71..324fe2dc4 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -145,7 +145,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         meta_md5: str,
         element: Dict,
         data: Union[np.ndarray, List],
-        columns: Optional[Iterable[str]] = None,
+        col_names: Optional[Iterable[str]] = None,
         rows_col_name: Optional[str] = None,
     ) -> None:
         """Store 2D dataframe.
@@ -158,8 +158,8 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             The element as a dictionary.
         data : numpy.ndarray or List
             The data to store.
-        columns : list or tuple of str, optional
-            The columns (default None).
+        col_names : list or tuple of str, optional
+            The column labels (default None).
         rows_col_name : str, optional
             The column name to use in case number of rows greater than 1.
             If None and number of rows greater than 1, then the name will be
@@ -173,7 +173,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         )
         # Prepare new dataframe
         data_df = pd.DataFrame(
-            data, columns=columns, index=idx  # type: ignore
+            data=data, columns=col_names, index=idx  # type: ignore
         )
         # Store dataframe
         self.store_df(meta_md5=meta_md5, element=element, df=data_df)
@@ -203,8 +203,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             meta_md5=meta_md5,
             element=element,
             data=data,
-            columns=columns,
-            rows_col_name=rows_col_name,
+            col_names=col_names,
         )
 
     def store_timeseries(
@@ -232,6 +231,6 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             meta_md5=meta_md5,
             element=element,
             data=data,
-            columns=columns,
+            col_names=col_names,
             rows_col_name="timepoint",
         )
-- 
2.52.0


From f0045c87ad9900dfedc8207b8429974cf74f0a7a Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:47:31 +0100
Subject: [PATCH 26/51] chore: refine PandasBaseFeatureStorage._store_2d()

---
 junifer/storage/pandas_base.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 324fe2dc4..7769eb37c 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -166,17 +166,16 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             "idx" (default None).
 
         """
-        n_rows = len(data)
         # Convert element metadata to index
         idx = self.element_to_index(
-            element=element, n_rows=n_rows, rows_col_name=rows_col_name
+            element=element, n_rows=len(data), rows_col_name=rows_col_name
         )
         # Prepare new dataframe
-        data_df = pd.DataFrame(
+        df = pd.DataFrame(
             data=data, columns=col_names, index=idx  # type: ignore
         )
         # Store dataframe
-        self.store_df(meta_md5=meta_md5, element=element, df=data_df)
+        self.store_df(meta_md5=meta_md5, element=element, df=df)
 
     def store_vector(
         self,
-- 
2.52.0


From e211bd10a20f9cc559437c69a848af7ac3363d7f Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 13:48:05 +0100
Subject: [PATCH 27/51] chore: update docstring for
 PandasBaseFeatureStorage._store_2d()

---
 junifer/storage/pandas_base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 7769eb37c..4c7fcf67a 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -148,7 +148,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
         col_names: Optional[Iterable[str]] = None,
         rows_col_name: Optional[str] = None,
     ) -> None:
-        """Store 2D dataframe.
+        """Store 2D data.
 
         Parameters
         ----------
@@ -156,7 +156,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             The metadata MD5 hash.
         element : dict
             The element as a dictionary.
-        data : numpy.ndarray or List
+        data : numpy.ndarray or list
             The data to store.
         col_names : list or tuple of str, optional
             The column labels (default None).
-- 
2.52.0


From 6ff9366719bccf26bb2557f1226784a3757cd8ad Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 14:10:42 +0100
Subject: [PATCH 28/51] update: improve docstring and error message for
 BaseFeatureStorage.store_matrix()

---
 junifer/storage/base.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/junifer/storage/base.py b/junifer/storage/base.py
index 03eb5b829..5d2aba2e4 100644
--- a/junifer/storage/base.py
+++ b/junifer/storage/base.py
@@ -204,11 +204,9 @@ class BaseFeatureStorage(ABC):
         data : numpy.ndarray
             The matrix data to store.
         col_names : list or tuple of str, optional
-            The column names (default None).
+            The column labels (default None).
         row_names : str, optional
-            The column name to use in case number of rows greater than 1.
-            If None and number of rows greater than 1, then the name will be
-            "idx" (default None).
+            The row labels (default None).
         matrix_kind : str, optional
             The kind of matrix:
 
@@ -218,11 +216,12 @@ class BaseFeatureStorage(ABC):
 
             (default "full").
         diagonal : bool, optional
-            Whether to store the diagonal. If `matrix_kind` is "full", setting
+            Whether to store the diagonal. If ``matrix_kind = full``, setting
             this to False will raise an error (default True).
+
         """
         raise_error(
-            msg="Concrete classes need to implement store_matrix2d().",
+            msg="Concrete classes need to implement store_matrix().",
             klass=NotImplementedError,
         )
 
-- 
2.52.0


From 8b70b343c0934b7e37d78b10a386ebeb17939472 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 14:59:20 +0100
Subject: [PATCH 29/51] chore: improve docstring for
 SQLiteFeatureStorage.store_matrix()

---
 junifer/storage/sqlite.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py
index 4a1b68bf6..b66494203 100644
--- a/junifer/storage/sqlite.py
+++ b/junifer/storage/sqlite.py
@@ -396,7 +396,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
         matrix_kind: Optional[str] = "full",
         diagonal: bool = True,
     ) -> None:
-        """Implement matrix storing.
+        """Store matrix.
 
         Parameters
         ----------
@@ -409,11 +409,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
         meta : dict
             The metadata as a dictionary.
         col_names : list or tuple of str, optional
-            The column names (default None).
+            The column labels (default None).
         row_names : str, optional
-            The column name to use in case number of rows greater than 1.
-            If None and number of rows greater than 1, then the name will be
-            "index" (default None).
+            The row labels (optional None).
         matrix_kind : str, optional
             The kind of matrix:
 
@@ -423,7 +421,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
 
             (default "full").
         diagonal : bool, optional
-            Whether to store the diagonal. If `matrix_kind` is "full", setting
+            Whether to store the diagonal. If ``matrix_kind = full``, setting
             this to False will raise an error (default True).
 
         """
-- 
2.52.0


From 6bc9713adbc32b96ff949b2aaef15841eb7b3263 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 14:59:59 +0100
Subject: [PATCH 30/51] chore: improve commentary for SQLiteFeatureStorage

---
 junifer/storage/sqlite.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py
index b66494203..61e116bda 100644
--- a/junifer/storage/sqlite.py
+++ b/junifer/storage/sqlite.py
@@ -216,13 +216,17 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
             and the values are the metadata of each feature.
 
         """
+        # Retrieve meta table from storage
         meta_df = pd.read_sql(
             sql="meta",
             con=self.get_engine(),
             index_col="meta_md5",
         )
+        # Format index names for retrieved data
         meta_df.index = meta_df.index.str.replace(r"meta_", "")
-        out = meta_df.to_dict(orient="index")  # type: ignore
+        # Convert dataframe to dictionary
+        out: Dict[str, Dict[str, str]] = meta_df.to_dict(orient="index")  # type: ignore
+        # Format output
         for md5, t_meta in out.items():
             for k, v in t_meta.items():
                 out[md5][k] = json.loads(v)
@@ -468,7 +472,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
                 klass=ValueError,
             )
 
+        # Subset data
         flat_data = data[data_idx]
+        # Generate flat 1D row X column names
         columns = [
             f"{row_names[i]}~{col_names[j]}"
             for i, j in zip(data_idx[0], data_idx[1])
@@ -482,6 +488,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
         # Prepare new dataframe
         data_df = pd.DataFrame(flat_data[None, :], columns=columns, index=idx)
 
+        # SQLite's SQLITE_MAX_COLUMN is 2000, so if more than that,
+        # convert it to long format
         if len(columns) > 2000:
             warn_with_log(
                 msg="The number of columns is greater than 2000. "
-- 
2.52.0


From 69a0ec62079010a2d2e50b52f28e76b73d8ea20b Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 15:00:22 +0100
Subject: [PATCH 31/51] chore: improve type annotations for
 SQLiteFeatureStorage

---
 junifer/storage/sqlite.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py
index 61e116bda..20991cde4 100644
--- a/junifer/storage/sqlite.py
+++ b/junifer/storage/sqlite.py
@@ -6,7 +6,7 @@
 
 import json
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -205,7 +205,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
                         msg=f"Invalid option {if_exists} for if_exists."
                     )
 
-    def list_features(self) -> Dict:
+    def list_features(self) -> Dict[str, Dict[str, Any]]:
         """List the features in the storage.
 
         Returns
-- 
2.52.0


From d2f872c8279d3bdb6398b2157a04d0a5bfe1988e Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Wed, 1 Feb 2023 15:00:54 +0100
Subject: [PATCH 32/51] update: rename table to vector in
 SQLiteFeatureStorage.store_vector() tests

---
 junifer/storage/tests/test_sqlite.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py
index d5a006b3b..9c84168fc 100644
--- a/junifer/storage/tests/test_sqlite.py
+++ b/junifer/storage/tests/test_sqlite.py
@@ -361,8 +361,8 @@ def test_store_metadata(tmp_path: Path) -> None:
     assert meta_md5 == feature_md5
 
 
-def test_store_table(tmp_path: Path) -> None:
-    """Test table store.
+def test_store_vector(tmp_path: Path) -> None:
+    """Test vector store.
 
     Parameters
     ----------
@@ -370,7 +370,7 @@ def test_store_table(tmp_path: Path) -> None:
         The path to the test directory.
 
     """
-    uri = tmp_path / "test_store_table.sqlite"
+    uri = tmp_path / "test_store_vector.sqlite"
     storage = SQLiteFeatureStorage(uri=uri)
     # Metadata to store
     element = {"subject": "test"}
@@ -402,7 +402,7 @@ def test_store_table(tmp_path: Path) -> None:
     df = pd.DataFrame(data, columns=["f1", "f2"], index=idx)
 
     # Store table
-    storage.store_table(
+    storage.store_vector(
         meta_md5=meta_md5,
         element=element_to_store,
         data=data,
@@ -427,7 +427,7 @@ def test_store_table(tmp_path: Path) -> None:
     # Check warning
     with pytest.warns(RuntimeWarning, match=r"Some rows"):
         # Store table
-        storage.store_table(
+        storage.store_vector(
             meta_md5=meta_md5,
             element=element_to_store,
             data=data_new,
@@ -767,21 +767,21 @@ def test_store_multiple_output(tmp_path: Path):
     storage.store_metadata(
         meta_md5=hash3, element=element_to_store3, meta=meta_to_store3
     )
-    storage.store_table(
+    storage.store_vector(
         meta_md5=hash1,
         element=element_to_store1,
         data=data1,
         columns=["f1", "f2"],
         rows_col_name="scan",
     )
-    storage.store_table(
+    storage.store_vector(
         meta_md5=hash2,
         element=element_to_store2,
         data=data2,
         columns=["f1", "f2"],
         rows_col_name="scan",
     )
-    storage.store_table(
+    storage.store_vector(
         meta_md5=hash3,
         element=element_to_store3,
         data=data3,
@@ -871,21 +871,21 @@ def test_collect(tmp_path: Path) -> None:
     storage.store_metadata(
         meta_md5=hash3, element=element_to_store3, meta=meta_to_store3
     )
-    storage.store_table(
+    storage.store_vector(
         meta_md5=hash1,
         element=element_to_store1,
         data=data1,
         columns=["f1", "f2"],
         rows_col_name="scan",
     )
-    storage.store_table(
+    storage.store_vector(
         meta_md5=hash2,
         element=element_to_store2,
         data=data2,
         columns=["f1", "f2"],
         rows_col_name="scan",
     )
-    storage.store_table(
+    storage.store_vector(
         meta_md5=hash3,
         element=element_to_store3,
         data=data3,
-- 
2.52.0


From 2df53f34eca3a805d33a5d63fe7c3c7274185241 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Fri, 10 Feb 2023 17:33:10 +0100
Subject: [PATCH 33/51] chore: update marker docstrings to reflect proper
 output from compute()

---
 junifer/markers/ets_rss.py             | 3 +--
 junifer/markers/falff/falff_base.py    | 3 +--
 junifer/markers/falff/falff_parcels.py | 2 +-
 junifer/markers/falff/falff_spheres.py | 2 +-
 junifer/markers/parcel_aggregation.py  | 2 +-
 junifer/markers/reho/reho_parcels.py   | 3 +--
 junifer/markers/reho/reho_spheres.py   | 3 +--
 junifer/markers/sphere_aggregation.py  | 2 +-
 8 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py
index f82c82343..881248497 100644
--- a/junifer/markers/ets_rss.py
+++ b/junifer/markers/ets_rss.py
@@ -110,8 +110,7 @@ class RSSETSMarker(BaseMarker):
             keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``columns`` : the column labels for the computed values as a list
-            * ``row_names`` (if more than one row is present in data): "scan"
+            * ``col_names`` : the column labels for the computed values as a list
 
         References
         ----------
diff --git a/junifer/markers/falff/falff_base.py b/junifer/markers/falff/falff_base.py
index 6b72c04fd..17799a5e9 100644
--- a/junifer/markers/falff/falff_base.py
+++ b/junifer/markers/falff/falff_base.py
@@ -132,8 +132,7 @@ class AmplitudeLowFrequencyFluctuationBase(BaseMarker):
             with this as a parameter. The dictionary has the following keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``columns`` : the column labels for the computed values as a list
-            * ``row_names`` (if more than one row is present in data): "scan"
+            * ``col_names`` : the column labels for the computed values as a list
 
         """
         if self.use_afni is None:
diff --git a/junifer/markers/falff/falff_parcels.py b/junifer/markers/falff/falff_parcels.py
index 25c3e8962..be9e3a7d4 100644
--- a/junifer/markers/falff/falff_parcels.py
+++ b/junifer/markers/falff/falff_parcels.py
@@ -109,7 +109,7 @@ class AmplitudeLowFrequencyFluctuationParcels(
             keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``columns`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as a list
 
         """
         pa = ParcelAggregation(
diff --git a/junifer/markers/falff/falff_spheres.py b/junifer/markers/falff/falff_spheres.py
index e614e5320..50c8367e2 100644
--- a/junifer/markers/falff/falff_spheres.py
+++ b/junifer/markers/falff/falff_spheres.py
@@ -115,7 +115,7 @@ class AmplitudeLowFrequencyFluctuationSpheres(
             keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``columns`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as a list
 
         """
         pa = SphereAggregation(
diff --git a/junifer/markers/parcel_aggregation.py b/junifer/markers/parcel_aggregation.py
index 6115d53bc..35b50eb3a 100644
--- a/junifer/markers/parcel_aggregation.py
+++ b/junifer/markers/parcel_aggregation.py
@@ -123,7 +123,7 @@ class ParcelAggregation(BaseMarker):
             with this as a parameter. The dictionary has the following keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``columns`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as a list
 
         """
         t_input_img = input["data"]
diff --git a/junifer/markers/reho/reho_parcels.py b/junifer/markers/reho/reho_parcels.py
index c18814644..2db8c09d7 100644
--- a/junifer/markers/reho/reho_parcels.py
+++ b/junifer/markers/reho/reho_parcels.py
@@ -122,8 +122,7 @@ class ReHoParcels(ReHoBase):
             keys:
 
             * ``data`` : the actual computed values as a 1D numpy.ndarray
-            * ``columns`` : the column labels for the parcels as a list
-            * ``row_names`` : ``None``
+            * ``col_names`` : the column labels for the parcels as a list
 
         """
         logger.info("Calculating ReHo for parcels.")
diff --git a/junifer/markers/reho/reho_spheres.py b/junifer/markers/reho/reho_spheres.py
index 4871477a9..0232befb3 100644
--- a/junifer/markers/reho/reho_spheres.py
+++ b/junifer/markers/reho/reho_spheres.py
@@ -129,8 +129,7 @@ class ReHoSpheres(ReHoBase):
             keys:
 
             * ``data`` : the actual computed values as a 1D numpy.ndarray
-            * ``columns`` : the column labels for the spheres as a list
-            * ``rows_col_name`` : ``None``
+            * ``col_names`` : the column labels for the spheres as a list
 
         """
         logger.info("Calculating ReHo for spheres.")
diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py
index d25dd5a9e..0bb9f0066 100644
--- a/junifer/markers/sphere_aggregation.py
+++ b/junifer/markers/sphere_aggregation.py
@@ -126,7 +126,7 @@ class SphereAggregation(BaseMarker):
             with this as a parameter. The dictionary has the following keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``columns`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as a list
 
         """
         t_input_img = input["data"]
-- 
2.52.0


From e55a558e309f0cc702b2229d439b76b171c2186a Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Fri, 10 Feb 2023 17:34:52 +0100
Subject: [PATCH 34/51] update: rename all table storing markers to vector

---
 junifer/markers/falff/falff_base.py   | 2 +-
 junifer/markers/parcel_aggregation.py | 2 +-
 junifer/markers/reho/reho_base.py     | 2 +-
 junifer/markers/sphere_aggregation.py | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/junifer/markers/falff/falff_base.py b/junifer/markers/falff/falff_base.py
index 17799a5e9..b3a30db32 100644
--- a/junifer/markers/falff/falff_base.py
+++ b/junifer/markers/falff/falff_base.py
@@ -104,7 +104,7 @@ class AmplitudeLowFrequencyFluctuationBase(BaseMarker):
             The storage type output by the marker.
 
         """
-        return "table"
+        return "vector"
 
     def compute(
         self,
diff --git a/junifer/markers/parcel_aggregation.py b/junifer/markers/parcel_aggregation.py
index 35b50eb3a..61ebfbfb4 100644
--- a/junifer/markers/parcel_aggregation.py
+++ b/junifer/markers/parcel_aggregation.py
@@ -91,7 +91,7 @@ class ParcelAggregation(BaseMarker):
         """
 
         if input_type in ["VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"]:
-            return "table"
+            return "vector"
         elif input_type == "BOLD":
             return "timeseries"
         else:
diff --git a/junifer/markers/reho/reho_base.py b/junifer/markers/reho/reho_base.py
index 6aaadbf99..7e4f7b478 100644
--- a/junifer/markers/reho/reho_base.py
+++ b/junifer/markers/reho/reho_base.py
@@ -70,7 +70,7 @@ class ReHoBase(BaseMarker):
             The storage type output by the marker.
 
         """
-        return "table"
+        return "vector"
 
     def compute_reho_map(
         self,
diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py
index 0bb9f0066..133ded707 100644
--- a/junifer/markers/sphere_aggregation.py
+++ b/junifer/markers/sphere_aggregation.py
@@ -94,7 +94,7 @@ class SphereAggregation(BaseMarker):
         """
 
         if input_type in ["VBM_GM", "VBM_WM", "fALFF", "GCOR", "LCOR"]:
-            return "table"
+            return "vector"
         elif input_type == "BOLD":
             return "timeseries"
         else:
-- 
2.52.0


From ea8667c6fdd3a7a6984c3ad568a4d4fad36bdda6 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Fri, 10 Feb 2023 17:35:41 +0100
Subject: [PATCH 35/51] update: rename columns to col_names for returning from
 markers compute()

---
 .../functional_connectivity/functional_connectivity_base.py   | 4 ++--
 junifer/markers/parcel_aggregation.py                         | 2 +-
 junifer/markers/sphere_aggregation.py                         | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/junifer/markers/functional_connectivity/functional_connectivity_base.py b/junifer/markers/functional_connectivity/functional_connectivity_base.py
index 6de34ab07..6f779cbd3 100644
--- a/junifer/markers/functional_connectivity/functional_connectivity_base.py
+++ b/junifer/markers/functional_connectivity/functional_connectivity_base.py
@@ -143,7 +143,7 @@ class FunctionalConnectivityBase(BaseMarker):
         out = {}
         out["data"] = connectivity.fit_transform([aggregation["data"]])[0]
         # Create column names
-        out["row_names"] = aggregation["columns"]
-        out["col_names"] = aggregation["columns"]
+        out["row_names"] = aggregation["col_names"]
+        out["col_names"] = aggregation["col_names"]
         out["matrix_kind"] = "tril"
         return out
diff --git a/junifer/markers/parcel_aggregation.py b/junifer/markers/parcel_aggregation.py
index 61ebfbfb4..215c81a8e 100644
--- a/junifer/markers/parcel_aggregation.py
+++ b/junifer/markers/parcel_aggregation.py
@@ -218,5 +218,5 @@ class ParcelAggregation(BaseMarker):
             out_labels.append(labels[t_v - 1])
 
         out_values = np.array(out_values).T
-        out = {"data": out_values, "columns": out_labels}
+        out = {"data": out_values, "col_names": out_labels}
         return out
diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py
index 133ded707..b34c3ed99 100644
--- a/junifer/markers/sphere_aggregation.py
+++ b/junifer/markers/sphere_aggregation.py
@@ -151,5 +151,5 @@ class SphereAggregation(BaseMarker):
         # Fit and transform the marker on the data
         out_values = masker.fit_transform(t_input_img)
         # Format the output
-        out = {"data": out_values, "columns": out_labels}
+        out = {"data": out_values, "col_names": out_labels}
         return out
-- 
2.52.0


From cb77c0706d9e717f185296230ea48c5b52d0350b Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:02:07 +0100
Subject: [PATCH 36/51] chore: flake8 fix for SQLiteFeatureStorage

---
 junifer/storage/sqlite.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py
index 20991cde4..217d71161 100644
--- a/junifer/storage/sqlite.py
+++ b/junifer/storage/sqlite.py
@@ -225,7 +225,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
         # Format index names for retrieved data
         meta_df.index = meta_df.index.str.replace(r"meta_", "")
         # Convert dataframe to dictionary
-        out: Dict[str, Dict[str, str]] = meta_df.to_dict(orient="index")  # type: ignore
+        out: Dict[str, Dict[str, str]] = meta_df.to_dict(
+            orient="index"
+        )  # type: ignore
         # Format output
         for md5, t_meta in out.items():
             for k, v in t_meta.items():
-- 
2.52.0


From b548e1498dc20981d0ecce1768631b71a0bb617f Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:03:09 +0100
Subject: [PATCH 37/51] update: correct test for
 SQLiteFeatureStorage.store_vector()

---
 junifer/storage/tests/test_sqlite.py | 43 ++++------------------------
 1 file changed, 6 insertions(+), 37 deletions(-)

diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py
index 9c84168fc..eef47ede5 100644
--- a/junifer/storage/tests/test_sqlite.py
+++ b/junifer/storage/tests/test_sqlite.py
@@ -389,60 +389,29 @@ def test_store_vector(tmp_path: Path) -> None:
     )
 
     # Data to store
-    data = [
-        [1, 10],
-        [2, 20],
-        [3, 30],
-        [4, 40],
-        [5, 50],
-    ]
+    data = [[10, 20, 30, 40, 50]]
+    col_names = ["f1", "f2", "f3", "f4", "f5"]
     # Convert element to index
-    idx = storage.element_to_index(element, n_rows=5, rows_col_name="scan")
+    idx = storage.element_to_index(element=element)
     # Create dataframe
-    df = pd.DataFrame(data, columns=["f1", "f2"], index=idx)
+    df = pd.DataFrame(data=data, columns=col_names, index=idx)
 
     # Store table
     storage.store_vector(
         meta_md5=meta_md5,
         element=element_to_store,
         data=data,
-        columns=["f1", "f2"],
-        rows_col_name="scan",
+        col_names=col_names,
     )
     # Read stored table
     c_df = _read_sql(
         table_name=f"meta_{meta_md5}",
         uri=uri.as_posix(),
-        index_col=["subject", "scan"],
+        index_col=["subject"],
     )
     # Check if dataframes are equal
     assert_frame_equal(df, c_df)
 
-    # New data to store
-    data_new = [[1, 10], [2, 20], [3, 300], [4, 40], [5, 50], [6, 600]]
-    # Convert element to index
-    idx_new = storage.element_to_index(element, n_rows=6, rows_col_name="scan")
-    # Create dataframe
-    df_new = pd.DataFrame(data_new, columns=["f1", "f2"], index=idx_new)
-    # Check warning
-    with pytest.warns(RuntimeWarning, match=r"Some rows"):
-        # Store table
-        storage.store_vector(
-            meta_md5=meta_md5,
-            element=element_to_store,
-            data=data_new,
-            columns=["f1", "f2"],
-            rows_col_name="scan",
-        )
-    # Read stored table
-    c_df_new = _read_sql(
-        table_name=f"meta_{meta_md5}",
-        uri=uri.as_posix(),
-        index_col=["subject", "scan"],
-    )
-    # Check if dataframes are equal
-    assert_frame_equal(df_new, c_df_new)
-
 
 def test_store_matrix(tmp_path: Path) -> None:
     """Test matrix store.
-- 
2.52.0


From 167e792ea1c2f2ae15cecc0bccc91e46c2b26baa Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:04:44 +0100
Subject: [PATCH 38/51] update: add test for
 SQLiteFeatureStorage.store_timeseries()

---
 junifer/storage/tests/test_sqlite.py | 54 ++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py
index eef47ede5..d55e77fd2 100644
--- a/junifer/storage/tests/test_sqlite.py
+++ b/junifer/storage/tests/test_sqlite.py
@@ -656,6 +656,60 @@ def test_store_matrix(tmp_path: Path) -> None:
     )
 
 
+def test_store_timeseries(tmp_path: Path) -> None:
+    """Test timeseries store.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    uri = tmp_path / "test_store_timeseries.sqlite"
+    storage = SQLiteFeatureStorage(uri=uri)
+    # Metadata to store
+    element = {"subject": "test"}
+    dependencies = ["numpy"]
+    meta = {
+        "element": element,
+        "dependencies": dependencies,
+        "marker": {"name": "fc"},
+        "type": "BOLD",
+    }
+
+    meta_md5, meta_to_store, element_to_store = process_meta(meta)
+    # Store metadata
+    storage.store_metadata(
+        meta_md5=meta_md5, element=element_to_store, meta=meta_to_store
+    )
+
+    # Data to store
+    data = np.array([[10], [20], [30], [40], [50]])
+    col_names = ["signal"]
+    # Convert element to index
+    idx = storage.element_to_index(
+        element=element, n_rows=5, rows_col_name="timepoint"
+    )
+    # Create dataframe
+    df = pd.DataFrame(data=data, columns=col_names, index=idx)
+
+    # Store table
+    storage.store_timeseries(
+        meta_md5=meta_md5,
+        element=element_to_store,
+        data=data,
+        col_names=col_names,
+    )
+    # Read stored table
+    c_df = _read_sql(
+        table_name=f"meta_{meta_md5}",
+        uri=uri.as_posix(),
+        index_col=["subject", "timepoint"],
+    )
+    # Check if dataframes are equal
+    assert_frame_equal(df, c_df)
+
+
 # TODO: can the test be parametrized?
 def test_store_multiple_output(tmp_path: Path):
     """Test storing using single_output=False.
-- 
2.52.0


From 0c1e5e719a9157f885d09eff72287f07d97ca49d Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:06:34 +0100
Subject: [PATCH 39/51] update: fix test for multiple data store using
 SQLiteFeatureStorage

---
 junifer/storage/tests/test_sqlite.py | 57 ++++++++++------------------
 1 file changed, 19 insertions(+), 38 deletions(-)

diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py
index d55e77fd2..0310876f9 100644
--- a/junifer/storage/tests/test_sqlite.py
+++ b/junifer/storage/tests/test_sqlite.py
@@ -742,41 +742,28 @@ def test_store_multiple_output(tmp_path: Path):
         "type": "BOLD",
     }
     # Data to store
-    data1 = np.array(
-        [
-            [1, 10],
-            [2, 20],
-            [3, 30],
-            [4, 40],
-            [5, 50],
-        ]
-    )
+    data1 = np.array([[10, 20, 30, 40, 50]])
     data2 = data1 * 10
     data3 = data1 * 20
+    col_names = ["f1", "f2", "f3", "f4", "f5"]
     # Process metadata for storage
     hash1, meta_to_store1, element_to_store1 = process_meta(meta1)
     # Convert element to index
-    idx1 = storage.element_to_index(
-        element_to_store1, n_rows=5, rows_col_name="scan"
-    )
+    idx1 = storage.element_to_index(element=element_to_store1)
     # Create dataframe
-    df1 = pd.DataFrame(data1, columns=["f1", "f2"], index=idx1)
+    df1 = pd.DataFrame(data1, columns=col_names, index=idx1)
     # Process metadata for storage
     hash2, meta_to_store2, element_to_store2 = process_meta(meta2)
     # Convert element to index
-    idx2 = storage.element_to_index(
-        element_to_store2, n_rows=5, rows_col_name="scan"
-    )
+    idx2 = storage.element_to_index(element=element_to_store2)
     # Create dataframe
-    df2 = pd.DataFrame(data2, columns=["f1", "f2"], index=idx2)
+    df2 = pd.DataFrame(data2, columns=col_names, index=idx2)
     # Process metadata for storage
     hash3, meta_to_store3, element_to_store3 = process_meta(meta3)
     # Convert element to index
-    idx3 = storage.element_to_index(
-        element_to_store3, n_rows=5, rows_col_name="scan"
-    )
+    idx3 = storage.element_to_index(element=element_to_store3)
     # Create dataframe
-    df3 = pd.DataFrame(data3, columns=["f1", "f2"], index=idx3)
+    df3 = pd.DataFrame(data3, columns=col_names, index=idx3)
     # Check hash equality
     assert hash1 == hash2
     assert hash2 == hash3
@@ -794,22 +781,19 @@ def test_store_multiple_output(tmp_path: Path):
         meta_md5=hash1,
         element=element_to_store1,
         data=data1,
-        columns=["f1", "f2"],
-        rows_col_name="scan",
+        col_names=col_names,
     )
     storage.store_vector(
         meta_md5=hash2,
         element=element_to_store2,
         data=data2,
-        columns=["f1", "f2"],
-        rows_col_name="scan",
+        col_names=col_names,
     )
     storage.store_vector(
         meta_md5=hash3,
         element=element_to_store3,
         data=data3,
-        columns=["f1", "f2"],
-        rows_col_name="scan",
+        col_names=col_names,
     )
     # Check that URI does not exist yet
     assert not uri.exists()
@@ -826,12 +810,12 @@ def test_store_multiple_output(tmp_path: Path):
     assert uri2.exists()
     assert uri3.exists()
     # Set index columns
-    cols = ["subject", "session", "scan"]
+    idx_cols = ["subject", "session"]
     table_name = f"meta_{hash1}"
     # Read stored tables
-    cdf1 = _read_sql(table_name, uri1.as_posix(), index_col=cols)
-    cdf2 = _read_sql(table_name, uri2.as_posix(), index_col=cols)
-    cdf3 = _read_sql(table_name, uri3.as_posix(), index_col=cols)
+    cdf1 = _read_sql(table_name, uri1.as_posix(), index_col=idx_cols)
+    cdf2 = _read_sql(table_name, uri2.as_posix(), index_col=idx_cols)
+    cdf3 = _read_sql(table_name, uri3.as_posix(), index_col=idx_cols)
     # Check if dataframes are equal
     assert_frame_equal(df1, cdf1)
     assert_frame_equal(df2, cdf2)
@@ -898,22 +882,19 @@ def test_collect(tmp_path: Path) -> None:
         meta_md5=hash1,
         element=element_to_store1,
         data=data1,
-        columns=["f1", "f2"],
-        rows_col_name="scan",
+        col_names=["f1", "f2"],
     )
     storage.store_vector(
         meta_md5=hash2,
         element=element_to_store2,
         data=data2,
-        columns=["f1", "f2"],
-        rows_col_name="scan",
+        col_names=["f1", "f2"],
     )
     storage.store_vector(
         meta_md5=hash3,
         element=element_to_store3,
         data=data3,
-        columns=["f1", "f2"],
-        rows_col_name="scan",
+        col_names=["f1", "f2"],
     )
     # Convert element to prefix
     prefix1 = element_to_prefix(meta1["element"])
@@ -934,7 +915,7 @@ def test_collect(tmp_path: Path) -> None:
     # Check that URI exists now
     assert uri.exists()
     # Set index columns
-    cols = ["subject", "session", "scan"]
+    cols = ["subject", "session"]
     # Store metadata
     table_name = f"meta_{hash1}"
     # Read stored tables
-- 
2.52.0


From 4c8c95591d11baeca465332f7137ba94cb53581e Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:07:17 +0100
Subject: [PATCH 40/51] chore: flake8 fix for
 PandasBaseFeatureStorage.element_to_index() unit tests

---
 junifer/storage/tests/test_pandas_base.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/junifer/storage/tests/test_pandas_base.py b/junifer/storage/tests/test_pandas_base.py
index 85a6e5999..67c6c7767 100644
--- a/junifer/storage/tests/test_pandas_base.py
+++ b/junifer/storage/tests/test_pandas_base.py
@@ -39,7 +39,9 @@ def test_element_to_index() -> None:
     # Check second index level name
     assert index.levels[1].name == "idx"  # type: ignore
     # Check second index level values
-    assert all(x == i for i, x in enumerate(index.levels[1].values))  # type: ignore
+    assert all(
+        x == i for i, x in enumerate(index.levels[1].values)  # type: ignore
+    )
     # Check second index level values shape
     assert index.levels[1].values.shape == (10,)  # type: ignore
 
@@ -72,7 +74,9 @@ def test_element_to_index() -> None:
     # Check second index level name
     assert index.levels[1].name == "scan"  # type: ignore
     # Check second index level values
-    assert all(x == i for i, x in enumerate(index.levels[1].values))  # type: ignore
+    assert all(
+        x == i for i, x in enumerate(index.levels[1].values)  # type: ignore
+    )
     # Check second index level values shape
     assert index.levels[1].values.shape == (7,)  # type: ignore
 
@@ -98,6 +102,8 @@ def test_element_to_index() -> None:
     # Check third index level name
     assert index.levels[2].name == "idx"  # type: ignore
     # Check third index level values
-    assert all(x == i for i, x in enumerate(index.levels[2].values))  # type: ignore
+    assert all(
+        x == i for i, x in enumerate(index.levels[2].values)  # type: ignore
+    )
     # Check third index level values shape
     assert index.levels[2].values.shape == (10,)  # type: ignore
-- 
2.52.0


From 39b2241527e178324e19134e33652b04cfb24b2f Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:09:21 +0100
Subject: [PATCH 41/51] update: improve logic of
 PandasBaseFeatureStorage.store_vector()

---
 junifer/storage/pandas_base.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 4c7fcf67a..fa925c125 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -198,6 +198,16 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             The column labels (default None).
 
         """
+        if isinstance(data, list):
+            # Flatten out list and convert to np.ndarray
+            processed_data = np.array(np.ravel(data))
+        elif isinstance(data, np.ndarray):
+            # Flatten out array
+            processed_data = data.ravel()
+
+        # Make it 2D
+        processed_data = processed_data[np.newaxis, :]
+
         self._store_2d(
             meta_md5=meta_md5,
             element=element,
-- 
2.52.0


From 48a1af5e48931b5e0644b2a1a4442ed81e8eb308 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:10:16 +0100
Subject: [PATCH 42/51] update: return pandas.Index for vector store and single
 element access variable in PandasBaseFeatureStorage

---
 junifer/storage/pandas_base.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index fa925c125..4856018cf 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -6,7 +6,7 @@
 
 import json
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Union
+from typing import Dict, Iterable, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -78,7 +78,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
     @staticmethod
     def element_to_index(
         element: Dict, n_rows: int = 1, rows_col_name: Optional[str] = None
-    ) -> pd.MultiIndex:
+    ) -> Union[pd.Index, pd.MultiIndex]:
         """Convert the element metadata to index.
 
         Parameters
@@ -93,7 +93,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
 
         Returns
         -------
-        pandas.MultiIndex
+        pandas.Index or pandas.MultiIndex
             The index of the dataframe to store.
 
         """
@@ -102,7 +102,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             k: [v] * n_rows for k, v in element.items()
         }
 
-        # Set rows_col_name if n_rows > 1
+        # Set rows_col_name if n_rows > 1 (timeseries)
         if n_rows > 1:
             # Set rows_col_name if None
             if rows_col_name is None:
@@ -110,10 +110,18 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             # Set extra column for variable number of rows per element
             elem_idx[rows_col_name] = np.arange(n_rows)
 
-        # Create index
-        index = pd.MultiIndex.from_frame(
-            pd.DataFrame(elem_idx, index=range(n_rows))
-        )
+        # Create correct index for elements with single access variable
+        if len(elem_idx) == 1:
+            # Create normal index for vector
+            index = pd.Index(
+                data=list(elem_idx.values())[0], name=list(elem_idx.keys())[0]
+            )
+        else:
+            # Create multiindex for timeseries
+            index = pd.MultiIndex.from_frame(
+                pd.DataFrame(elem_idx, index=range(n_rows))
+            )
+
         return index
 
     def store_df(
-- 
2.52.0


From fa5df7eabed2e12b9228d821771bf1d0f1881fbd Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:19:56 +0100
Subject: [PATCH 43/51] chore: flake8

---
 junifer/markers/ets_rss.py             | 2 +-
 junifer/markers/falff/falff_base.py    | 2 +-
 junifer/markers/falff/falff_parcels.py | 2 +-
 junifer/markers/falff/falff_spheres.py | 2 +-
 junifer/markers/parcel_aggregation.py  | 2 +-
 junifer/markers/sphere_aggregation.py  | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py
index 881248497..96f50d5d9 100644
--- a/junifer/markers/ets_rss.py
+++ b/junifer/markers/ets_rss.py
@@ -110,7 +110,7 @@ class RSSETSMarker(BaseMarker):
             keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``col_names`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as list
 
         References
         ----------
diff --git a/junifer/markers/falff/falff_base.py b/junifer/markers/falff/falff_base.py
index b3a30db32..fe8ed478b 100644
--- a/junifer/markers/falff/falff_base.py
+++ b/junifer/markers/falff/falff_base.py
@@ -132,7 +132,7 @@ class AmplitudeLowFrequencyFluctuationBase(BaseMarker):
             with this as a parameter. The dictionary has the following keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``col_names`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as list
 
         """
         if self.use_afni is None:
diff --git a/junifer/markers/falff/falff_parcels.py b/junifer/markers/falff/falff_parcels.py
index be9e3a7d4..6ef7d8e79 100644
--- a/junifer/markers/falff/falff_parcels.py
+++ b/junifer/markers/falff/falff_parcels.py
@@ -109,7 +109,7 @@ class AmplitudeLowFrequencyFluctuationParcels(
             keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``col_names`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as list
 
         """
         pa = ParcelAggregation(
diff --git a/junifer/markers/falff/falff_spheres.py b/junifer/markers/falff/falff_spheres.py
index 50c8367e2..b0d5b417c 100644
--- a/junifer/markers/falff/falff_spheres.py
+++ b/junifer/markers/falff/falff_spheres.py
@@ -115,7 +115,7 @@ class AmplitudeLowFrequencyFluctuationSpheres(
             keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``col_names`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as list
 
         """
         pa = SphereAggregation(
diff --git a/junifer/markers/parcel_aggregation.py b/junifer/markers/parcel_aggregation.py
index 215c81a8e..ef8af0fa2 100644
--- a/junifer/markers/parcel_aggregation.py
+++ b/junifer/markers/parcel_aggregation.py
@@ -123,7 +123,7 @@ class ParcelAggregation(BaseMarker):
             with this as a parameter. The dictionary has the following keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``col_names`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as list
 
         """
         t_input_img = input["data"]
diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py
index b34c3ed99..2a904f46e 100644
--- a/junifer/markers/sphere_aggregation.py
+++ b/junifer/markers/sphere_aggregation.py
@@ -126,7 +126,7 @@ class SphereAggregation(BaseMarker):
             with this as a parameter. The dictionary has the following keys:
 
             * ``data`` : the actual computed values as a numpy.ndarray
-            * ``col_names`` : the column labels for the computed values as a list
+            * ``col_names`` : the column labels for the computed values as list
 
         """
         t_input_img = input["data"]
-- 
2.52.0


From 490ff9bf9d930b430793b1c0f3e180d8bd52c05c Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 11:20:54 +0100
Subject: [PATCH 44/51] chore: lint

---
 junifer/data/masks.py                 | 15 +++++++--------
 junifer/data/tests/test_masks.py      | 18 ++++++------------
 junifer/datagrabber/tests/test_hcp.py |  8 +++-----
 junifer/markers/sphere_aggregation.py |  2 +-
 4 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/junifer/data/masks.py b/junifer/data/masks.py
index 754089796..55ddf967a 100644
--- a/junifer/data/masks.py
+++ b/junifer/data/masks.py
@@ -7,24 +7,23 @@ from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Dict,
     List,
     Optional,
     Tuple,
     Union,
-    Callable,
 )
 
-import numpy as np
-
 import nibabel as nib
-from nilearn.masking import (
-    compute_brain_mask,
-    compute_background_mask,
-    compute_epi_mask,
-)
+import numpy as np
 from nilearn.datasets import fetch_icbm152_brain_gm_mask
 from nilearn.image import resample_to_img
+from nilearn.masking import (
+    compute_background_mask,
+    compute_brain_mask,
+    compute_epi_mask,
+)
 
 from ..utils.logging import logger, raise_error
 from .utils import closest_resolution
diff --git a/junifer/data/tests/test_masks.py b/junifer/data/tests/test_masks.py
index ec3739612..832500787 100644
--- a/junifer/data/tests/test_masks.py
+++ b/junifer/data/tests/test_masks.py
@@ -5,33 +5,27 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+from pathlib import Path
 from typing import Callable, Dict, Union
 
-from pathlib import Path
-
 import pytest
-from numpy.testing import (
-    assert_array_almost_equal,
-    assert_array_equal,
-)
-
+from nilearn.datasets import fetch_icbm152_brain_gm_mask
 from nilearn.image import resample_to_img
 from nilearn.masking import (
-    compute_brain_mask,
     compute_background_mask,
+    compute_brain_mask,
     compute_epi_mask,
 )
-from nilearn.datasets import fetch_icbm152_brain_gm_mask
+from numpy.testing import assert_array_almost_equal, assert_array_equal
 
 from junifer.data.masks import (
+    _available_masks,
     _load_vickery_patil_mask,
+    get_mask,
     list_masks,
     load_mask,
     register_mask,
-    get_mask,
-    _available_masks,
 )
-
 from junifer.datareader import DefaultDataReader
 from junifer.testing.datagrabbers import (
     OasisVBMTestingDatagrabber,
diff --git a/junifer/datagrabber/tests/test_hcp.py b/junifer/datagrabber/tests/test_hcp.py
index 90ddd4329..375f4409d 100644
--- a/junifer/datagrabber/tests/test_hcp.py
+++ b/junifer/datagrabber/tests/test_hcp.py
@@ -3,11 +3,11 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Optional, Iterable
+from typing import Iterable, Optional
 
 import pytest
 
-from junifer.datagrabber.hcp import DataladHCP1200, HCP1200
+from junifer.datagrabber.hcp import HCP1200, DataladHCP1200
 from junifer.utils import configure_logging
 
 
@@ -332,9 +332,7 @@ def test_hcp1200_datagrabber_elements(
     )
     with dg:
         # Get all elements
-        expected_subjects = [
-            f"sub-{x:02d}" for x in range(1, 10)
-        ]
+        expected_subjects = [f"sub-{x:02d}" for x in range(1, 10)]
         found_subjects = []
         all_elements = dg.get_elements()
         # Check only specified task and phase encoding are found
diff --git a/junifer/markers/sphere_aggregation.py b/junifer/markers/sphere_aggregation.py
index 2a904f46e..36d3c1ebb 100644
--- a/junifer/markers/sphere_aggregation.py
+++ b/junifer/markers/sphere_aggregation.py
@@ -7,7 +7,7 @@
 from typing import Any, Dict, List, Optional, Union
 
 from ..api.decorators import register_marker
-from ..data import load_coordinates, get_mask
+from ..data import get_mask, load_coordinates
 from ..external.nilearn import JuniferNiftiSpheresMasker
 from ..stats import get_aggfunc_by_name
 from ..utils import logger
-- 
2.52.0


From b9e3cc1344e23254c8e7242bff5c247ddb77f10d Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 13:37:14 +0100
Subject: [PATCH 45/51] update: rename remaining table to vector in unit tests

---
 junifer/markers/tests/test_parcel_aggregation.py | 2 +-
 junifer/markers/tests/test_sphere_aggregation.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/junifer/markers/tests/test_parcel_aggregation.py b/junifer/markers/tests/test_parcel_aggregation.py
index 007eccfe1..2cf264253 100644
--- a/junifer/markers/tests/test_parcel_aggregation.py
+++ b/junifer/markers/tests/test_parcel_aggregation.py
@@ -26,7 +26,7 @@ def test_ParcelAggregation_input_output() -> None:
     marker = ParcelAggregation(
         parcellation="Schaefer100x7", method="mean", on="VBM_GM"
     )
-    for in_, out_ in [("VBM_GM", "table"), ("BOLD", "timeseries")]:
+    for in_, out_ in [("VBM_GM", "vector"), ("BOLD", "timeseries")]:
         assert marker.get_output_type(in_) == out_
 
     with pytest.raises(ValueError, match="Unknown input"):
diff --git a/junifer/markers/tests/test_sphere_aggregation.py b/junifer/markers/tests/test_sphere_aggregation.py
index 0d373faa5..6b66deed6 100644
--- a/junifer/markers/tests/test_sphere_aggregation.py
+++ b/junifer/markers/tests/test_sphere_aggregation.py
@@ -27,7 +27,7 @@ RADIUS = 8
 def test_SphereAggregation_input_output() -> None:
     """Test SphereAggregation input and output types."""
     marker = SphereAggregation(coords="DMNBuckner", method="mean", on="VBM_GM")
-    for in_, out_ in [("VBM_GM", "table"), ("BOLD", "timeseries")]:
+    for in_, out_ in [("VBM_GM", "vector"), ("BOLD", "timeseries")]:
         assert marker.get_output_type(in_) == out_
 
     with pytest.raises(ValueError, match="Unknown input"):
-- 
2.52.0


From 302b3df95efc1725ffb0951a872b38e4dd4ec11b Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 13:37:35 +0100
Subject: [PATCH 46/51] update: rename remaining table to vector in
 PandasBaseFeatureStorage

---
 junifer/storage/pandas_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/junifer/storage/pandas_base.py b/junifer/storage/pandas_base.py
index 4856018cf..c56f1d73f 100644
--- a/junifer/storage/pandas_base.py
+++ b/junifer/storage/pandas_base.py
@@ -51,7 +51,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
             storage interface.
 
         """
-        return ["matrix", "table", "timeseries"]
+        return ["matrix", "vector", "timeseries"]
 
     def _meta_row(self, meta: Dict, meta_md5: str) -> pd.DataFrame:
         """Convert the metadata to a pandas DataFrame.
-- 
2.52.0


From c530aa87cf138928a023e38d347d8c61efdf1fbd Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 13:38:15 +0100
Subject: [PATCH 47/51] update: rename remaining table to vector in
 SQLiteFeatureStorage

---
 junifer/storage/sqlite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py
index 217d71161..706dc64ae 100644
--- a/junifer/storage/sqlite.py
+++ b/junifer/storage/sqlite.py
@@ -78,7 +78,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
             )
             uri.parent.mkdir(parents=True, exist_ok=True)
         # Available storage kinds
-        storage_types = ["table", "timeseries", "matrix"]
+        storage_types = ["vector", "timeseries", "matrix"]
         super().__init__(
             uri=uri,
             storage_types=storage_types,
-- 
2.52.0


From 992bd0d0dbc6ce6ea0b5a9e4cad86ff9052a5bfb Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 14:47:27 +0100
Subject: [PATCH 48/51] update: replace remaining columns to col_names for
 markers' tests

---
 .../crossparcellation_functional_connectivity.py          | 4 ++--
 .../edge_functional_connectivity_parcels.py               | 4 ++--
 .../edge_functional_connectivity_spheres.py               | 4 ++--
 junifer/markers/reho/tests/test_reho_parcels.py           | 2 +-
 junifer/markers/reho/tests/test_reho_spheres.py           | 2 +-
 junifer/markers/tests/test_collection.py                  | 8 ++++----
 junifer/markers/tests/test_parcel_aggregation.py          | 6 +++---
 7 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/junifer/markers/functional_connectivity/crossparcellation_functional_connectivity.py b/junifer/markers/functional_connectivity/crossparcellation_functional_connectivity.py
index 74baa08c9..25896d947 100644
--- a/junifer/markers/functional_connectivity/crossparcellation_functional_connectivity.py
+++ b/junifer/markers/functional_connectivity/crossparcellation_functional_connectivity.py
@@ -150,6 +150,6 @@ class CrossParcellationFC(BaseMarker):
 
         return {
             "data": result,
-            "col_names": parcellation_one_dict["columns"],
-            "row_names": parcellation_two_dict["columns"],
+            "col_names": parcellation_one_dict["col_names"],
+            "row_names": parcellation_two_dict["col_names"],
         }
diff --git a/junifer/markers/functional_connectivity/edge_functional_connectivity_parcels.py b/junifer/markers/functional_connectivity/edge_functional_connectivity_parcels.py
index eb02f56c0..5232bf582 100644
--- a/junifer/markers/functional_connectivity/edge_functional_connectivity_parcels.py
+++ b/junifer/markers/functional_connectivity/edge_functional_connectivity_parcels.py
@@ -84,7 +84,7 @@ class EdgeCentricFCParcels(FunctionalConnectivityBase):
 
         bold_aggregated = parcel_aggregation.compute(input)
         ets, edge_names = _ets(
-            bold_aggregated["data"], bold_aggregated["columns"]
+            bold_aggregated["data"], bold_aggregated["col_names"]
         )
 
-        return dict(data=ets, columns=edge_names)
+        return {"data": ets, "col_names": edge_names}
diff --git a/junifer/markers/functional_connectivity/edge_functional_connectivity_spheres.py b/junifer/markers/functional_connectivity/edge_functional_connectivity_spheres.py
index 6fcabffbe..b8a5f39de 100644
--- a/junifer/markers/functional_connectivity/edge_functional_connectivity_spheres.py
+++ b/junifer/markers/functional_connectivity/edge_functional_connectivity_spheres.py
@@ -91,7 +91,7 @@ class EdgeCentricFCSpheres(FunctionalConnectivityBase):
         )
         bold_aggregated = sphere_aggregation.compute(input)
         ets, edge_names = _ets(
-            bold_aggregated["data"], bold_aggregated["columns"]
+            bold_aggregated["data"], bold_aggregated["col_names"]
         )
 
-        return dict(data=ets, columns=edge_names)
+        return {"data": ets, "col_names": edge_names}
diff --git a/junifer/markers/reho/tests/test_reho_parcels.py b/junifer/markers/reho/tests/test_reho_parcels.py
index ef3f6bcb3..4b427def7 100644
--- a/junifer/markers/reho/tests/test_reho_parcels.py
+++ b/junifer/markers/reho/tests/test_reho_parcels.py
@@ -35,7 +35,7 @@ def test_reho_parcels_computation() -> None:
         reho_parcels_output_bold = reho_parcels_output["BOLD"]
         # Assert BOLD output keys
         assert "data" in reho_parcels_output_bold
-        assert "columns" in reho_parcels_output_bold
+        assert "col_names" in reho_parcels_output_bold
 
         reho_parcels_output_bold_data = reho_parcels_output_bold["data"]
         # Assert BOLD output data dimension
diff --git a/junifer/markers/reho/tests/test_reho_spheres.py b/junifer/markers/reho/tests/test_reho_spheres.py
index d737cbf23..4bd42a2b5 100644
--- a/junifer/markers/reho/tests/test_reho_spheres.py
+++ b/junifer/markers/reho/tests/test_reho_spheres.py
@@ -35,7 +35,7 @@ def test_reho_spheres_computation() -> None:
         reho_spheres_output_bold = reho_spheres_output["BOLD"]
         # Assert BOLD output keys
         assert "data" in reho_spheres_output_bold
-        assert "columns" in reho_spheres_output_bold
+        assert "col_names" in reho_spheres_output_bold
 
         reho_spheres_output_bold_data = reho_spheres_output_bold["data"]
         # Assert BOLD output data dimension
diff --git a/junifer/markers/tests/test_collection.py b/junifer/markers/tests/test_collection.py
index 2a8f1ec7b..b2b3ad8ec 100644
--- a/junifer/markers/tests/test_collection.py
+++ b/junifer/markers/tests/test_collection.py
@@ -87,7 +87,7 @@ def test_marker_collection() -> None:
             assert "VBM_GM" in out[t_name]
             t_vbm = out[t_name]["VBM_GM"]
             assert "data" in t_vbm
-            assert "columns" in t_vbm
+            assert "col_names" in t_vbm
             assert "meta" in t_vbm
 
     # Test preprocessing
@@ -199,19 +199,19 @@ def test_marker_collection_storage(tmp_path: Path) -> None:
     t_feature = storage.read_df(feature_md5=feature_md5)
     fname = "gmd_schaefer100x7_mean"
     t_data = out[fname]["VBM_GM"]["data"]  # type: ignore
-    cols = out[fname]["VBM_GM"]["columns"]  # type: ignore
+    cols = out[fname]["VBM_GM"]["col_names"]  # type: ignore
     assert_array_equal(t_feature[cols].values, t_data)  # type: ignore
 
     feature_md5 = list(features.keys())[1]
     t_feature = storage.read_df(feature_md5=feature_md5)
     fname = "gmd_schaefer100x7_std"
     t_data = out[fname]["VBM_GM"]["data"]  # type: ignore
-    cols = out[fname]["VBM_GM"]["columns"]  # type: ignore
+    cols = out[fname]["VBM_GM"]["col_names"]  # type: ignore
     assert_array_equal(t_feature[cols].values, t_data)  # type: ignore
 
     feature_md5 = list(features.keys())[2]
     t_feature = storage.read_df(feature_md5=feature_md5)
     fname = "gmd_schaefer100x7_trim_mean90"
     t_data = out[fname]["VBM_GM"]["data"]  # type: ignore
-    cols = out[fname]["VBM_GM"]["columns"]  # type: ignore
+    cols = out[fname]["VBM_GM"]["col_names"]  # type: ignore
     assert_array_equal(t_feature[cols].values, t_data)  # type: ignore
diff --git a/junifer/markers/tests/test_parcel_aggregation.py b/junifer/markers/tests/test_parcel_aggregation.py
index 2cf264253..329581366 100644
--- a/junifer/markers/tests/test_parcel_aggregation.py
+++ b/junifer/markers/tests/test_parcel_aggregation.py
@@ -364,7 +364,7 @@ def test_ParcelAggregation_3D_multiple_non_overlapping(tmp_path: Path) -> None:
 
     # Data and labels should be the same
     assert_array_equal(orig_mean_data, split_mean_data)
-    assert orig_mean["columns"] == split_mean["columns"]
+    assert orig_mean["col_names"] == split_mean["col_names"]
 
 
 def test_ParcelAggregation_3D_multiple_overlapping(tmp_path: Path) -> None:
@@ -446,5 +446,5 @@ def test_ParcelAggregation_3D_multiple_overlapping(tmp_path: Path) -> None:
     assert_array_equal(orig_mean_data, split_mean_data)
 
     # Labels should be "low" for the first 50 and "high" for the second 50
-    assert all(x.startswith("low") for x in split_mean["columns"][:50])
-    assert all(x.startswith("high") for x in split_mean["columns"][50:])
+    assert all(x.startswith("low") for x in split_mean["col_names"][:50])
+    assert all(x.startswith("high") for x in split_mean["col_names"][50:])
-- 
2.52.0


From 2ed8f616089edaa5759ff49acb358762e8fd15a3 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 14:48:00 +0100
Subject: [PATCH 49/51] update: fix unit tests for PandasBaseFeatureStorage

---
 junifer/storage/tests/test_pandas_base.py | 30 +++++++++--------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/junifer/storage/tests/test_pandas_base.py b/junifer/storage/tests/test_pandas_base.py
index 67c6c7767..2bd46a183 100644
--- a/junifer/storage/tests/test_pandas_base.py
+++ b/junifer/storage/tests/test_pandas_base.py
@@ -14,15 +14,12 @@ def test_element_to_index() -> None:
 
     # First test; no extra column
     index = PandasBaseFeatureStorage.element_to_index(element=element)
-    # Check index names
-    assert index.names == ["foo"]
-    # Check first index level name
-    assert index.levels[0].name == "foo"  # type: ignore
-    # Check first index level values
-    assert index.levels[0].values[0] == "bar"  # type: ignore
-    assert all(x == "bar" for x in index.levels[0].values)  # type: ignore
-    # Check first index level values shape
-    assert index.levels[0].values.shape == (1,)  # type: ignore
+    # Check index name
+    assert index.name == "foo"
+    # # Check index values
+    assert all(x == "bar" for x in index.values)  # type: ignore
+    # Check index values shape
+    assert index.shape == (1,)  # type: ignore
 
     # Second test; add extra column
     index = PandasBaseFeatureStorage.element_to_index(
@@ -49,15 +46,12 @@ def test_element_to_index() -> None:
     index = PandasBaseFeatureStorage.element_to_index(
         element=element, n_rows=1, rows_col_name="scan"
     )
-    # Check index names
-    assert index.names == ["foo"]
-    # Check first index level name
-    assert index.levels[0].name == "foo"  # type: ignore
-    # Check first index level values
-    assert index.levels[0].values[0] == "bar"  # type: ignore
-    assert all(x == "bar" for x in index.levels[0].values)  # type: ignore
-    # Check first index level values shape
-    assert index.levels[0].values.shape == (1,)  # type: ignore
+    # Check index name
+    assert index.name == "foo"
+    # Check index values
+    assert all(x == "bar" for x in index.values)  # type: ignore
+    # Check index values shape
+    assert index.shape == (1,)  # type: ignore
 
     # Fourth test; custom extra column name has effect
     index = PandasBaseFeatureStorage.element_to_index(
-- 
2.52.0


From 828701acb494d6eb4a4a4cbed5ccb7f9a62d1b72 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 15:01:50 +0100
Subject: [PATCH 50/51] update: rename columns to col_names in RSSETSMarker

---
 junifer/markers/ets_rss.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/junifer/markers/ets_rss.py b/junifer/markers/ets_rss.py
index 96f50d5d9..763460b19 100644
--- a/junifer/markers/ets_rss.py
+++ b/junifer/markers/ets_rss.py
@@ -134,5 +134,5 @@ class RSSETSMarker(BaseMarker):
         # Compute the RSS
         out["data"] = np.sum(edge_ts**2, 1) ** 0.5
         # Set correct column label
-        out["columns"] = ["root_sum_of_squares_ets"]
+        out["col_names"] = ["root_sum_of_squares_ets"]
         return out
-- 
2.52.0


From d8089ab6e12ac1aae18b2afedf680380b39fcf48 Mon Sep 17 00:00:00 2001
From: Synchon Mandal <synchon@protonmail.com>
Date: Mon, 13 Feb 2023 16:24:31 +0100
Subject: [PATCH 51/51] chore: update latest.inc

---
 docs/changes/latest.inc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/changes/latest.inc b/docs/changes/latest.inc
index 0207cf57d..854fd1a02 100644
--- a/docs/changes/latest.inc
+++ b/docs/changes/latest.inc
@@ -49,4 +49,8 @@ API changes
 ~~~~~~~~~~~
 
 - Add ``confounds_format`` parameter to :class:`junifer.datagrabber.PatternDataGrabber` constructor for
-  improved handling of confounds specified via ``BOLD_confounds`` data type (:gh:`158` by `Synchon Mandal`_).
\ No newline at end of file
+  improved handling of confounds specified via ``BOLD_confounds`` data type (:gh:`158` by `Synchon Mandal`_).
+
+- Rename ``store_table()`` to ``store_vector()`` for storage-like objects and adapt marker-like objects to
+  use ``"vector"`` in place of ``"table"`` for storage. Also, improve the logic of storing vectors
+  (:gh:`181` by `Synchon Mandal`_).
\ No newline at end of file
-- 
2.52.0