From 31f75157b1e88e4e0bae146a07038cf3a830f5c2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 26 Jan 2023 20:12:50 +0100 Subject: [PATCH 001/122] update: add h5io as dependency in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 635f1781e..66dfccfee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "sqlalchemy>=1.4.27,<= 1.5.0", "pyyaml>=5.1.2,<7.0", "importlib_metadata; python_version < '3.10'", + "h5io>=0.1.7,<=0.2.0", ] dynamic = ["version"] -- 2.52.0 From dd60bc8a6320df929265eea830b095433193cad4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 26 Jan 2023 20:20:51 +0100 Subject: [PATCH 002/122] update: add HDF5FeatureStorage implementation --- junifer/storage/hdf5.py | 838 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 838 insertions(+) create mode 100644 junifer/storage/hdf5.py diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py new file mode 100644 index 000000000..63052f430 --- /dev/null +++ b/junifer/storage/hdf5.py @@ -0,0 +1,838 @@ +"""Provide concrete implementation for feature storage via HDF5.""" + +# Authors: Synchon Mandal +# Federico Raimondo +# License: AGPL + + +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Union + +import numpy as np +import pandas as pd +from h5io import read_hdf5, write_hdf5 +from tqdm import tqdm + +from ..api.decorators import register_storage +from ..utils import logger, raise_error +from .base import BaseFeatureStorage +from .utils import element_to_prefix + + +@register_storage +class HDF5FeatureStorage(BaseFeatureStorage): + """Concrete implementation for feature storage via HDF5. + + Parameters + ---------- + uri : str or pathlib.Path + The path to the file to be used. + single_output : bool, optional + If False, will create one HDF5 file per element. The name + of the file will be prefixed with the respective element. + If True, will create only one HDF5 file as specified in the + ``uri`` and store all the elements in the same file. Concurrent + writes should be handled with care (default True). + overwrite : bool or "update", optional + Whether to overwrite existing file. If True, will overwrite and + if "update", will update existing entry or append (default "update"). + compression : {0-9}, optional + Level of gzip compression: 0 (lowest) to 9 (highest) (default 7). + **kwargs : dict + The keyword arguments passed to the superclass. + + See Also + -------- + SQLiteFeatureStorage : The concrete class for SQLite-based feature storage. + + """ + + def __init__( + self, + uri: Union[str, Path], + single_output: bool = True, + overwrite: Union[bool, str] = "update", + compression: int = 7, + **kwargs: str, + ) -> None: + # Convert str to Path + if not isinstance(uri, Path): + uri = Path(uri) + + # Create parent directories if not present + if not uri.parent.exists(): + logger.info( + f"Output directory ({str(uri.parent.absolute())}) " + "does not exist, creating now." + ) + uri.parent.mkdir(parents=True, exist_ok=True) + + # Available storage kinds + storage_types = ["table", "timeseries", "matrix"] + + super().__init__( + uri=uri, + storage_types=storage_types, + single_output=single_output, + **kwargs, + ) + + self.overwrite = overwrite + self.compression = compression + + def get_valid_inputs(self) -> List[str]: + """Get valid storage types for input. + + Returns + ------- + list of str + The list of storage types that can be used as input for this + storage. + + """ + return ["matrix", "table", "timeseries"] + + def _fetch_correct_uri_for_io(self, element: Optional[Dict]) -> str: + """Return proper URI for I/O based on `element`. + + Parameters + ---------- + element : dict, optional + The element as dictionary (default None). + + Returns + ------- + str + Formatted URI for accessing metadata and data. + + """ + prefix = "" + if not self.single_output: + if not element: + raise_error( + msg=( + "`element` must be provided when `single_output` " + "is False" + ), + klass=RuntimeError, + ) + prefix = element_to_prefix(element=element) + # Format URI + return f"{self.uri.parent}/{prefix}{self.uri.name}" # type: ignore + + def _read_metadata( + self, element: Optional[Dict[str, str]] = None + ) -> List[Dict[str, Any]]: + """Read metadata (should not be called directly). + + Parameters + ---------- + element : dict, optional + The element as dictionary (default None). + + Returns + ------- + list of dict + The stored metadata for the element. + + Raises + ------ + IOError + If HDF5 file does not exist. + + """ + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element) + + try: + logger.info(f"Loading HDF5 metadata from: {uri}") + metadata = read_hdf5( + fname=uri, + title="meta", + slash="ignore", + ) + except IOError: + raise_error( + msg=f"HDF5 file not found at: {uri}", + klass=IOError, + ) + else: + logger.info(f"Loaded HDF5 metadata from: {uri}") + # Convert to list if single entry found + if not isinstance(metadata, list): + metadata = [metadata] + return metadata + + def list_features( + self, element: Optional[Dict[str, str]] = None + ) -> Dict[str, Dict[str, Any]]: + """List the features in the storage. + + Parameters + ---------- + element : dict, optional + The element as dictionary (default None). + + Returns + ------- + dict + List of features in the storage. The keys are the feature names to + be used in :func:`read_df` and the values are the metadata of each + feature. + + """ + # Read metadata + metadata = self._read_metadata(element=element) + + # Initialize dictionary to populate + meta_listing = {} + for meta in metadata: + # Make a copy so as not to affect original metadata + meta_copy = meta.copy() + # Format metadata as key-value pair + meta_key = meta_copy.pop("meta_md5") + meta_listing[meta_key] = meta_copy + + return meta_listing + + def _read_data( + self, md5: str, element: Optional[Dict[str, str]] = None + ) -> Dict: + """Read data (should not be called directly). + + Parameters + ---------- + md5 : str + The MD5 used as the HDF5 group name. + element : dict, optional + The element as dictionary (default None). + + Returns + ------- + dict + The retrieved data. + + Raises + ------ + IOError + If HDF5 file does not exist. + + """ + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element) + + try: + logger.info(f"Loading HDF5 data for {md5} from: {uri}") + data = read_hdf5( + fname=uri, + title=md5, + slash="ignore", + ) + except IOError: + raise_error( + msg=f"HDF5 file not found at: {uri}", + klass=IOError, + ) + else: + logger.info(f"Loaded HDF5 data for {md5} from: {uri}") + return data + + def _index_dict_to_multiindex( + self, + index_dict: Dict[Union[str, Iterable[str]], Union[str, Iterable[str]]], + index_columns_order: List[str], + n_rows: int, + ) -> pd.MultiIndex: + """Convert index to pandas.MultiIndex (should not be called directly). + + Parameters + ---------- + index_dict : dict + The index data as returned by + :func:`_element_metadata_to_index_dict`. + index_columns_order: list of str + The ordering for index columns. + n_rows : int + Number of rows for the index. + + Returns + ------- + pandas.MultiIndex + The multi-index for use in :func:`read_df`. + + """ + logger.debug("Converting index dictionary to pandas MultiIndex ...") + # Create multi-index; dict comprehension required to undo keys sorting + # during serialization / deserialization + index = pd.MultiIndex.from_frame( + pd.DataFrame( + data={key: index_dict[key] for key in index_columns_order}, + index=range(n_rows), + ) + ) + logger.debug("Converted index dictionary to pandas MultiIndex ...") + return index + + def read_df( + self, + feature_name: Optional[str] = None, + feature_md5: Optional[bool] = None, + element: Optional[Dict[str, str]] = None, + ) -> pd.DataFrame: + """Read feature into a pandas DataFrame. + + Either one of ``feature_name`` or ``feature_md5`` needs to be + specified. + + Parameters + ---------- + feature_name : str, optional + Name of the feature to read (default None). + feature_md5 : str, optional + MD5 hash of the feature to read (default None). + element : dict, optional + The element as dictionary (default None). + + Returns + ------- + pandas.DataFrame + The features as a dataframe. + + Raises + ------ + IOError + If HDF5 file does not exist. + + """ + # Parameter conflict + if feature_md5 and feature_name: + raise_error( + msg=( + "Only one of `feature_name` or `feature_md5` can be " + "specified." + ) + ) + # Parameter absence + elif not feature_md5 and not feature_name: + raise_error( + msg=( + "At least one of `feature_name` or `feature_md5` " + "must be specified." + ) + ) + # Parameter check pass + else: + # Read metadata + metadata = self._read_metadata(element=element) + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element) + # Initialize MD5 variable + md5: str = "" + + # Consider feature_md5 + if feature_md5 and not feature_name: + logger.debug( + f"Validating feature MD5 '{feature_md5}' in metadata " + f"for: {uri} ..." + ) + # Validate MD5 + for meta in metadata: + if meta["meta_md5"] == feature_md5: + md5 = feature_md5 # type: ignore + if not md5: + raise_error(msg=f"Feature MD5 {feature_md5} not found") + + # Consider feature_name + elif not feature_md5 and feature_name: + logger.debug( + f"Validating feature name '{feature_name}' in metadata " + f"for: {uri} ..." + ) + # Retrieve MD5 for feature_name + # Implicit counter for duplicate feature_name + feature_name_query = [] + for meta in metadata: + if meta["name"] == feature_name: + feature_name_query.append(meta) + + # Check for no / duplicate feature_name + if len(feature_name_query) == 0: + raise_error(msg=f"Feature {feature_name} not found") + elif len(feature_name_query) > 1: + raise_error( + msg=( + f"More than one feature with name {feature_name} " + "found. You can bypass this issue by specifying " + "a `feature_md5`." + ) + ) + + md5 = feature_name_query[0]["meta_md5"] + + # Read data from HDF5 + hdf_data = self._read_data(md5=md5, element=element) + + # Generate index for the data + hdf_data_index = self._index_dict_to_multiindex( + index_dict=hdf_data["idx_data"], + index_columns_order=hdf_data["idx_columns_order"], + n_rows=hdf_data["n_rows"], + ) + + logger.debug( + f"Converting HDF5 data for {md5} to pandas.DataFrame ..." + ) + # Convert to DataFrame + df = pd.DataFrame( + data=hdf_data["data"], + index=hdf_data_index, + columns=hdf_data["column_headers"], + dtype=hdf_data["data"].dtype, + ) + logger.debug( + f"Converted HDF5 data for {md5} to pandas.DataFrame ..." + ) + + return df + + def _write_processed_data( + self, fname: str, processed_data: Dict[str, Any], title: str + ) -> None: + """Write processed data to HDF5 (should not be called directly). + + This is used primarily in :func:`_store_metadata` and + :func:`_store_data`. + + Parameters + ---------- + fname : str + The HDF5 file to store data. + processed_data : dict + The processed data as dictionary. + title : str + The top-level directory name. + + """ + logger.debug(f"Writing processed HDF5 data to: {fname} ...") + # Write to HDF5 + write_hdf5( + fname=fname, + data=processed_data, + overwrite=self.overwrite, # type: ignore + compression=self.compression, + title=title, + slash="error", + use_json=True, + ) + logger.debug(f"Wrote processed HDF5 data to: {fname} ...") + + def store_metadata(self, meta_md5: str, element: Dict, meta: Dict) -> None: + """Store metadata. + + This method first loads existing metadata (if any) using + :func:`_read_metadata` and appends to it the new metadata + and then saves the updated metadata using + :func:`_write_processed_data`. + + Parameters + ---------- + meta_md5 : str + The metadata MD5 hash. + element : dict + The element as a dictionary. + meta : dict + The metadata as a dictionary. + + """ + # Update meta dictionary with MD5 + meta.update({"meta_md5": meta_md5}) + + # Read metadata; if no file found, create an empty list + try: + metadata = self._read_metadata(element=element) + except IOError: + logger.debug(f"Creating new metadata list for {meta_md5} ...") + metadata = [] + + logger.debug(f"HDF5 metadata for {meta_md5} updated ...") + # Update metadata + metadata.append(meta) + + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element) + + logger.info(f"Writing HDF5 metadata for {meta_md5} to: {uri}") + logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") + logger.debug( + f"HDF5 gzip compression level is set to: {self.compression} ..." + ) + + # Write metadata + self._write_processed_data( + fname=uri, + processed_data=meta, + title="meta", + ) + + logger.info(f"Wrote HDF5 metadata for {meta_md5} to: {uri}") + + def _element_metadata_to_index_dict( + self, + element: Dict, + n_rows: int = 1, + row_headers: Union[str, Iterable[str], None] = None, + ) -> Dict[ + str, + Union[Dict[Union[str, Iterable[str]], Union[str, Iterable[str]]], int], + ]: + """Convert the element meta to index (should not be called directly). + + This method is only used in :func:`_store_data`. + + Parameters + ---------- + element : dict + The element as dictionary. + n_rows : int, optional + Number of rows to create (default 1). + row_headers: str, list or tuple of str, optional + The column name to use in case ``n_rows`` > 1. If None and + ``n_rows`` > 1, the name will be "idx" (default None). + + Returns + ------- + dict + The index to store. Is used for creating index for + pandas.DataFrame returned by :func:`read_df`. + + """ + logger.debug( + "Converting element metadata to index dictionary for " + "HDF5 storage ..." + ) + + # Check row_headers + if not row_headers: + row_headers = "idx" + + # Create dictionary with element access options as keys + # and corresponding access values as values for n_rows + element_index: Dict[ + Union[str, Iterable[str]], Union[str, Iterable[str]] + ] = {key: [val] * n_rows for key, val in element.items()} + # Set index value for row_headers + element_index[row_headers] = np.arange(n_rows) + + logger.debug( + "Converted element metadata to index dictionary for " + "HDF5 storage ..." + ) + + return { + "idx_data": element_index, + # required as deserialization sorts keys of the index dict + "idx_columns_order": list(element_index.keys()), # type: ignore + "n_rows": n_rows, + } + + def _store_data( + self, + meta_md5: str, + element: Dict[str, str], + data: Union[np.ndarray, List], + column_headers: Union[str, Iterable[str], None] = None, + row_headers: Union[str, Iterable[str], None] = None, + ) -> None: + """Store data. + + This method first prepares the index for the element's metadata + using :func:`_element_metadata_to_index_dict` and updates + the `data` with the index and then saves the data using + :func:`_write_processed_data`. + + Parameters + ---------- + meta_md5 : str + The metadata MD5 hash. + element : dict + The element as dictionary. + data : numpy.ndarray or list + The data to store. + column_headers : str, list or tuple of str, optional + The column labels (default None). + row_headers : str, list or tuple of str, optional + The column label to use in case number of rows greater than 1. + If None and number of rows greater than 1, then the name will be + "idx". For matrix storage, they are the row labels. (default None). + + """ + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element) + + # Initialize dictionary to aggregate data to write + data_to_write = { + "md5": meta_md5, + "element": element, + "data": data, + "column_headers": column_headers, + "row_headers": row_headers, + } + # Add index data to store + index_data = self._element_metadata_to_index_dict( + element=element, + n_rows=len(data), + row_headers=row_headers, + ) + data_to_write.update(index_data) # type: ignore + + logger.info(f"Writing HDF5 data to: {uri}") + logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") + logger.debug( + f"HDF5 gzip compression level is set to: {self.compression} ..." + ) + + # Write data + self._write_processed_data( + fname=uri, + processed_data=data_to_write, + title=meta_md5, + ) + + logger.info(f"Wrote HDF5 data to: {uri}") + + def store_matrix( + self, + meta_md5: str, + element: Dict[str, str], + data: np.ndarray, + col_names: Optional[Iterable[str]] = None, + row_names: Optional[Iterable[str]] = None, + matrix_kind: Optional[str] = "full", + diagonal: bool = True, + ) -> None: + """Store matrix. + + This method performs parameter checks and then calls + :func:`_store_data` for storing the data. + + Parameters + ---------- + meta_md5 : str + The metadata MD5 hash. + element : dict + The element as dictionary. + data : numpy.ndarray + The matrix data to store. + col_names : list or tuple of str, optional + The column labels (default None). + row_names : list or tuple of str, optional + The row labels (default None). + matrix_kind : str, optional + The kind of matrix: + + * ``triu`` : store upper triangular only + * ``tril`` : store lower triangular + * ``full`` : full matrix + + (default "full"). + diagonal : bool, optional + Whether to store the diagonal. If `matrix_kind` is "full", setting + this to False will raise an error (default True). + + """ + # diagonal validation + if diagonal is False and matrix_kind not in ["triu", "tril"]: + raise_error( + msg="Diagonal cannot be False if kind is not full", + klass=ValueError, + ) + # matrix_kind and shape validation + if matrix_kind in ["triu", "tril"]: + if data.shape[0] != data.shape[1]: + raise_error( + "Cannot store a non-square matrix as a triangular matrix", + klass=ValueError, + ) + # matrix_kind validation and computation + if matrix_kind == "triu": + k = 0 if diagonal is True else 1 + data_idx = np.triu_indices(data.shape[0], k=k) + elif matrix_kind == "tril": + k = 0 if diagonal is True else -1 + data_idx = np.tril_indices(data.shape[0], k=k) + elif matrix_kind == "full": + data_idx = ( + np.repeat(np.arange(data.shape[0]), data.shape[1]), + np.tile(np.arange(data.shape[1]), data.shape[0]), + ) + else: + raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError) + + # Row data and label validation + if row_names is None: + row_names = [f"r{i}" for i in range(data.shape[0])] + elif len(row_names) != data.shape[0]: # type: ignore + raise_error( + msg="Number of row names does not match number of rows", + klass=ValueError, + ) + + # Column data and label validation + if col_names is None: + col_names = [f"c{i}" for i in range(data.shape[1])] + elif len(col_names) != data.shape[1]: # type: ignore + raise_error( + msg="Number of column names does not match number of columns", + klass=ValueError, + ) + + # Subset data + flat_data = data[data_idx] + # Format column names + columns = [ + f"{row_names[i]}~{col_names[j]}" # type: ignore + for i, j in zip(data_idx[0], data_idx[1]) + ] + # Store + self._store_data( + meta_md5=meta_md5, + element=element, + data=flat_data[None, :], + column_headers=columns, + row_headers=None, + ) + + def store_table( + self, + meta_md5: str, + element: Dict[str, str], + data: Union[np.ndarray, List], + columns: Optional[Iterable[str]] = None, + rows_col_name: Optional[str] = None, + ) -> None: + """Store table. + + Parameters + ---------- + meta_md5 : str + The metadata MD5 hash. + element : dict + The element as dictionary. + data : numpy.ndarray or list + The table data to store. + columns : list or tuple of str, optional + The columns (default None). + rows_col_name : str, optional + The column name to use in case number of rows greater than 1. + If None and number of rows greater than 1, then the name will be + "index" (default None). + + """ + self._store_data( + meta_md5=meta_md5, + element=element, + data=data, + column_headers=columns, + row_headers=rows_col_name, + ) + + def store_timeseries( + self, + meta_md5: str, + element: Dict[str, str], + data: np.ndarray, + columns: Optional[Iterable[str]] = None, + ) -> None: + """Implement timeseries storing. + + Parameters + ---------- + meta_md5 : str + The metadata MD5 hash. + element : dict + The element as dictionary. + data : numpy.ndarray + The timeseries data to store. + columns : list or tuple of str, optional + The column labels (default None). + + """ + self._store_data( + meta_md5=meta_md5, + element=element, + data=data, + column_headers=columns, + row_headers="timepoint", + ) + + def collect(self) -> None: + """Implement data collection. + + This method globs the element files and runs a loop + over them while reading metadata and then runs a loop + over all the stored features in the metadata, storing + the metadata and the feature data right after reading. + + Raises + ------ + NotImplementedError + If ``single_output`` is True. + + """ + if self.single_output is True: + raise_error( + msg="collect() is not implemented for single output.", + klass=NotImplementedError, + ) + + logger.info( + "Collecting data from " + f"{self.uri.parent}/*{self.uri.name}" # type: ignore + ) + + # Create new storage instance + out_storage = HDF5FeatureStorage(uri=self.uri, overwrite="update") + + # Glob files + globbed_files = self.uri.parent.glob( + f"*{self.uri.name}" + ) # type: ignore + + # Run loop to aggregate + for file in tqdm(globbed_files, desc="file"): + logger.debug(f"Reading HDF5 file: {file.resolve()} ...") + # Create new storage instance to load data + in_storage = HDF5FeatureStorage(uri=file) + + # Load metadata from new instance + in_metadata = in_storage._read_metadata() + + logger.info( + f"Updating HDF5 metadata with metadata from: {file.resolve()}" + ) + # Save metadata + out_storage._write_processed_data( + fname=self.uri.resolve(), + processed_data=in_metadata, + title="meta", + ) + + for meta in tqdm(in_metadata, desc="feature"): + logger.debug(f"Collecting feature MD5: {meta['meta_md5']} ...") + # Load data from new instance + in_data = in_storage._read_data(md5=meta["meta_md5"]) + + logger.info( + f"Updating HDF5 data with data from: {meta['meta_md5']}" + ) + # Save data + out_storage._write_processed_data( + fname=self.uri.resolve(), + processed_data=in_data, + title=meta["meta_md5"], + ) -- 2.52.0 From f7e26157057998632e104d49ec4509d8e7411827 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 26 Jan 2023 20:21:45 +0100 Subject: [PATCH 003/122] update: add HDF5FeatureStorage import in storage.__init__ --- junifer/storage/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/junifer/storage/__init__.py b/junifer/storage/__init__.py index b8c4f8942..1e756e6cc 100644 --- a/junifer/storage/__init__.py +++ b/junifer/storage/__init__.py @@ -7,3 +7,4 @@ from .base import BaseFeatureStorage from .pandas_base import PandasBaseFeatureStorage from .sqlite import SQLiteFeatureStorage +from .hdf5 import HDF5FeatureStorage -- 2.52.0 From 2eef5367bde17726e2b91321bea52f31ef0e0e05 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 27 Jan 2023 18:33:37 +0100 Subject: [PATCH 004/122] update: improve data and metadata read for multi output HDF5FeatureStorage after collect() --- junifer/storage/hdf5.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 63052f430..608557ac5 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -79,6 +79,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): self.overwrite = overwrite self.compression = compression + # Tracks whether to bypass element checks after collection + self._collected = False def get_valid_inputs(self) -> List[str]: """Get valid storage types for input. @@ -108,7 +110,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): """ prefix = "" if not self.single_output: - if not element: + if not element and not self._collected: raise_error( msg=( "`element` must be provided when `single_output` " @@ -116,7 +118,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): ), klass=RuntimeError, ) - prefix = element_to_prefix(element=element) + elif element and not self._collected: + prefix = element_to_prefix(element=element) # Format URI return f"{self.uri.parent}/{prefix}{self.uri.name}" # type: ignore @@ -836,3 +839,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): processed_data=in_data, title=meta["meta_md5"], ) + + # Toggle check to bypass element checks + self._collected = True -- 2.52.0 From 1fed4527a8d9b033f07a0c6c28f25de4563aa991 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 27 Jan 2023 18:34:37 +0100 Subject: [PATCH 005/122] update: append metadata for multi output HDF5FeatureStorage.collect() --- junifer/storage/hdf5.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 608557ac5..601120a6a 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -818,10 +818,17 @@ class HDF5FeatureStorage(BaseFeatureStorage): logger.info( f"Updating HDF5 metadata with metadata from: {file.resolve()}" ) + # Load metadata; empty list if first entry + try: + out_metadata = out_storage._read_metadata() + except IOError: + out_metadata = [] + # Update metadata + out_metadata.append(in_metadata) # Save metadata out_storage._write_processed_data( fname=self.uri.resolve(), - processed_data=in_metadata, + processed_data=out_metadata, title="meta", ) -- 2.52.0 From a583f388a3799e990b50fd436673c3c8d0da5db2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 27 Jan 2023 18:35:21 +0100 Subject: [PATCH 006/122] update: add tests for HDF5FeatureStorage --- junifer/storage/tests/test_hdf5.py | 765 +++++++++++++++++++++++++++++ 1 file changed, 765 insertions(+) create mode 100644 junifer/storage/tests/test_hdf5.py diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py new file mode 100644 index 000000000..aa14fd02e --- /dev/null +++ b/junifer/storage/tests/test_hdf5.py @@ -0,0 +1,765 @@ +"""Provide tests for HDF5 storage interface.""" + +# Authors: Synchon Mandal +# Federico Raimondo +# License: AGPL + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from numpy.testing import assert_array_equal +from pandas.testing import assert_frame_equal + +from junifer.storage import HDF5FeatureStorage +from junifer.storage.utils import element_to_prefix, process_meta + + +def test_get_valid_inputs() -> None: + """Test valid inputs.""" + storage = HDF5FeatureStorage(uri="/tmp") + assert storage.get_valid_inputs() == ["matrix", "table", "timeseries"] + + +def test_single_output(tmp_path: Path) -> None: + """Test engine retrieval with single output. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_single_output.hdf5" + # Single storage, must be the uri + storage = HDF5FeatureStorage(uri=uri, single_output=True) + assert storage.single_output is True + + +def test_multi_output_error(tmp_path: Path) -> None: + """Test error for multi output. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_multi_output.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=False) + with pytest.raises(RuntimeError, match="`element` must be provided"): + storage._fetch_correct_uri_for_io(element=None) + + +def test_single_output_parent_path_creation(tmp_path: Path) -> None: + """Test path creation with single output creation. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + to_create_hdf5 = tmp_path / "to_create_hdf5" + # Path does not exist yet + assert not to_create_hdf5.exists() + + uri = to_create_hdf5.absolute() / "test_single_output.hdf5" + _ = HDF5FeatureStorage(uri=uri, single_output=True) + # Path exists now + assert to_create_hdf5.exists() + + +def test_store_metadata_and_list_features(tmp_path: Path) -> None: + """Test metadata store and features listing. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_metadata_store.hdf5" + # Single storage, must be the uri + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "test"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + # List the stored features + features = storage.list_features() + # Get the first MD5 + feature_md5 = list(features.keys())[0] + # Check the MD5 + assert meta_md5 == feature_md5 + + +def test_store_matrix(tmp_path: Path) -> None: + """Test matrix store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_matrix.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Store 4 X 3 full matrix + data = np.array( + [[1, 2, 3], [11, 22, 33], [111, 222, 333], [1111, 2222, 3333]] + ) + row_headers = ["row1", "row2", "row3", "row4"] + col_headers = ["col1", "col2", "col3"] + + # Store matrix + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + ) + + # List the stored features + features = storage.list_features() + # Get the first MD5 + feature_md5 = list(features.keys())[0] + # Check the MD5 + assert "BOLD_fc" == features[feature_md5]["name"] + + # Read into dataframe + read_df = storage.read_df(feature_md5=feature_md5) + # Check shape of dataframe + assert read_df.shape == (1, 12) + # Check data of dataframe + assert_array_equal(read_df.values[0], data.flatten()) + + # Format column headers + stored_col_headers = [f"{i}~{j}" for i in row_headers for j in col_headers] + # Check column headers + assert list(read_df.columns) == stored_col_headers + + # Check for errors + with pytest.raises(ValueError, match="Invalid kind"): + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="wrong", + ) + + with pytest.raises(ValueError, match="non-square"): + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="triu", + ) + + with pytest.raises(ValueError, match="cannot be False"): + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="full", + diagonal=False, + ) + + +def test_store_matrix_without_headers(tmp_path: Path) -> None: + """Test matrix store without headers. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_matrix_no_headers.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Store 4 X 3 full matrix + data = np.array( + [[1, 2, 3], [11, 22, 33], [111, 222, 333], [1111, 2222, 3333]] + ) + + # Store matrix + storage.store_matrix( + meta_md5=meta_md5, element=element_to_store, data=data + ) + + # List the stored features + features = storage.list_features() + # Get the first MD5 + feature_md5 = list(features.keys())[0] + # Check the MD5 + assert "BOLD_fc" == features[feature_md5]["name"] + + # Read the dataframe + read_df = storage.read_df(feature_md5=feature_md5) + # Check shape of dataframe + assert read_df.shape == (1, 12) + # Check data of dataframe + assert_array_equal(read_df.values[0], data.flatten()) + + # Format column headers + stored_col_headers = [ + f"r{i}~c{j}" + for i in range(data.shape[0]) + for j in range(data.shape[1]) + ] + # Check column headers + assert list(read_df.columns) == stored_col_headers + + +def test_store_upper_triangular_matrix(tmp_path: Path) -> None: + """Test upper triangular matrix store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_matrix_triu.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Store upper triangular matrix + data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]]) + row_headers = ["row1", "row2", "row3"] + col_headers = ["col1", "col2", "col3"] + + # Store matrix + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="triu", + ) + + # List the stored features + features = storage.list_features() + # Get the first MD5 + feature_md5 = list(features.keys())[0] + # Check the MD5 + assert "BOLD_fc" == features[feature_md5]["name"] + + # Read into dataframe + read_df = storage.read_df(feature_md5=feature_md5) + # Check data of dataframe + assert_array_equal( + read_df.values, data[np.triu_indices(n=data.shape[0])][None, :] + ) + + # Format column headers + stored_col_headers = [ + "row1~col1", + "row1~col2", + "row1~col3", + "row2~col2", + "row2~col3", + "row3~col3", + ] + # Check column headers + assert list(read_df.columns) == stored_col_headers + + +def test_store_upper_triangular_matrix_without_diagonal( + tmp_path: Path, +) -> None: + """Test upper triangular matrix without diagonal store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_matrix_triu_no_diagonal.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Store upper triangular matrix without diagonal + data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]]) + row_headers = ["row1", "row2", "row3"] + col_headers = ["col1", "col2", "col3"] + + # Store matrix + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="triu", + diagonal=False, + ) + + # List the stored features + features = storage.list_features() + # Get the first MD5 + feature_md5 = list(features.keys())[0] + # Check the MD5 + assert "BOLD_fc" == features[feature_md5]["name"] + + # Read into dataframe + read_df = storage.read_df(feature_md5=feature_md5) + # Check data of dataframe + assert_array_equal( + read_df.values, data[np.triu_indices(n=data.shape[0], k=1)][None, :] + ) + + # Format column headers + stored_col_headers = [ + "row1~col2", + "row1~col3", + "row2~col3", + ] + # Check column headers + assert list(read_df.columns) == stored_col_headers + + +def test_store_lower_triangular_matrix(tmp_path: Path) -> None: + """Test lower triangular matrix store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_matrix_tril.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Store lower triangular matrix + data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]]) + row_headers = ["row1", "row2", "row3"] + col_headers = ["col1", "col2", "col3"] + + # Store matrix + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="tril", + ) + + # List the stored features + features = storage.list_features() + # Get the first MD5 + feature_md5 = list(features.keys())[0] + # Check the MD5 + assert "BOLD_fc" == features[feature_md5]["name"] + + # Read into dataframe + read_df = storage.read_df(feature_md5=feature_md5) + # Check data of dataframe + assert_array_equal( + read_df.values, data[np.tril_indices(n=data.shape[0])][None, :] + ) + + # Format column headers + stored_col_headers = [ + "row1~col1", + "row2~col1", + "row2~col2", + "row3~col1", + "row3~col2", + "row3~col3", + ] + # Check column headers + assert list(read_df.columns) == stored_col_headers + + +def test_store_lower_triangular_matrix_without_diagonal( + tmp_path: Path, +) -> None: + """Test lower triangular matrix without diagonal store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_matrix_tril_no_diagonal.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Store lower triangular matrix without diagonal + data = np.array([[1, 2, 3], [11, 22, 33], [111, 222, 333]]) + row_headers = ["row1", "row2", "row3"] + col_headers = ["col1", "col2", "col3"] + + # Store matrix + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="tril", + diagonal=False, + ) + + # List the stored features + features = storage.list_features() + # Get the first MD5 + feature_md5 = list(features.keys())[0] + # Check the MD5 + assert "BOLD_fc" == features[feature_md5]["name"] + + # Read into dataframe + read_df = storage.read_df(feature_md5=feature_md5) + # Check data of dataframe + assert_array_equal( + read_df.values, data[np.tril_indices(n=data.shape[0], k=-1)][None, :] + ) + + # Format column headers + stored_col_headers = [ + "row2~col1", + "row3~col1", + "row3~col2", + ] + # Check column headers + assert list(read_df.columns) == stored_col_headers + + +def test_store_table(tmp_path: Path) -> None: + """Test table store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_table.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + element = {"subject": "test"} + meta = { + "element": element, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Data to store + data = np.array( + [ + [1, 10], + [2, 20], + [3, 30], + [4, 40], + [5, 50], + ] + ) + row_header = "scan" + col_headers = ["f1", "f2"] + + # Store table + storage.store_table( + meta_md5=meta_md5, + element=element_to_store, + data=data, + columns=col_headers, + rows_col_name=row_header, + ) + + # Convert element to index dict + idx_dict = storage._element_metadata_to_index_dict( + element=element, + n_rows=5, + row_headers="scan", + ) + # Convert index dict to index + idx = storage._index_dict_to_multiindex( + index_dict=idx_dict["idx_data"], # type: ignore + index_columns_order=idx_dict["idx_columns_order"], # type: ignore + n_rows=idx_dict["n_rows"], # type: ignore + ) + # Create dataframe + df = pd.DataFrame(data, columns=["f1", "f2"], index=idx) + + # Read into dataframe + read_df = storage.read_df(feature_md5=meta_md5) + # Check if dataframes are equal + assert_frame_equal(df, read_df) + + # New data to store + data_new = np.array( + [[1, 10], [2, 20], [3, 300], [4, 40], [5, 50], [6, 600]] + ) + # Convert element to index dict + idx_dict_new = storage._element_metadata_to_index_dict( + element=element, + n_rows=6, + row_headers="scan", + ) + # Convert index dict to index + idx_new = storage._index_dict_to_multiindex( + index_dict=idx_dict_new["idx_data"], # type: ignore + index_columns_order=idx_dict_new["idx_columns_order"], # type: ignore + n_rows=idx_dict_new["n_rows"], # type: ignore + ) + # Create new dataframe + df_new = pd.DataFrame(data_new, columns=["f1", "f2"], index=idx_new) + + # Store table + storage.store_table( + meta_md5=meta_md5, + element=element_to_store, + data=data_new, + columns=["f1", "f2"], + rows_col_name="scan", + ) + + # Read into dataframe + read_df_new = storage.read_df(feature_md5=meta_md5) + # Check if dataframes are equal + assert_frame_equal(df_new, read_df_new) + + +def test_multi_output_store_and_collect(tmp_path: Path): + """Test multi output storing and collection. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_multi_output_store_and_collect.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=False) + + # Metadata to store + meta_1 = { + "element": {"subject": "test-01", "session": "ses-01"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + meta_2 = { + "element": {"subject": "test-02", "session": "ses-01"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + meta_3 = { + "element": {"subject": "test-01", "session": "ses-02"}, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + + # Data to store + data_1 = np.array( + [ + [1, 10], + [2, 20], + [3, 30], + [4, 40], + [5, 50], + ] + ) + data_2 = data_1 * 10 + data_3 = data_1 * 20 + row_header = "scan" + col_headers = ["f1", "f2"] + + # Process metadata for storage + hash_1, meta_to_store_1, element_to_store_1 = process_meta(meta_1) + + # Process metadata for storage + hash_2, meta_to_store_2, element_to_store_2 = process_meta(meta_2) + + # Process metadata for storage + hash_3, meta_to_store_3, element_to_store_3 = process_meta(meta_3) + + # Check hash equality as element is not considered for hash + assert hash_1 == hash_2 + assert hash_2 == hash_3 + + # Store metadata for tables + storage.store_metadata( + meta_md5=hash_1, + element=element_to_store_1, + meta=meta_to_store_1, + ) + storage.store_metadata( + meta_md5=hash_2, + element=element_to_store_2, + meta=meta_to_store_2, + ) + storage.store_metadata( + meta_md5=hash_3, + element=element_to_store_3, + meta=meta_to_store_3, + ) + + # Store tables + storage.store_table( + meta_md5=hash_1, + element=element_to_store_1, + data=data_1, + columns=col_headers, + rows_col_name=row_header, + ) + storage.store_table( + meta_md5=hash_2, + element=element_to_store_2, + data=data_2, + columns=col_headers, + rows_col_name=row_header, + ) + storage.store_table( + meta_md5=hash_3, + element=element_to_store_3, + data=data_3, + columns=col_headers, + rows_col_name=row_header, + ) + + # Check that base URI does not exist yet + assert not uri.exists() + + # Convert element to preifx + prefix_1 = element_to_prefix(meta_1["element"]) # type: ignore + prefix_2 = element_to_prefix(meta_2["element"]) # type: ignore + prefix_3 = element_to_prefix(meta_3["element"]) # type: ignore + + # URIs for data storage + uri_1 = uri.parent / f"{prefix_1}{uri.name}" + uri_2 = uri.parent / f"{prefix_2}{uri.name}" + uri_3 = uri.parent / f"{prefix_3}{uri.name}" + + # Check URIs for data storage exist + assert uri_1.exists() + assert uri_2.exists() + assert uri_3.exists() + + # Read stored metadata from different files using element + read_meta_1 = storage._read_metadata(element=meta_1["element"]) + read_meta_2 = storage._read_metadata(element=meta_2["element"]) + read_meta_3 = storage._read_metadata(element=meta_3["element"]) + + # Check if metadata are equal + assert read_meta_1 == read_meta_2 + assert read_meta_2 == read_meta_3 + + # Collect data + storage.collect() + # Check that base URI exists now + assert uri.exists() + + # Read unified metadata + read_unified_meta = storage._read_metadata() + + # Check if aggregated metadata are equal + assert read_unified_meta == [read_meta_1, read_meta_2, read_meta_3] -- 2.52.0 From ccec087fce574834b44ecdf78ae590902a439999 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 30 Jan 2023 13:17:22 +0100 Subject: [PATCH 007/122] fix: remove sphinx directives for private methods in HDF5FeatureStorage --- junifer/storage/hdf5.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 601120a6a..f64c5c78d 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -406,8 +406,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): ) -> None: """Write processed data to HDF5 (should not be called directly). - This is used primarily in :func:`_store_metadata` and - :func:`_store_data`. + This is used primarily in ``_store_metadata`` and ``_store_data``. Parameters ---------- @@ -436,9 +435,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): """Store metadata. This method first loads existing metadata (if any) using - :func:`_read_metadata` and appends to it the new metadata - and then saves the updated metadata using - :func:`_write_processed_data`. + ``_read_metadata`` and appends to it the new metadata and then saves + the updated metadata using ``_write_processed_data``. Parameters ---------- @@ -494,7 +492,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): ]: """Convert the element meta to index (should not be called directly). - This method is only used in :func:`_store_data`. + This method is only used in ``_store_data``. Parameters ---------- @@ -621,7 +619,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): """Store matrix. This method performs parameter checks and then calls - :func:`_store_data` for storing the data. + ``_store_data`` for storing the data. Parameters ---------- -- 2.52.0 From e5abad916e5d000cc8a88c71d347f7f9719a790c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 30 Jan 2023 14:53:09 +0100 Subject: [PATCH 008/122] update: add more tests for HDF5FeatureStorage --- junifer/storage/tests/test_hdf5.py | 151 +++++++++++++++++++++++++---- 1 file changed, 130 insertions(+), 21 deletions(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index aa14fd02e..6feeb9230 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -104,6 +104,85 @@ def test_store_metadata_and_list_features(tmp_path: Path) -> None: assert meta_md5 == feature_md5 +def test_read_df_params_error() -> None: + """Test parameter validation errors for read_df.""" + storage = HDF5FeatureStorage(uri="/tmp") + + with pytest.raises(ValueError, match="Only one of"): + storage.read_df(feature_name="name", feature_md5="md5") + + with pytest.raises(ValueError, match="At least one of"): + storage.read_df() + + +def test_read_df(tmp_path: Path) -> None: + """Test read_df. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_read_df.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "test"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + # Data to store + data = np.array( + [ + [1, 10], + [2, 20], + ] + ) + row_header = "scan" + col_headers = ["f1", "f2"] + # Store table + storage.store_table( + meta_md5=meta_md5, + element=element_to_store, + data=data, + columns=col_headers, + rows_col_name=row_header, + ) + # Read into dataframe and check + df_md5 = storage.read_df(feature_md5=meta_md5) + df_name = storage.read_df(feature_name="BOLD_test") + assert_frame_equal(df_md5, df_name) + + # Check for errors about no / duplicate feature name + with pytest.raises(ValueError, match="not found"): + storage.read_df(feature_name="BOLD") + + # TODO: fix this check + # # Store duplicate entry and check error + # storage.store_metadata( + # meta_md5=meta_md5, + # element={"subject": "test-clone"}, + # meta=meta_to_store, + # ) + # storage.store_table( + # meta_md5=meta_md5, + # element={"subject": "test-clone"}, + # data=data, + # columns=col_headers, + # rows_col_name=row_header, + # ) + # with pytest.raises(ValueError, match="More than one feature with name"): + # storage.read_df(feature_name="BOLD_test") + + def test_store_matrix(tmp_path: Path) -> None: """Test matrix store. @@ -164,27 +243,7 @@ def test_store_matrix(tmp_path: Path) -> None: # Check column headers assert list(read_df.columns) == stored_col_headers - # Check for errors - with pytest.raises(ValueError, match="Invalid kind"): - storage.store_matrix( - meta_md5=meta_md5, - element=element_to_store, - data=data, - row_names=row_headers, - col_names=col_headers, - matrix_kind="wrong", - ) - - with pytest.raises(ValueError, match="non-square"): - storage.store_matrix( - meta_md5=meta_md5, - element=element_to_store, - data=data, - row_names=row_headers, - col_names=col_headers, - matrix_kind="triu", - ) - + # Diagonal validation error with pytest.raises(ValueError, match="cannot be False"): storage.store_matrix( meta_md5=meta_md5, @@ -195,6 +254,46 @@ def test_store_matrix(tmp_path: Path) -> None: matrix_kind="full", diagonal=False, ) + # Matrix kind and shape validation + with pytest.raises(ValueError, match="non-square"): + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="triu", + ) + # Matrix kind validation + with pytest.raises(ValueError, match="Invalid kind"): + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=col_headers, + matrix_kind="wrong", + ) + # Row data and label validation + with pytest.raises(ValueError, match="of row names"): + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=["row1", "row2", "row3"], + col_names=col_headers, + matrix_kind="full", + ) + # Column data and label validation + with pytest.raises(ValueError, match="of column names"): + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=data, + row_names=row_headers, + col_names=["col1", "col2", "col3", "col4"], + matrix_kind="full", + ) def test_store_matrix_without_headers(tmp_path: Path) -> None: @@ -763,3 +862,13 @@ def test_multi_output_store_and_collect(tmp_path: Path): # Check if aggregated metadata are equal assert read_unified_meta == [read_meta_1, read_meta_2, read_meta_3] + + +def test_collect_error_single_output() -> None: + """Test error for collect in single output storage.""" + with pytest.raises( + NotImplementedError, + match="is not implemented for single output.", + ): + storage = HDF5FeatureStorage(uri="/tmp", single_output=True) + storage.collect() -- 2.52.0 From d0763421cf384abde32a8ffb54346c2b93821985 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 12:02:13 +0100 Subject: [PATCH 009/122] update: remove element param from HDF5FeatureStorage.list_features() --- junifer/storage/hdf5.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index f64c5c78d..d98e32068 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -167,16 +167,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): metadata = [metadata] return metadata - def list_features( - self, element: Optional[Dict[str, str]] = None - ) -> Dict[str, Dict[str, Any]]: + def list_features(self) -> Dict[str, Dict[str, Any]]: """List the features in the storage. - Parameters - ---------- - element : dict, optional - The element as dictionary (default None). - Returns ------- dict @@ -186,7 +179,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): """ # Read metadata - metadata = self._read_metadata(element=element) + metadata = self._read_metadata() # Initialize dictionary to populate meta_listing = {} -- 2.52.0 From ee24c9b3ff5367489c8b722fd961433cfab95327 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 12:03:26 +0100 Subject: [PATCH 010/122] chore: update docstring for HDF5FeatureStorage.list_features() --- junifer/storage/hdf5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index d98e32068..66e3d4aa7 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -173,9 +173,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): Returns ------- dict - List of features in the storage. The keys are the feature names to - be used in :func:`read_df` and the values are the metadata of each - feature. + List of features in the storage. The keys are the feature MD5 to + be used in :meth:`junifer.storage.HDF5FeatureStorage.read_df` + and the values are the metadata of each feature. """ # Read metadata -- 2.52.0 From 2413d9d5d87015729abadfce2a56b93d34bbcf5a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 12:13:47 +0100 Subject: [PATCH 011/122] update: remove _collected flag and refine correct HDF5 URI fetching for HDF5FeatureStorage --- junifer/storage/hdf5.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 66e3d4aa7..db2032ead 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -79,8 +79,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): self.overwrite = overwrite self.compression = compression - # Tracks whether to bypass element checks after collection - self._collected = False def get_valid_inputs(self) -> List[str]: """Get valid storage types for input. @@ -97,6 +95,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): def _fetch_correct_uri_for_io(self, element: Optional[Dict]) -> str: """Return proper URI for I/O based on `element`. + If `element` is None, will return `self.uri`. + Parameters ---------- element : dict, optional @@ -108,19 +108,21 @@ class HDF5FeatureStorage(BaseFeatureStorage): Formatted URI for accessing metadata and data. """ - prefix = "" - if not self.single_output: - if not element and not self._collected: - raise_error( - msg=( - "`element` must be provided when `single_output` " - "is False" - ), - klass=RuntimeError, - ) - elif element and not self._collected: - prefix = element_to_prefix(element=element) - # Format URI + if not self.single_output and not element: + raise_error( + msg=( + "`element` must be provided when `single_output` " + "is False" + ), + klass=RuntimeError, + ) + elif not self.single_output and element: + # element access for multi output only + prefix = element_to_prefix(element=element) + else: + # parent access for single output, ignore element + prefix = "" + # Format URI based on prefix return f"{self.uri.parent}/{prefix}{self.uri.name}" # type: ignore def _read_metadata( @@ -837,6 +839,3 @@ class HDF5FeatureStorage(BaseFeatureStorage): processed_data=in_data, title=meta["meta_md5"], ) - - # Toggle check to bypass element checks - self._collected = True -- 2.52.0 From 22253ae07314275f11853d54f7eebae86ce2d4b0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 12:15:47 +0100 Subject: [PATCH 012/122] chore: add HDF5FeatureStorage ref in "See Also" of SQLiteFeatureStorage --- junifer/storage/sqlite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 706dc64ae..7f6b83eb6 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -49,6 +49,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): See Also -------- PandasBaseFeatureStorage : The base class for Pandas-based feature storage. + HDF5FeatureStorage : The concrete class for HDF5-based feature storage. """ -- 2.52.0 From d29a0b534e785ac6befb3bcb0eab6a461f671595 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 14:14:51 +0100 Subject: [PATCH 013/122] chore: improve docstring for HDF5FeatureStorage.store_matrix() --- junifer/storage/hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index db2032ead..64847d1d5 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -637,8 +637,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): (default "full"). diagonal : bool, optional - Whether to store the diagonal. If `matrix_kind` is "full", setting - this to False will raise an error (default True). + Whether to store the diagonal. If ``matrix_kind`` is "full", + setting this to False will raise an error (default True). """ # diagonal validation -- 2.52.0 From fb7815d64ee7f20fa80b210d6f6f2518c2beee65 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 14:16:04 +0100 Subject: [PATCH 014/122] update: rename table to vector in HDF5FeatureStorage --- junifer/storage/hdf5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 64847d1d5..48148eb8c 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -703,7 +703,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): row_headers=None, ) - def store_table( + def store_vector( self, meta_md5: str, element: Dict[str, str], @@ -711,7 +711,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): columns: Optional[Iterable[str]] = None, rows_col_name: Optional[str] = None, ) -> None: - """Store table. + """Store vector. Parameters ---------- @@ -720,13 +720,13 @@ class HDF5FeatureStorage(BaseFeatureStorage): element : dict The element as dictionary. data : numpy.ndarray or list - The table data to store. columns : list or tuple of str, optional The columns (default None). rows_col_name : str, optional The column name to use in case number of rows greater than 1. If None and number of rows greater than 1, then the name will be "index" (default None). + The vector data to store. """ self._store_data( -- 2.52.0 From 6ad01d2c1363137295f6221735202f7bd275c209 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 14:23:02 +0100 Subject: [PATCH 015/122] update: replace columns and rows_col_name with col_names in HDF5FeatureStorage.store_vector() --- junifer/storage/hdf5.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 48148eb8c..c91183c95 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -708,8 +708,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): meta_md5: str, element: Dict[str, str], data: Union[np.ndarray, List], - columns: Optional[Iterable[str]] = None, - rows_col_name: Optional[str] = None, + col_names: Optional[Iterable[str]] = None, ) -> None: """Store vector. @@ -720,21 +719,16 @@ class HDF5FeatureStorage(BaseFeatureStorage): element : dict The element as dictionary. data : numpy.ndarray or list - columns : list or tuple of str, optional - The columns (default None). - rows_col_name : str, optional - The column name to use in case number of rows greater than 1. - If None and number of rows greater than 1, then the name will be - "index" (default None). The vector data to store. + col_names : list or tuple of str, optional + The column labels (default None). """ self._store_data( meta_md5=meta_md5, element=element, data=data, - column_headers=columns, - row_headers=rows_col_name, + column_headers=col_names, ) def store_timeseries( -- 2.52.0 From 48fd2a36b381be14f1e1b5da96cdcf6a1abec34e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 14:23:43 +0100 Subject: [PATCH 016/122] update: rename columns to col_names in HDF5FeatureStorage.store_timeseries() --- junifer/storage/hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index c91183c95..2e08950e7 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -736,7 +736,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): meta_md5: str, element: Dict[str, str], data: np.ndarray, - columns: Optional[Iterable[str]] = None, + col_names: Optional[Iterable[str]] = None, ) -> None: """Implement timeseries storing. @@ -756,7 +756,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): meta_md5=meta_md5, element=element, data=data, - column_headers=columns, + column_headers=col_names, row_headers="timepoint", ) -- 2.52.0 From 83610a5767da2990216c61a6480ea712174883a3 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 14:57:25 +0100 Subject: [PATCH 017/122] update: remove element param from HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 2e08950e7..7ebb0ae6b 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -277,7 +277,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): self, feature_name: Optional[str] = None, feature_md5: Optional[bool] = None, - element: Optional[Dict[str, str]] = None, ) -> pd.DataFrame: """Read feature into a pandas DataFrame. @@ -290,8 +289,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): Name of the feature to read (default None). feature_md5 : str, optional MD5 hash of the feature to read (default None). - element : dict, optional - The element as dictionary (default None). Returns ------- @@ -323,10 +320,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Parameter check pass else: # Read metadata - metadata = self._read_metadata(element=element) - # Get correct URI for element; - # is different from uri if single_output is False - uri = self._fetch_correct_uri_for_io(element=element) + metadata = self._read_metadata() # Initialize MD5 variable md5: str = "" @@ -334,7 +328,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): if feature_md5 and not feature_name: logger.debug( f"Validating feature MD5 '{feature_md5}' in metadata " - f"for: {uri} ..." + f"for: {self.uri.resolve()} ..." # type: ignore ) # Validate MD5 for meta in metadata: @@ -347,7 +341,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): elif not feature_md5 and feature_name: logger.debug( f"Validating feature name '{feature_name}' in metadata " - f"for: {uri} ..." + f"for: {self.uri.resolve()} ..." # type: ignore ) # Retrieve MD5 for feature_name # Implicit counter for duplicate feature_name @@ -371,7 +365,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): md5 = feature_name_query[0]["meta_md5"] # Read data from HDF5 - hdf_data = self._read_data(md5=md5, element=element) + hdf_data = self._read_data(md5=md5) # Generate index for the data hdf_data_index = self._index_dict_to_multiindex( -- 2.52.0 From 661c0f1ef0e45fed5262ff15415c82f167006001 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 15:07:33 +0100 Subject: [PATCH 018/122] update: remove "meta_md5" storing from HDF5FeatureStorage.store_metadata() --- junifer/storage/hdf5.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 7ebb0ae6b..f410d0c5a 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -437,9 +437,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): The metadata as a dictionary. """ - # Update meta dictionary with MD5 - meta.update({"meta_md5": meta_md5}) - # Read metadata; if no file found, create an empty list try: metadata = self._read_metadata(element=element) -- 2.52.0 From 58b2b3b248c63ddc2a0bf5286d3f6ec938afc566 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 17:20:49 +0100 Subject: [PATCH 019/122] update: change remaining declarations of table to vector in HDF5FeatureStorage --- junifer/storage/hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index f410d0c5a..fe8bfe401 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -68,7 +68,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): uri.parent.mkdir(parents=True, exist_ok=True) # Available storage kinds - storage_types = ["table", "timeseries", "matrix"] + storage_types = ["vector", "timeseries", "matrix"] super().__init__( uri=uri, @@ -90,7 +90,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): storage. """ - return ["matrix", "table", "timeseries"] + return ["matrix", "vector", "timeseries"] def _fetch_correct_uri_for_io(self, element: Optional[Dict]) -> str: """Return proper URI for I/O based on `element`. -- 2.52.0 From 65b5af97639cff76141e6259eb2c2dfc3a89496c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 17:29:10 +0100 Subject: [PATCH 020/122] update: make metadata dict of dict in HDF5FeatureStorage._read_metadata() --- junifer/storage/hdf5.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index fe8bfe401..598cdae58 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -127,7 +127,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): def _read_metadata( self, element: Optional[Dict[str, str]] = None - ) -> List[Dict[str, Any]]: + ) -> Dict[str, Dict[str, Any]]: """Read metadata (should not be called directly). Parameters @@ -137,7 +137,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): Returns ------- - list of dict + dict of dict The stored metadata for the element. Raises @@ -164,9 +164,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): ) else: logger.info(f"Loaded HDF5 metadata from: {uri}") - # Convert to list if single entry found - if not isinstance(metadata, list): - metadata = [metadata] return metadata def list_features(self) -> Dict[str, Dict[str, Any]]: -- 2.52.0 From d3ab6119bc14167e1b3b1d8a0c835ca911e66424 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 17:29:52 +0100 Subject: [PATCH 021/122] update: refine HDF5FeatureStorage.list_features() --- junifer/storage/hdf5.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 598cdae58..5602a7f5c 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -179,17 +179,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): """ # Read metadata metadata = self._read_metadata() - - # Initialize dictionary to populate - meta_listing = {} - for meta in metadata: - # Make a copy so as not to affect original metadata - meta_copy = meta.copy() - # Format metadata as key-value pair - meta_key = meta_copy.pop("meta_md5") - meta_listing[meta_key] = meta_copy - - return meta_listing + return metadata def _read_data( self, md5: str, element: Optional[Dict[str, str]] = None -- 2.52.0 From 3c36bd1ba23731ea1e9bfcd0ea6b55d66542d516 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 17:32:37 +0100 Subject: [PATCH 022/122] update: refine HDF5FeatureStorage.store_metadata() and avoid duplicate MD5 writes --- junifer/storage/hdf5.py | 51 +++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 5602a7f5c..bf22b683c 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -412,7 +412,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): This method first loads existing metadata (if any) using ``_read_metadata`` and appends to it the new metadata and then saves - the updated metadata using ``_write_processed_data``. + the updated metadata using ``_write_processed_data``. It will only + store metadata if ``meta_md5`` is not found already. Parameters ---------- @@ -424,35 +425,41 @@ class HDF5FeatureStorage(BaseFeatureStorage): The metadata as a dictionary. """ - # Read metadata; if no file found, create an empty list + # Read metadata; if no file found, create an empty dictionary try: metadata = self._read_metadata(element=element) except IOError: - logger.debug(f"Creating new metadata list for {meta_md5} ...") - metadata = [] + logger.debug(f"Creating new metadata map for {meta_md5} ...") + metadata = {} - logger.debug(f"HDF5 metadata for {meta_md5} updated ...") - # Update metadata - metadata.append(meta) + # Only add entry if MD5 is not present + if meta_md5 not in metadata: + logger.debug(f"HDF5 metadata for {meta_md5} not found, adding ...") + # Update metadata + metadata[meta_md5] = meta - # Get correct URI for element; - # is different from uri if single_output is False - uri = self._fetch_correct_uri_for_io(element=element) + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element) - logger.info(f"Writing HDF5 metadata for {meta_md5} to: {uri}") - logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") - logger.debug( - f"HDF5 gzip compression level is set to: {self.compression} ..." - ) + logger.info(f"Writing HDF5 metadata for {meta_md5} to: {uri}") + logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") + logger.debug( + f"HDF5 gzip compression level is set to: {self.compression} ..." + ) - # Write metadata - self._write_processed_data( - fname=uri, - processed_data=meta, - title="meta", - ) + # Write metadata + self._write_processed_data( + fname=uri, + processed_data=metadata, + title="meta", + ) - logger.info(f"Wrote HDF5 metadata for {meta_md5} to: {uri}") + logger.info(f"Wrote HDF5 metadata for {meta_md5} to: {uri}") + else: + logger.debug( + f"HDF5 metadata for {meta_md5} found, skipping store ..." + ) def _element_metadata_to_index_dict( self, -- 2.52.0 From 5dc39f5dfe99604dcc66ec78d8e90e243357cc30 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 17:33:51 +0100 Subject: [PATCH 023/122] chore: update type annotations for HDF5FeatureStorage.store_metadata() --- junifer/storage/hdf5.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index bf22b683c..b4d83d492 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -407,7 +407,12 @@ class HDF5FeatureStorage(BaseFeatureStorage): ) logger.debug(f"Wrote processed HDF5 data to: {fname} ...") - def store_metadata(self, meta_md5: str, element: Dict, meta: Dict) -> None: + def store_metadata( + self, + meta_md5: str, + element: Dict[str, str], + meta: Dict[str, Any], + ) -> None: """Store metadata. This method first loads existing metadata (if any) using -- 2.52.0 From fea2fe5452736952c109ec9bd26492eb0217e4c8 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 17:49:32 +0100 Subject: [PATCH 024/122] update: refine HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index b4d83d492..638992de2 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -318,10 +318,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): f"for: {self.uri.resolve()} ..." # type: ignore ) # Validate MD5 - for meta in metadata: - if meta["meta_md5"] == feature_md5: - md5 = feature_md5 # type: ignore - if not md5: + if feature_md5 in metadata: + md5 = feature_md5 + else: raise_error(msg=f"Feature MD5 {feature_md5} not found") # Consider feature_name @@ -331,16 +330,17 @@ class HDF5FeatureStorage(BaseFeatureStorage): f"for: {self.uri.resolve()} ..." # type: ignore ) # Retrieve MD5 for feature_name - # Implicit counter for duplicate feature_name - feature_name_query = [] - for meta in metadata: + # Implicit counter for duplicate feature_name with different + # MD5; happens when something is wrong with marker computation + feature_name_duplicates_with_different_md5 = [] + for md5, meta in metadata.items(): if meta["name"] == feature_name: - feature_name_query.append(meta) + feature_name_duplicates_with_different_md5.append(md5) # Check for no / duplicate feature_name - if len(feature_name_query) == 0: + if len(feature_name_duplicates_with_different_md5) == 0: raise_error(msg=f"Feature {feature_name} not found") - elif len(feature_name_query) > 1: + elif len(feature_name_duplicates_with_different_md5) > 1: raise_error( msg=( f"More than one feature with name {feature_name} " @@ -349,7 +349,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): ) ) - md5 = feature_name_query[0]["meta_md5"] + md5 = feature_name_duplicates_with_different_md5[0] # Read data from HDF5 hdf_data = self._read_data(md5=md5) -- 2.52.0 From 6c6851ca58ac12ed936970b196201ba5c32ba9ce Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 2 Feb 2023 18:06:23 +0100 Subject: [PATCH 025/122] update: improve docstring of HDF5FeatureStorage.store_timeseries() --- junifer/storage/hdf5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 638992de2..2c5636ba2 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -738,7 +738,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): The element as dictionary. data : numpy.ndarray The timeseries data to store. - columns : list or tuple of str, optional + col_names : list or tuple of str, optional The column labels (default None). """ -- 2.52.0 From 0a43aae401cb284f7248f2f168be2ee33dd4fbae Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 6 Feb 2023 07:26:10 +0100 Subject: [PATCH 026/122] update: handle "meta" group reading failure in HDF5FeatureStorage._read_metadata() --- junifer/storage/hdf5.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 2c5636ba2..66f6b7572 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -143,7 +143,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): Raises ------ IOError - If HDF5 file does not exist. + If HDF5 file or `meta` does not exist. """ # Get correct URI for element; @@ -162,6 +162,11 @@ class HDF5FeatureStorage(BaseFeatureStorage): msg=f"HDF5 file not found at: {uri}", klass=IOError, ) + except ValueError: + raise_error( + msg=f"`meta` not found in: {uri}", + klass=IOError, + ) else: logger.info(f"Loaded HDF5 metadata from: {uri}") return metadata -- 2.52.0 From ac6c28d7118f4db559fb07356889652dad1c981d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 6 Feb 2023 07:46:14 +0100 Subject: [PATCH 027/122] update: handle data group reading failure in HDF5FeatureStorage._read_data() --- junifer/storage/hdf5.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 66f6b7572..db555fc8a 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -206,7 +206,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): Raises ------ IOError - If HDF5 file does not exist. + If HDF5 file or data does not exist. """ # Get correct URI for element; @@ -225,6 +225,11 @@ class HDF5FeatureStorage(BaseFeatureStorage): msg=f"HDF5 file not found at: {uri}", klass=IOError, ) + except ValueError: + raise_error( + msg=f"`{md5}` not found in: {uri}", + klass=IOError, + ) else: logger.info(f"Loaded HDF5 data for {md5} from: {uri}") return data -- 2.52.0 From fb20576c2e2bea1bcb7c60592b42a63e28d64532 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 6 Feb 2023 17:47:13 +0100 Subject: [PATCH 028/122] update: refine HDF5FeatureStorage._store_data() --- junifer/storage/hdf5.py | 82 +++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index db555fc8a..6c3efe597 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -537,56 +537,74 @@ class HDF5FeatureStorage(BaseFeatureStorage): def _store_data( self, + kind: str, meta_md5: str, element: Dict[str, str], - data: Union[np.ndarray, List], - column_headers: Union[str, Iterable[str], None] = None, - row_headers: Union[str, Iterable[str], None] = None, + data: np.ndarray, + **kwargs: Any, ) -> None: """Store data. - This method first prepares the index for the element's metadata - using :func:`_element_metadata_to_index_dict` and updates - the `data` with the index and then saves the data using - :func:`_write_processed_data`. + This method first loads existing data (if any) using + ``_read_data`` and appends to it the `element` and `data` + values, and writes them and other information passed via + `**kwargs` using ``_write_processed_data``. Parameters ---------- + kind : {"matrix", "vector", "timeseries"} + The storage kind. meta_md5 : str The metadata MD5 hash. element : dict The element as dictionary. - data : numpy.ndarray or list + data : numpy.ndarray The data to store. - column_headers : str, list or tuple of str, optional - The column labels (default None). - row_headers : str, list or tuple of str, optional - The column label to use in case number of rows greater than 1. - If None and number of rows greater than 1, then the name will be - "idx". For matrix storage, they are the row labels. (default None). + **kwargs : dict + Keyword arguments passed from the calling method. """ + # Read existing data; if no file found, create an empty dictionary + try: + stored_data = self._read_data(md5=meta_md5, element=element) + except IOError: + logger.debug(f"Creating new data map for {meta_md5} ...") + stored_data = {} + + # TODO: Validate stored and to be stored kwargs + + # Initialize dictionary to aggregate data to write + data_to_write = kwargs + + # Handle cases for existing and new entry + if not stored_data: + logger.debug(f"Writing new data for {meta_md5} ...") + # New entry; add as is + data_to_write.update({ + # change to list for easy subsequent storing + "element": [element], + "data": data, + # for serialization / deserialization of storage type + "kind": kind, + }) + elif stored_data: + logger.debug(f"Existing data found for {meta_md5}, appending to it ...") + # Existing entry; append to existing + # "element" and "data" + data_to_write.update({ + "element": [*stored_data["element"], element], + "data": np.concatenate( + (stored_data["data"], data), axis=0 + ), + # for serialization / deserialization of storage type + "kind": kind, + }) + # Get correct URI for element; # is different from uri if single_output is False uri = self._fetch_correct_uri_for_io(element=element) - # Initialize dictionary to aggregate data to write - data_to_write = { - "md5": meta_md5, - "element": element, - "data": data, - "column_headers": column_headers, - "row_headers": row_headers, - } - # Add index data to store - index_data = self._element_metadata_to_index_dict( - element=element, - n_rows=len(data), - row_headers=row_headers, - ) - data_to_write.update(index_data) # type: ignore - - logger.info(f"Writing HDF5 data to: {uri}") + logger.info(f"Writing HDF5 data for {meta_md5} to: {uri}") logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") logger.debug( f"HDF5 gzip compression level is set to: {self.compression} ..." @@ -599,7 +617,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): title=meta_md5, ) - logger.info(f"Wrote HDF5 data to: {uri}") + logger.info(f"Wrote HDF5 data for {meta_md5} to: {uri}") def store_matrix( self, -- 2.52.0 From 0ce9faba9d55f69cba3a40b67925b2f9d8c8614f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 6 Feb 2023 17:48:34 +0100 Subject: [PATCH 029/122] update: refine HDF5FeatureStorage.store_matrix() --- junifer/storage/hdf5.py | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 6c3efe597..56e992228 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -659,34 +659,19 @@ class HDF5FeatureStorage(BaseFeatureStorage): setting this to False will raise an error (default True). """ - # diagonal validation + # Diagonal validation if diagonal is False and matrix_kind not in ["triu", "tril"]: raise_error( msg="Diagonal cannot be False if kind is not full", klass=ValueError, ) - # matrix_kind and shape validation + # Matrix kind and shape validation if matrix_kind in ["triu", "tril"]: if data.shape[0] != data.shape[1]: raise_error( "Cannot store a non-square matrix as a triangular matrix", klass=ValueError, ) - # matrix_kind validation and computation - if matrix_kind == "triu": - k = 0 if diagonal is True else 1 - data_idx = np.triu_indices(data.shape[0], k=k) - elif matrix_kind == "tril": - k = 0 if diagonal is True else -1 - data_idx = np.tril_indices(data.shape[0], k=k) - elif matrix_kind == "full": - data_idx = ( - np.repeat(np.arange(data.shape[0]), data.shape[1]), - np.tile(np.arange(data.shape[1]), data.shape[0]), - ) - else: - raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError) - # Row data and label validation if row_names is None: row_names = [f"r{i}" for i in range(data.shape[0])] @@ -695,7 +680,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): msg="Number of row names does not match number of rows", klass=ValueError, ) - # Column data and label validation if col_names is None: col_names = [f"c{i}" for i in range(data.shape[1])] @@ -704,21 +688,16 @@ class HDF5FeatureStorage(BaseFeatureStorage): msg="Number of column names does not match number of columns", klass=ValueError, ) - - # Subset data - flat_data = data[data_idx] - # Format column names - columns = [ - f"{row_names[i]}~{col_names[j]}" # type: ignore - for i, j in zip(data_idx[0], data_idx[1]) - ] # Store self._store_data( + kind="matrix", meta_md5=meta_md5, element=element, - data=flat_data[None, :], - column_headers=columns, - row_headers=None, + data=data[np.newaxis, :, :], # convert to 3D + column_headers=col_names, + row_headers=row_names, + matrix_kind=matrix_kind, + diagonal=diagonal, ) def store_vector( -- 2.52.0 From d8e94af640269f4a07bf0e0805eacc3312b5ec6e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 6 Feb 2023 17:48:53 +0100 Subject: [PATCH 030/122] update: refine HDF5FeatureStorage.store_vector() --- junifer/storage/hdf5.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 56e992228..6682d474f 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -721,10 +721,28 @@ class HDF5FeatureStorage(BaseFeatureStorage): The column labels (default None). """ + if isinstance(data, list): + logger.debug( + f"Flattening and converting vector data list for {meta_md5}, " + "to numpy.ndarray ..." + ) + # Flatten out list and convert to np.ndarray + processed_data = np.array(np.ravel(data)) + elif isinstance(data, np.ndarray): + logger.debug( + f"Flattening vector data numpy.ndarray for {meta_md5} ..." + ) + # Flatten out array + processed_data = data.ravel() + + # Make it 2D + processed_data = processed_data[np.newaxis, :] + self._store_data( + kind="vector", meta_md5=meta_md5, element=element, - data=data, + data=processed_data, column_headers=col_names, ) -- 2.52.0 From 59063bd8da75dcd43e2093f66d483682e69c0824 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 6 Feb 2023 17:49:26 +0100 Subject: [PATCH 031/122] update: refine HDF5FeatureStorage.store_timeseries() --- junifer/storage/hdf5.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 6682d474f..f7e4d4894 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -768,9 +768,10 @@ class HDF5FeatureStorage(BaseFeatureStorage): """ self._store_data( + kind="timeseries", meta_md5=meta_md5, element=element, - data=data, + data=data[np.newaxis, :, :], # convert to 3D column_headers=col_names, row_headers="timepoint", ) -- 2.52.0 From e6a1c50b42bf9ecf5914d3d451ce8133a2ca2daa Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 6 Feb 2023 17:50:03 +0100 Subject: [PATCH 032/122] chore: update type annotation for HDF5FeatureStorage._read_data() --- junifer/storage/hdf5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index f7e4d4894..9351b98b3 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -188,7 +188,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): def _read_data( self, md5: str, element: Optional[Dict[str, str]] = None - ) -> Dict: + ) -> Dict[str, Any]: """Read data (should not be called directly). Parameters -- 2.52.0 From 52a240b5baf747c20bfc13424cf26e0b9dfbf6a7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 7 Feb 2023 11:31:09 +0100 Subject: [PATCH 033/122] update: add kwargs validation check for HDF5FeatureStorage._store_data() --- junifer/storage/hdf5.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 9351b98b3..a05e50efa 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -571,8 +571,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): logger.debug(f"Creating new data map for {meta_md5} ...") stored_data = {} - # TODO: Validate stored and to be stored kwargs - # Initialize dictionary to aggregate data to write data_to_write = kwargs @@ -588,6 +586,24 @@ class HDF5FeatureStorage(BaseFeatureStorage): "kind": kind, }) elif stored_data: + # Set up stored kwargs + stored_kwargs = [key for key in stored_data.keys() if key not in ("element", "data")] + # Set up to be stored kwargs + to_be_stored_kwargs = kwargs + # Update with kind + to_be_stored_kwargs["kind"] = kind + # Validate the kwargs + if set(stored_kwargs) != set(to_be_stored_kwargs): + raise_error( + msg=( + f"The additional data for {meta_md5} do not match " + "the ones already stored. This can be due " + "to some changes in marker computation, please " + "verify and try again." + ), + klass=RuntimeError, + ) + logger.debug(f"Existing data found for {meta_md5}, appending to it ...") # Existing entry; append to existing # "element" and "data" -- 2.52.0 From d8ab8ca70e23befdccf15717b59aab42a1afb2a5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:24:02 +0100 Subject: [PATCH 034/122] chore: improve error messages for HDF5FeatureStorage --- junifer/storage/hdf5.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index a05e50efa..0341db8ad 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -331,7 +331,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): if feature_md5 in metadata: md5 = feature_md5 else: - raise_error(msg=f"Feature MD5 {feature_md5} not found") + raise_error(msg=f"Feature MD5 '{feature_md5}' not found") # Consider feature_name elif not feature_md5 and feature_name: @@ -349,13 +349,13 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Check for no / duplicate feature_name if len(feature_name_duplicates_with_different_md5) == 0: - raise_error(msg=f"Feature {feature_name} not found") + raise_error(msg=f"Feature '{feature_name}' not found") elif len(feature_name_duplicates_with_different_md5) > 1: raise_error( msg=( - f"More than one feature with name {feature_name} " - "found. You can bypass this issue by specifying " - "a `feature_md5`." + "More than one feature with name " + f"'{feature_name}' found. You can bypass this " + "issue by specifying a `feature_md5`." ) ) -- 2.52.0 From 44b2d59d3c4d61ac561efe9b0731b2b00be61a41 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:25:45 +0100 Subject: [PATCH 035/122] chore: update docstrings for HDF5FeatureStorage --- junifer/storage/hdf5.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 0341db8ad..c75aafe3e 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -275,7 +275,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): feature_name: Optional[str] = None, feature_md5: Optional[bool] = None, ) -> pd.DataFrame: - """Read feature into a pandas DataFrame. + """Read feature into a pandas.DataFrame. Either one of ``feature_name`` or ``feature_md5`` needs to be specified. @@ -392,7 +392,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): ) -> None: """Write processed data to HDF5 (should not be called directly). - This is used primarily in ``_store_metadata`` and ``_store_data``. + This is used primarily in + :func:`junifer.storage.HDF5FeatureStorage.store_metadata` and + ``_store_data``. Parameters ---------- @@ -769,7 +771,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): data: np.ndarray, col_names: Optional[Iterable[str]] = None, ) -> None: - """Implement timeseries storing. + """Store timeseries. Parameters ---------- -- 2.52.0 From 35c779aaef0d9fe82bd917388c9c1076c084746e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:26:33 +0100 Subject: [PATCH 036/122] chore: improve logging messages for HDF5FeatureStorage --- junifer/storage/hdf5.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index c75aafe3e..5e48c3654 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -462,7 +462,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): logger.info(f"Writing HDF5 metadata for {meta_md5} to: {uri}") logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") logger.debug( - f"HDF5 gzip compression level is set to: {self.compression} ..." + "HDF5 gzip compression level is set to: " + f"{self.compression} ..." ) # Write metadata @@ -606,7 +607,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): klass=RuntimeError, ) - logger.debug(f"Existing data found for {meta_md5}, appending to it ...") + logger.debug( + f"Existing data found for {meta_md5}, appending to it ..." + ) # Existing entry; append to existing # "element" and "data" data_to_write.update({ -- 2.52.0 From 9a233be6c97e0e1216246c95ff9b009d90ddd960 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:28:33 +0100 Subject: [PATCH 037/122] update: rename row_headers to row_header_column_name in HDF5FeatureStorage.store_timeseries() --- junifer/storage/hdf5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 5e48c3654..11ef0e86b 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -794,7 +794,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): element=element, data=data[np.newaxis, :, :], # convert to 3D column_headers=col_names, - row_headers="timepoint", + row_header_column_name="timepoint", ) def collect(self) -> None: -- 2.52.0 From 2bb430bb27a75101d92b531a49871335fc94959e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:29:40 +0100 Subject: [PATCH 038/122] update: add row_header_col_name parameter for HDF5FeatureStorage.store_matrix() --- junifer/storage/hdf5.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 11ef0e86b..ae7e1e2be 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -649,6 +649,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): row_names: Optional[Iterable[str]] = None, matrix_kind: Optional[str] = "full", diagonal: bool = True, + row_header_col_name: str = "ROI", ) -> None: """Store matrix. @@ -678,6 +679,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): diagonal : bool, optional Whether to store the diagonal. If ``matrix_kind`` is "full", setting this to False will raise an error (default True). + row_header_col_name : str, optional + The column name for the row header column (default "ROI"). """ # Diagonal validation @@ -719,6 +722,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): row_headers=row_names, matrix_kind=matrix_kind, diagonal=diagonal, + row_header_column_name=row_header_col_name, ) def store_vector( -- 2.52.0 From a4fdbfd4fa9fc7d4a138cfcf101f9da894f6e1fe Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:31:07 +0100 Subject: [PATCH 039/122] chore: black formatting for HDF5FeatureStorage._store_data() --- junifer/storage/hdf5.py | 42 ++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index ae7e1e2be..6d457182e 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -544,7 +544,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): meta_md5: str, element: Dict[str, str], data: np.ndarray, - **kwargs: Any, + **kwargs: Any, ) -> None: """Store data. @@ -581,16 +581,22 @@ class HDF5FeatureStorage(BaseFeatureStorage): if not stored_data: logger.debug(f"Writing new data for {meta_md5} ...") # New entry; add as is - data_to_write.update({ - # change to list for easy subsequent storing - "element": [element], - "data": data, - # for serialization / deserialization of storage type - "kind": kind, - }) + data_to_write.update( + { + # change to list for easy subsequent storing + "element": [element], + "data": data, + # for serialization / deserialization of storage type + "kind": kind, + } + ) elif stored_data: # Set up stored kwargs - stored_kwargs = [key for key in stored_data.keys() if key not in ("element", "data")] + stored_kwargs = [ + key + for key in stored_data.keys() + if key not in ("element", "data") + ] # Set up to be stored kwargs to_be_stored_kwargs = kwargs # Update with kind @@ -612,14 +618,16 @@ class HDF5FeatureStorage(BaseFeatureStorage): ) # Existing entry; append to existing # "element" and "data" - data_to_write.update({ - "element": [*stored_data["element"], element], - "data": np.concatenate( - (stored_data["data"], data), axis=0 - ), - # for serialization / deserialization of storage type - "kind": kind, - }) + data_to_write.update( + { + "element": [*stored_data["element"], element], + "data": np.concatenate( + (stored_data["data"], data), axis=0 + ), + # for serialization / deserialization of storage type + "kind": kind, + } + ) # Get correct URI for element; # is different from uri if single_output is False -- 2.52.0 From b474644a808ef6b66f7ff6da3456e56d12b36f90 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:31:45 +0100 Subject: [PATCH 040/122] update: remove HDF5FeatureStorage._element_metadata_to_index_dict() --- junifer/storage/hdf5.py | 59 ----------------------------------------- 1 file changed, 59 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 6d457182e..eada0008f 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -479,65 +479,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): f"HDF5 metadata for {meta_md5} found, skipping store ..." ) - def _element_metadata_to_index_dict( - self, - element: Dict, - n_rows: int = 1, - row_headers: Union[str, Iterable[str], None] = None, - ) -> Dict[ - str, - Union[Dict[Union[str, Iterable[str]], Union[str, Iterable[str]]], int], - ]: - """Convert the element meta to index (should not be called directly). - - This method is only used in ``_store_data``. - - Parameters - ---------- - element : dict - The element as dictionary. - n_rows : int, optional - Number of rows to create (default 1). - row_headers: str, list or tuple of str, optional - The column name to use in case ``n_rows`` > 1. If None and - ``n_rows`` > 1, the name will be "idx" (default None). - - Returns - ------- - dict - The index to store. Is used for creating index for - pandas.DataFrame returned by :func:`read_df`. - - """ - logger.debug( - "Converting element metadata to index dictionary for " - "HDF5 storage ..." - ) - - # Check row_headers - if not row_headers: - row_headers = "idx" - - # Create dictionary with element access options as keys - # and corresponding access values as values for n_rows - element_index: Dict[ - Union[str, Iterable[str]], Union[str, Iterable[str]] - ] = {key: [val] * n_rows for key, val in element.items()} - # Set index value for row_headers - element_index[row_headers] = np.arange(n_rows) - - logger.debug( - "Converted element metadata to index dictionary for " - "HDF5 storage ..." - ) - - return { - "idx_data": element_index, - # required as deserialization sorts keys of the index dict - "idx_columns_order": list(element_index.keys()), # type: ignore - "n_rows": n_rows, - } - def _store_data( self, kind: str, -- 2.52.0 From e7cdafa336b326e7955bc5bd9ad5664906fca5a7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:32:24 +0100 Subject: [PATCH 041/122] update: remove HDF5FeatureStorage._index_dict_to_multiindex() --- junifer/storage/hdf5.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index eada0008f..dcf03b37f 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -234,42 +234,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): logger.info(f"Loaded HDF5 data for {md5} from: {uri}") return data - def _index_dict_to_multiindex( - self, - index_dict: Dict[Union[str, Iterable[str]], Union[str, Iterable[str]]], - index_columns_order: List[str], - n_rows: int, - ) -> pd.MultiIndex: - """Convert index to pandas.MultiIndex (should not be called directly). - - Parameters - ---------- - index_dict : dict - The index data as returned by - :func:`_element_metadata_to_index_dict`. - index_columns_order: list of str - The ordering for index columns. - n_rows : int - Number of rows for the index. - - Returns - ------- - pandas.MultiIndex - The multi-index for use in :func:`read_df`. - - """ - logger.debug("Converting index dictionary to pandas MultiIndex ...") - # Create multi-index; dict comprehension required to undo keys sorting - # during serialization / deserialization - index = pd.MultiIndex.from_frame( - pd.DataFrame( - data={key: index_dict[key] for key in index_columns_order}, - index=range(n_rows), - ) - ) - logger.debug("Converted index dictionary to pandas MultiIndex ...") - return index - def read_df( self, feature_name: Optional[str] = None, -- 2.52.0 From b5173db351b43ef44b915c225d1d621b8fa2aa88 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Feb 2023 16:32:47 +0100 Subject: [PATCH 042/122] update: refine HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 51 ++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index dcf03b37f..6c52f49d7 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -5,6 +5,7 @@ # License: AGPL +from collections import defaultdict from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Union @@ -327,21 +328,55 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Read data from HDF5 hdf_data = self._read_data(md5=md5) + reshaped_data = hdf_data["data"] # Generate index for the data - hdf_data_index = self._index_dict_to_multiindex( - index_dict=hdf_data["idx_data"], - index_columns_order=hdf_data["idx_columns_order"], - n_rows=hdf_data["n_rows"], - ) + logger.debug(f"Generating pandas.MultiIndex for {md5} ...") + # Create dictionary for aggregating index data + element_idx_dict = defaultdict(list) + if hdf_data["kind"] == "matrix": + # Get row count + n_rows, _ = hdf_data["data"][0, :, :].shape + for element in hdf_data["element"]: + for key, val in element.items(): + element_idx_dict[key].extend([val] * n_rows) + # Add extra column for row headers + element_idx_dict[ + hdf_data["row_header_column_name"] + ].extend(hdf_data["row_headers"]) + # Convert data from 3D to 2D + reshaped_data = hdf_data["data"].reshape(-1, n_rows) + elif hdf_data["kind"] == "vector": + for element in hdf_data["element"]: + for key, val in element.items(): + element_idx_dict[key].append(val) + elif hdf_data["kind"] == "timeseries": + for idx, element in enumerate(hdf_data["element"]): + # Get row count for the element + n_rows, _ = hdf_data["data"][idx, :, :].shape + for key, val in element.items(): + element_idx_dict[key].extend([val] * n_rows) + # Add extra column for timepoints + element_idx_dict[ + hdf_data["row_header_column_name"] + ].extend(np.arange(n_rows)) + # Convert data from 3D to 2D + reshaped_data = hdf_data["data"].reshape(-1, 1) + + # Create dataframe for index + idx_df = pd.DataFrame(data=element_idx_dict) + # Create multiindex from dataframe + hdf_data_idx = pd.MultiIndex.from_frame(df=idx_df) + logger.debug(f"Generated pandas.MultiIndex for {md5} ...") + + # Convert to DataFrame logger.debug( f"Converting HDF5 data for {md5} to pandas.DataFrame ..." ) - # Convert to DataFrame df = pd.DataFrame( - data=hdf_data["data"], - index=hdf_data_index, + data=reshaped_data, + index=hdf_data_idx, columns=hdf_data["column_headers"], dtype=hdf_data["data"].dtype, ) -- 2.52.0 From 606e32639a702dfe6aca2af30c96c87475be458b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 17:48:14 +0100 Subject: [PATCH 043/122] update: make element param a list of dict for HDF5FeatureStorage._store_data() --- junifer/storage/hdf5.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 6c52f49d7..c3d6a8d9c 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -482,7 +482,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): self, kind: str, meta_md5: str, - element: Dict[str, str], + element: List[Dict[str, str]], data: np.ndarray, **kwargs: Any, ) -> None: @@ -499,8 +499,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): The storage kind. meta_md5 : str The metadata MD5 hash. - element : dict - The element as dictionary. + element : list of dict + The element as list of dictionary. data : numpy.ndarray The data to store. **kwargs : dict @@ -509,7 +509,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): """ # Read existing data; if no file found, create an empty dictionary try: - stored_data = self._read_data(md5=meta_md5, element=element) + stored_data = self._read_data(md5=meta_md5, element=element[0]) except IOError: logger.debug(f"Creating new data map for {meta_md5} ...") stored_data = {} @@ -523,8 +523,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): # New entry; add as is data_to_write.update( { - # change to list for easy subsequent storing - "element": [element], + "element": element, "data": data, # for serialization / deserialization of storage type "kind": kind, @@ -560,7 +559,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): # "element" and "data" data_to_write.update( { - "element": [*stored_data["element"], element], + "element": stored_data["element"] + element, "data": np.concatenate( (stored_data["data"], data), axis=0 ), @@ -571,7 +570,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Get correct URI for element; # is different from uri if single_output is False - uri = self._fetch_correct_uri_for_io(element=element) + uri = self._fetch_correct_uri_for_io(element=element[0]) logger.info(f"Writing HDF5 data for {meta_md5} to: {uri}") logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") @@ -664,7 +663,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): self._store_data( kind="matrix", meta_md5=meta_md5, - element=element, + element=[element], # convert to list data=data[np.newaxis, :, :], # convert to 3D column_headers=col_names, row_headers=row_names, @@ -714,7 +713,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): self._store_data( kind="vector", meta_md5=meta_md5, - element=element, + element=[element], # convert to list data=processed_data, column_headers=col_names, ) @@ -743,7 +742,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): self._store_data( kind="timeseries", meta_md5=meta_md5, - element=element, + element=[element], # convert to list data=data[np.newaxis, :, :], # convert to 3D column_headers=col_names, row_header_column_name="timepoint", -- 2.52.0 From 394a6852069d28dee653f6dae3922806098bc980 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 17:49:03 +0100 Subject: [PATCH 044/122] update: add duplicate element store check in HDF5FeatureStorage._store_data() --- junifer/storage/hdf5.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index c3d6a8d9c..bbdee707d 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -552,6 +552,17 @@ class HDF5FeatureStorage(BaseFeatureStorage): klass=RuntimeError, ) + # Check for duplicate elements; if found, return immediately + logger.debug(f"Checking duplicate elements for {meta_md5} ...") + for stored_element in stored_data["element"]: + if stored_element == element[0]: + logger.info( + f"Duplicate element: {element[0]} found for " + f"{meta_md5}, skipping store ... " + ) + return None + logger.debug(f"No duplicate elements found for {meta_md5} ...") + logger.debug( f"Existing data found for {meta_md5}, appending to it ..." ) -- 2.52.0 From e62b3008c0cb08a5f7f11b6a377e1c46b981b5da Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 17:51:44 +0100 Subject: [PATCH 045/122] update: refine HDF5FeatureStorage.collect() --- junifer/storage/hdf5.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index bbdee707d..e848013c4 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -804,13 +804,15 @@ class HDF5FeatureStorage(BaseFeatureStorage): logger.info( f"Updating HDF5 metadata with metadata from: {file.resolve()}" ) - # Load metadata; empty list if first entry + # Load metadata; empty dictionary if first entry; + # can be replaced with store_metadata() if run on a loop + # for the metadata entries from in_storage try: out_metadata = out_storage._read_metadata() except IOError: - out_metadata = [] + out_metadata = {} # Update metadata - out_metadata.append(in_metadata) + out_metadata.update(in_metadata) # Save metadata out_storage._write_processed_data( fname=self.uri.resolve(), @@ -818,17 +820,10 @@ class HDF5FeatureStorage(BaseFeatureStorage): title="meta", ) - for meta in tqdm(in_metadata, desc="feature"): - logger.debug(f"Collecting feature MD5: {meta['meta_md5']} ...") + for meta_md5 in tqdm(in_metadata.keys(), desc="feature"): # Load data from new instance - in_data = in_storage._read_data(md5=meta["meta_md5"]) - - logger.info( - f"Updating HDF5 data with data from: {meta['meta_md5']}" - ) + logger.debug(f"Collecting feature MD5: {meta_md5} ...") + in_data = in_storage._read_data(md5=meta_md5) + logger.info(f"Updating HDF5 data with data from: {meta_md5}") # Save data - out_storage._write_processed_data( - fname=self.uri.resolve(), - processed_data=in_data, - title=meta["meta_md5"], - ) + out_storage._store_data(meta_md5=meta_md5, **in_data) -- 2.52.0 From 712abc468d0ae7b2ed40405cc2a4d6b1580b415c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 17:52:06 +0100 Subject: [PATCH 046/122] update: make HDF5FeatureStorage.list_features() read metadata directly --- junifer/storage/hdf5.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index e848013c4..2ed8166c5 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -184,7 +184,11 @@ class HDF5FeatureStorage(BaseFeatureStorage): """ # Read metadata - metadata = self._read_metadata() + metadata = read_hdf5( + fname=self.uri, + title="meta", + slash="ignore", + ) return metadata def _read_data( -- 2.52.0 From 37345cc1f0d80ee3a758514cb62928db9ba51d73 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 17:53:02 +0100 Subject: [PATCH 047/122] update: make HDF5FeatureStorage.read_df() read metadata directly --- junifer/storage/hdf5.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 2ed8166c5..38ac538ff 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -286,7 +286,11 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Parameter check pass else: # Read metadata - metadata = self._read_metadata() + metadata = read_hdf5( + fname=self.uri, + title="meta", + slash="ignore", + ) # Initialize MD5 variable md5: str = "" -- 2.52.0 From 8b9737fc67232909e8b35707661af8ecf5548d9f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 18:09:12 +0100 Subject: [PATCH 048/122] update: make HDF5FeatureStorage.read_df() read data directly --- junifer/storage/hdf5.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 38ac538ff..4ea75d1c7 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -335,7 +335,11 @@ class HDF5FeatureStorage(BaseFeatureStorage): md5 = feature_name_duplicates_with_different_md5[0] # Read data from HDF5 - hdf_data = self._read_data(md5=md5) + hdf_data = read_hdf5( + fname=self.uri, + title=md5, + slash="ignore", + ) reshaped_data = hdf_data["data"] # Generate index for the data -- 2.52.0 From f0a315d44693dd5ec7146b597215bce926cd7b9a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 18:09:55 +0100 Subject: [PATCH 049/122] chore: add static type checking ignore --- junifer/storage/hdf5.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 4ea75d1c7..300a27314 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -302,7 +302,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): ) # Validate MD5 if feature_md5 in metadata: - md5 = feature_md5 + md5 = feature_md5 # type: ignore else: raise_error(msg=f"Feature MD5 '{feature_md5}' not found") @@ -800,9 +800,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): out_storage = HDF5FeatureStorage(uri=self.uri, overwrite="update") # Glob files - globbed_files = self.uri.parent.glob( - f"*{self.uri.name}" - ) # type: ignore + globbed_files = self.uri.parent.glob( # type: ignore + f"*{self.uri.name}" # type: ignore + ) # Run loop to aggregate for file in tqdm(globbed_files, desc="file"): @@ -827,7 +827,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): out_metadata.update(in_metadata) # Save metadata out_storage._write_processed_data( - fname=self.uri.resolve(), + fname=self.uri.resolve(), # type: ignore processed_data=out_metadata, title="meta", ) -- 2.52.0 From d2e1aaedd2978077d664ace3a78e4582be3e9531 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 9 Feb 2023 18:10:28 +0100 Subject: [PATCH 050/122] chore: improve logging messages for HDF5FeatureStorage --- junifer/storage/hdf5.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 300a27314..a43a7234b 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -63,8 +63,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Create parent directories if not present if not uri.parent.exists(): logger.info( - f"Output directory ({str(uri.parent.absolute())}) " - "does not exist, creating now." + f"Output directory: '{uri.parent.resolve()}' " + "does not exist, creating now" ) uri.parent.mkdir(parents=True, exist_ok=True) @@ -806,16 +806,14 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Run loop to aggregate for file in tqdm(globbed_files, desc="file"): - logger.debug(f"Reading HDF5 file: {file.resolve()} ...") + logger.debug(f"Reading HDF5 file: {file} ...") # Create new storage instance to load data in_storage = HDF5FeatureStorage(uri=file) # Load metadata from new instance in_metadata = in_storage._read_metadata() - logger.info( - f"Updating HDF5 metadata with metadata from: {file.resolve()}" - ) + logger.info(f"Updating HDF5 metadata with metadata from: {file}") # Load metadata; empty dictionary if first entry; # can be replaced with store_metadata() if run on a loop # for the metadata entries from in_storage -- 2.52.0 From 633afe1a56700eea88c4685613e494992911c8c2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 10 Feb 2023 13:17:22 +0100 Subject: [PATCH 051/122] update: add matrix_kind validation for HDF5FeatureStorage.store_matrix() --- junifer/storage/hdf5.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index a43a7234b..7ed875b74 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -653,6 +653,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): The column name for the row header column (default "ROI"). """ + # Matrix kind validation + if matrix_kind not in ("triu", "tril", "full"): + raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError) # Diagonal validation if diagonal is False and matrix_kind not in ["triu", "tril"]: raise_error( -- 2.52.0 From 1a6f4d9b36eef7ac089274d885f7bdc2241a0c58 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 10 Feb 2023 13:17:50 +0100 Subject: [PATCH 052/122] chore: update docstring for HDF5FeatureStorage.store_matrix() --- junifer/storage/hdf5.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 7ed875b74..a39cb9b02 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -652,6 +652,15 @@ class HDF5FeatureStorage(BaseFeatureStorage): row_header_col_name : str, optional The column name for the row header column (default "ROI"). + Raises + ------ + ValueError + If invalid ``matrix_kind`` is provided, ``diagonal = False`` + for ``matrix_kind = "full"``, non-square data is provided + for ``matrix_kind = {"triu", "tril"}``, length of ``row_names`` + do not match data row count, or length of ``col_names`` do not + match data column count. + """ # Matrix kind validation if matrix_kind not in ("triu", "tril", "full"): -- 2.52.0 From 98badc1d04e3ef34d30acba40c07402e98701ef4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 10 Feb 2023 13:18:22 +0100 Subject: [PATCH 053/122] fix: use column length for reshaping matrix in HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index a39cb9b02..c0cfb06fc 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -349,7 +349,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): if hdf_data["kind"] == "matrix": # Get row count - n_rows, _ = hdf_data["data"][0, :, :].shape + n_rows, n_cols = hdf_data["data"][0, :, :].shape for element in hdf_data["element"]: for key, val in element.items(): element_idx_dict[key].extend([val] * n_rows) @@ -358,7 +358,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): hdf_data["row_header_column_name"] ].extend(hdf_data["row_headers"]) # Convert data from 3D to 2D - reshaped_data = hdf_data["data"].reshape(-1, n_rows) + reshaped_data = hdf_data["data"].reshape(-1, n_cols) elif hdf_data["kind"] == "vector": for element in hdf_data["element"]: for key, val in element.items(): -- 2.52.0 From 3609531463b287b9ad0266b9946c4cc3aae32b8a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 10 Feb 2023 16:01:42 +0100 Subject: [PATCH 054/122] update: improve HDF5FeatureStorage unit tests --- junifer/storage/tests/test_hdf5.py | 272 +++++++++-------------------- 1 file changed, 87 insertions(+), 185 deletions(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index 6feeb9230..95e92cd93 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -7,7 +7,6 @@ from pathlib import Path import numpy as np -import pandas as pd import pytest from numpy.testing import assert_array_equal from pandas.testing import assert_frame_equal @@ -19,7 +18,7 @@ from junifer.storage.utils import element_to_prefix, process_meta def test_get_valid_inputs() -> None: """Test valid inputs.""" storage = HDF5FeatureStorage(uri="/tmp") - assert storage.get_valid_inputs() == ["matrix", "table", "timeseries"] + assert storage.get_valid_inputs() == ["matrix", "vector", "timeseries"] def test_single_output(tmp_path: Path) -> None: @@ -140,21 +139,14 @@ def test_read_df(tmp_path: Path) -> None: meta_md5=meta_md5, element=element_to_store, meta=meta_to_store ) # Data to store - data = np.array( - [ - [1, 10], - [2, 20], - ] - ) - row_header = "scan" + data = np.array([[1, 10]]) col_headers = ["f1", "f2"] # Store table - storage.store_table( + storage.store_vector( meta_md5=meta_md5, element=element_to_store, data=data, - columns=col_headers, - rows_col_name=row_header, + col_names=col_headers, ) # Read into dataframe and check df_md5 = storage.read_df(feature_md5=meta_md5) @@ -226,22 +218,17 @@ def test_store_matrix(tmp_path: Path) -> None: # List the stored features features = storage.list_features() - # Get the first MD5 - feature_md5 = list(features.keys())[0] # Check the MD5 - assert "BOLD_fc" == features[feature_md5]["name"] + assert "BOLD_fc" == features[meta_md5]["name"] # Read into dataframe - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=meta_md5) # Check shape of dataframe - assert read_df.shape == (1, 12) + assert read_df.shape == (4, 3) # Check data of dataframe - assert_array_equal(read_df.values[0], data.flatten()) - - # Format column headers - stored_col_headers = [f"{i}~{j}" for i in row_headers for j in col_headers] + assert_array_equal(read_df.values, data) # Check column headers - assert list(read_df.columns) == stored_col_headers + assert read_df.columns.to_list() == col_headers # Diagonal validation error with pytest.raises(ValueError, match="cannot be False"): @@ -333,26 +320,17 @@ def test_store_matrix_without_headers(tmp_path: Path) -> None: # List the stored features features = storage.list_features() - # Get the first MD5 - feature_md5 = list(features.keys())[0] # Check the MD5 - assert "BOLD_fc" == features[feature_md5]["name"] + assert "BOLD_fc" == features[meta_md5]["name"] # Read the dataframe - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=meta_md5) # Check shape of dataframe - assert read_df.shape == (1, 12) + assert read_df.shape == (4, 3) # Check data of dataframe - assert_array_equal(read_df.values[0], data.flatten()) - - # Format column headers - stored_col_headers = [ - f"r{i}~c{j}" - for i in range(data.shape[0]) - for j in range(data.shape[1]) - ] + assert_array_equal(read_df.values, data) # Check column headers - assert list(read_df.columns) == stored_col_headers + assert read_df.columns.to_list() == ["c0", "c1", "c2"] def test_store_upper_triangular_matrix(tmp_path: Path) -> None: @@ -397,29 +375,15 @@ def test_store_upper_triangular_matrix(tmp_path: Path) -> None: # List the stored features features = storage.list_features() - # Get the first MD5 - feature_md5 = list(features.keys())[0] # Check the MD5 - assert "BOLD_fc" == features[feature_md5]["name"] + assert "BOLD_fc" == features[meta_md5]["name"] # Read into dataframe - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal( - read_df.values, data[np.triu_indices(n=data.shape[0])][None, :] - ) - - # Format column headers - stored_col_headers = [ - "row1~col1", - "row1~col2", - "row1~col3", - "row2~col2", - "row2~col3", - "row3~col3", - ] + assert_array_equal(read_df.values, data) # Check column headers - assert list(read_df.columns) == stored_col_headers + assert read_df.columns.to_list() == col_headers def test_store_upper_triangular_matrix_without_diagonal( @@ -467,26 +431,15 @@ def test_store_upper_triangular_matrix_without_diagonal( # List the stored features features = storage.list_features() - # Get the first MD5 - feature_md5 = list(features.keys())[0] # Check the MD5 - assert "BOLD_fc" == features[feature_md5]["name"] + assert "BOLD_fc" == features[meta_md5]["name"] # Read into dataframe - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal( - read_df.values, data[np.triu_indices(n=data.shape[0], k=1)][None, :] - ) - - # Format column headers - stored_col_headers = [ - "row1~col2", - "row1~col3", - "row2~col3", - ] + assert_array_equal(read_df.values, data) # Check column headers - assert list(read_df.columns) == stored_col_headers + assert read_df.columns.to_list() == col_headers def test_store_lower_triangular_matrix(tmp_path: Path) -> None: @@ -531,29 +484,15 @@ def test_store_lower_triangular_matrix(tmp_path: Path) -> None: # List the stored features features = storage.list_features() - # Get the first MD5 - feature_md5 = list(features.keys())[0] # Check the MD5 - assert "BOLD_fc" == features[feature_md5]["name"] + assert "BOLD_fc" == features[meta_md5]["name"] # Read into dataframe - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal( - read_df.values, data[np.tril_indices(n=data.shape[0])][None, :] - ) - - # Format column headers - stored_col_headers = [ - "row1~col1", - "row2~col1", - "row2~col2", - "row3~col1", - "row3~col2", - "row3~col3", - ] + assert_array_equal(read_df.values, data) # Check column headers - assert list(read_df.columns) == stored_col_headers + assert read_df.columns.to_list() == col_headers def test_store_lower_triangular_matrix_without_diagonal( @@ -601,30 +540,19 @@ def test_store_lower_triangular_matrix_without_diagonal( # List the stored features features = storage.list_features() - # Get the first MD5 - feature_md5 = list(features.keys())[0] # Check the MD5 - assert "BOLD_fc" == features[feature_md5]["name"] + assert "BOLD_fc" == features[meta_md5]["name"] # Read into dataframe - read_df = storage.read_df(feature_md5=feature_md5) + read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal( - read_df.values, data[np.tril_indices(n=data.shape[0], k=-1)][None, :] - ) - - # Format column headers - stored_col_headers = [ - "row2~col1", - "row3~col1", - "row3~col2", - ] + assert_array_equal(read_df.values, data) # Check column headers - assert list(read_df.columns) == stored_col_headers + assert read_df.columns.to_list() == col_headers -def test_store_table(tmp_path: Path) -> None: - """Test table store. +def test_store_vector(tmp_path: Path) -> None: + """Test vector store. Parameters ---------- @@ -632,7 +560,7 @@ def test_store_table(tmp_path: Path) -> None: The path to the test directory. """ - uri = tmp_path / "test_store_table.hdf5" + uri = tmp_path / "test_store_vector.hdf5" storage = HDF5FeatureStorage(uri=uri) # Metadata to store element = {"subject": "test"} @@ -650,79 +578,65 @@ def test_store_table(tmp_path: Path) -> None: ) # Data to store - data = np.array( - [ - [1, 10], - [2, 20], - [3, 30], - [4, 40], - [5, 50], - ] - ) - row_header = "scan" - col_headers = ["f1", "f2"] + data = [10, 20, 30, 40, 50] + col_names = ["f1", "f2", "f3", "f4", "f5"] - # Store table - storage.store_table( + # Store vector + storage.store_vector( meta_md5=meta_md5, element=element_to_store, data=data, - columns=col_headers, - rows_col_name=row_header, + col_names=col_names, ) - # Convert element to index dict - idx_dict = storage._element_metadata_to_index_dict( - element=element, - n_rows=5, - row_headers="scan", - ) - # Convert index dict to index - idx = storage._index_dict_to_multiindex( - index_dict=idx_dict["idx_data"], # type: ignore - index_columns_order=idx_dict["idx_columns_order"], # type: ignore - n_rows=idx_dict["n_rows"], # type: ignore - ) - # Create dataframe - df = pd.DataFrame(data, columns=["f1", "f2"], index=idx) - # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) - # Check if dataframes are equal - assert_frame_equal(df, read_df) + # Check if data are equal + assert read_df.values.flatten().tolist() == data - # New data to store - data_new = np.array( - [[1, 10], [2, 20], [3, 300], [4, 40], [5, 50], [6, 600]] - ) - # Convert element to index dict - idx_dict_new = storage._element_metadata_to_index_dict( - element=element, - n_rows=6, - row_headers="scan", - ) - # Convert index dict to index - idx_new = storage._index_dict_to_multiindex( - index_dict=idx_dict_new["idx_data"], # type: ignore - index_columns_order=idx_dict_new["idx_columns_order"], # type: ignore - n_rows=idx_dict_new["n_rows"], # type: ignore - ) - # Create new dataframe - df_new = pd.DataFrame(data_new, columns=["f1", "f2"], index=idx_new) - # Store table - storage.store_table( +def test_store_timeseries(tmp_path: Path) -> None: + """Test timeseries store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_store_timeseries.hdf5" + storage = HDF5FeatureStorage(uri=uri) + # Metadata to store + element = {"subject": "test"} + meta = { + "element": element, + "dependencies": ["numpy"], + "marker": {"name": "fc"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store metadata + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + + # Data to store + data = np.array([[10], [20], [30], [40], [50]]) + col_names = ["signal"] + + # Store vector + storage.store_timeseries( meta_md5=meta_md5, element=element_to_store, - data=data_new, - columns=["f1", "f2"], - rows_col_name="scan", + data=data, + col_names=col_names, ) # Read into dataframe - read_df_new = storage.read_df(feature_md5=meta_md5) - # Check if dataframes are equal - assert_frame_equal(df_new, read_df_new) + read_df = storage.read_df(feature_md5=meta_md5) + # Check if data are equal + assert_array_equal(read_df.values, data) def test_multi_output_store_and_collect(tmp_path: Path): @@ -758,19 +672,10 @@ def test_multi_output_store_and_collect(tmp_path: Path): } # Data to store - data_1 = np.array( - [ - [1, 10], - [2, 20], - [3, 30], - [4, 40], - [5, 50], - ] - ) + data_1 = np.array([10, 20, 30, 40, 50]) data_2 = data_1 * 10 data_3 = data_1 * 20 - row_header = "scan" - col_headers = ["f1", "f2"] + col_headers = ["f1", "f2", "f3", "f4", "f5"] # Process metadata for storage hash_1, meta_to_store_1, element_to_store_1 = process_meta(meta_1) @@ -803,26 +708,23 @@ def test_multi_output_store_and_collect(tmp_path: Path): ) # Store tables - storage.store_table( + storage.store_vector( meta_md5=hash_1, element=element_to_store_1, data=data_1, - columns=col_headers, - rows_col_name=row_header, + col_names=col_headers, ) - storage.store_table( + storage.store_vector( meta_md5=hash_2, element=element_to_store_2, data=data_2, - columns=col_headers, - rows_col_name=row_header, + col_names=col_headers, ) - storage.store_table( + storage.store_vector( meta_md5=hash_3, element=element_to_store_3, data=data_3, - columns=col_headers, - rows_col_name=row_header, + col_names=col_headers, ) # Check that base URI does not exist yet @@ -858,10 +760,10 @@ def test_multi_output_store_and_collect(tmp_path: Path): assert uri.exists() # Read unified metadata - read_unified_meta = storage._read_metadata() + read_unified_meta = storage.list_features() # Check if aggregated metadata are equal - assert read_unified_meta == [read_meta_1, read_meta_2, read_meta_3] + assert read_unified_meta == {**read_meta_1, **read_meta_2, **read_meta_3} def test_collect_error_single_output() -> None: -- 2.52.0 From ab076d2ab417d93a58e036cd99ab7e47dce476ae Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 10 Feb 2023 16:57:36 +0100 Subject: [PATCH 055/122] update: add more unit tests for HDF5FeatureStorage --- junifer/storage/tests/test_hdf5.py | 195 ++++++++++++++++++++++++++--- 1 file changed, 175 insertions(+), 20 deletions(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index 95e92cd93..e72ebfcb3 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -22,7 +22,7 @@ def test_get_valid_inputs() -> None: def test_single_output(tmp_path: Path) -> None: - """Test engine retrieval with single output. + """Test single output setup. Parameters ---------- @@ -36,6 +36,45 @@ def test_single_output(tmp_path: Path) -> None: assert storage.single_output is True +def test_single_output_meta_not_found_error(tmp_path: Path) -> None: + """Test single output metadata not found error. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_single_output_no_meta.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Store data to create the file + storage._store_data( + kind="vector", + meta_md5="md5", + element=[{"sub": "001"}], + data=np.empty((1, 1)), + ) + # Check metadata error + with pytest.raises(IOError, match="`meta` not found in:"): + storage._read_metadata() + + +def test_single_output_file_not_found_error(tmp_path: Path) -> None: + """Test single output file not found error. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_single_output_no_file.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Check file error + with pytest.raises(IOError, match="HDF5 file not found at:"): + storage._read_data(md5="md5") + + def test_multi_output_error(tmp_path: Path) -> None: """Test error for multi output. @@ -103,9 +142,68 @@ def test_store_metadata_and_list_features(tmp_path: Path) -> None: assert meta_md5 == feature_md5 -def test_read_df_params_error() -> None: - """Test parameter validation errors for read_df.""" - storage = HDF5FeatureStorage(uri="/tmp") +def test_store_metadata_ignore_duplicate(tmp_path: Path) -> None: + """Test duplicate ignore for metadata store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_duplicate_metadata_store.hdf5" + # Single storage, must be the uri + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Store metadata first time + storage.store_metadata( + meta_md5="md5", + element={"sub": "001"}, + meta={ + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "test"}, + "type": "BOLD", + }, + ) + # Store metadata second time, should be ignored + storage.store_metadata( + meta_md5="md5", + element={"sub": "001"}, + meta={ + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "test"}, + "type": "BOLD", + }, + ) + # List the stored features + features = storage.list_features() + # Should only have one element + assert len(features) == 1 + + +def test_read_df_params_error(tmp_path: Path) -> None: + """Test parameter validation errors for read_df. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_read_df_params_error.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Store metadata to create the file + storage.store_metadata( + meta_md5="meta_md5", + element={"sub": "001"}, + meta={ + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "test"}, + "name": "BOLD", + }, + ) with pytest.raises(ValueError, match="Only one of"): storage.read_df(feature_name="name", feature_md5="md5") @@ -113,6 +211,9 @@ def test_read_df_params_error() -> None: with pytest.raises(ValueError, match="At least one of"): storage.read_df() + with pytest.raises(ValueError, match="Feature MD5"): + storage.read_df(feature_md5="md5") + def test_read_df(tmp_path: Path) -> None: """Test read_df. @@ -157,22 +258,76 @@ def test_read_df(tmp_path: Path) -> None: with pytest.raises(ValueError, match="not found"): storage.read_df(feature_name="BOLD") - # TODO: fix this check - # # Store duplicate entry and check error - # storage.store_metadata( - # meta_md5=meta_md5, - # element={"subject": "test-clone"}, - # meta=meta_to_store, - # ) - # storage.store_table( - # meta_md5=meta_md5, - # element={"subject": "test-clone"}, - # data=data, - # columns=col_headers, - # rows_col_name=row_header, - # ) - # with pytest.raises(ValueError, match="More than one feature with name"): - # storage.read_df(feature_name="BOLD_test") + # Store duplicate entry and check error + storage.store_metadata( + meta_md5=meta_md5, + element={"subject": "test-clone"}, + meta=meta_to_store, + ) + storage.store_vector( + meta_md5=meta_md5, + element={"subject": "test-clone"}, + data=data, + col_names=col_headers, + ) + + +def test_store_data_ignore_duplicate(tmp_path: Path) -> None: + """Test duplicate ignore for data store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_duplicate_data_store.hdf5" + # Single storage, must be the uri + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Store data first time + storage._store_data( + kind="vector", + meta_md5="md5", + element=[{"sub": "001"}], + data=np.empty((1, 1)), + ) + # Store data second time, should be ignored + storage._store_data( + kind="vector", + meta_md5="md5", + element=[{"sub": "001"}], + data=np.empty((1, 1)), + ) + + +def test_store_data_incorrect_kwargs(tmp_path: Path) -> None: + """Test incorrect kwargs for data store. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_incorrect_kwargs_data_store.hdf5" + # Single storage, must be the uri + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Store data first time + storage._store_data( + kind="vector", + meta_md5="md5", + element=[{"sub": "001"}], + data=np.empty((1, 1)), + ) + # Store data second time, should be ignored + with pytest.raises(RuntimeError, match="The additional data for"): + storage._store_data( + kind="vector", + meta_md5="md5", + element=[{"sub": "001"}], + data=np.empty((1, 1)), + col_names="col", + ) def test_store_matrix(tmp_path: Path) -> None: -- 2.52.0 From d793fd9fffb8bd2c0f4f6c553b512a1da9905f95 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 13 Feb 2023 16:30:45 +0100 Subject: [PATCH 056/122] chore: update latest.inc --- docs/changes/latest.inc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changes/latest.inc b/docs/changes/latest.inc index ff0da3932..23c012ce6 100644 --- a/docs/changes/latest.inc +++ b/docs/changes/latest.inc @@ -46,6 +46,9 @@ Enhancements - Add :class:`junifer.markers.TemporalSNRParcels` and :class:`junifer.markers.TemporalSNRSpheres` (:gh:`163` by `Leonard Sasse`_). +- Add support for HDF5 feature storage via :class:`junifer.storage.HDF5FeatureStorage` + (:gh:`147` by `Synchon Mandal`_). + Bugs ~~~~ -- 2.52.0 From 7c15628926fa3359a14d2f91b8c7dcd53e032d5c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 15 Feb 2023 10:32:33 +0100 Subject: [PATCH 057/122] update: add option for converting float64 to float32 for data storage via HDF5FeatureStorage --- junifer/storage/hdf5.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index c0cfb06fc..eb0570704 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -39,6 +39,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): if "update", will update existing entry or append (default "update"). compression : {0-9}, optional Level of gzip compression: 0 (lowest) to 9 (highest) (default 7). + force_float32 : bool, optional + Whether to force casting of numpy.ndarray values to float32 if float64 + values are found (default True). **kwargs : dict The keyword arguments passed to the superclass. @@ -54,6 +57,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): single_output: bool = True, overwrite: Union[bool, str] = "update", compression: int = 7, + force_float32: bool = True, **kwargs: str, ) -> None: # Convert str to Path @@ -80,6 +84,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): self.overwrite = overwrite self.compression = compression + self.force_float32 = force_float32 def get_valid_inputs(self) -> List[str]: """Get valid storage types for input. @@ -529,6 +534,10 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Initialize dictionary to aggregate data to write data_to_write = kwargs + # Optional casting of float64 values to float32 for numpy.ndarray + if data.dtype == np.dtype("float64") and self.force_float32: + data = data.astype(dtype=np.dtype("float32"), casting="same_kind") + # Handle cases for existing and new entry if not stored_data: logger.debug(f"Writing new data for {meta_md5} ...") -- 2.52.0 From c81e586588dfbbb626c75b6f148b7379b516546c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 15 Feb 2023 10:33:12 +0100 Subject: [PATCH 058/122] update: add unit tests for HDF5FeatureStorage data type casting --- junifer/storage/tests/test_hdf5.py | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index e72ebfcb3..7c514e66b 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -330,6 +330,55 @@ def test_store_data_incorrect_kwargs(tmp_path: Path) -> None: ) +@pytest.mark.parametrize( + "force, dtype", + [ + (True, "float32"), + (False, "float64"), + ], +) +def test_f64_to_f632_conversion( + tmp_path: Path, force: bool, dtype: str +) -> None: + """Test actual data is casted from float64 to float32. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + force : bool + The parametrized conversion option. + dtype : str + The parametrized expected data type. + + """ + uri = tmp_path / "test_data_conversion.hdf5" + storage = HDF5FeatureStorage(uri=uri, force_float32=force) + # Metadata to store + meta = { + "element": {"subject": "test"}, + "dependencies": ["numpy"], + "marker": {"name": "mark"}, + "type": "BOLD", + } + # Process the metadata + meta_md5, meta_to_store, element_to_store = process_meta(meta) + # Store matrix + storage.store_metadata( + meta_md5=meta_md5, element=element_to_store, meta=meta_to_store + ) + # Store data + storage.store_matrix( + meta_md5=meta_md5, + element=element_to_store, + data=np.arange(4, dtype="float64").reshape((2, 2)), + ) + # Read into dataframe + read_df = storage.read_df(feature_md5=meta_md5) + # Check data type + assert read_df.values.dtype == np.dtype(dtype) + + def test_store_matrix(tmp_path: Path) -> None: """Test matrix store. -- 2.52.0 From 385a970571af83a6ff92a037ab1df275573ba1ce Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Feb 2023 11:53:03 +0100 Subject: [PATCH 059/122] update: make timeseries store as (x, y, t) for HDF5FeatureStorage --- junifer/storage/hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index eb0570704..79f729197 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -371,7 +371,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): elif hdf_data["kind"] == "timeseries": for idx, element in enumerate(hdf_data["element"]): # Get row count for the element - n_rows, _ = hdf_data["data"][idx, :, :].shape + n_rows, _ = hdf_data["data"][:, :, idx].shape for key, val in element.items(): element_idx_dict[key].extend([val] * n_rows) # Add extra column for timepoints @@ -787,7 +787,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): kind="timeseries", meta_md5=meta_md5, element=[element], # convert to list - data=data[np.newaxis, :, :], # convert to 3D + data=data[:, :, np.newaxis], # convert to 3D column_headers=col_names, row_header_column_name="timepoint", ) -- 2.52.0 From 6d92fee48f627f6cc69461830ed97234887a9523 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Feb 2023 11:54:54 +0100 Subject: [PATCH 060/122] update: make vector store as (x, t) for HDF5FeatureStorage --- junifer/storage/hdf5.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 79f729197..3d51262af 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -368,6 +368,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): for element in hdf_data["element"]: for key, val in element.items(): element_idx_dict[key].append(val) + # Convert data to proper 2D + reshaped_data = hdf_data["data"].T elif hdf_data["kind"] == "timeseries": for idx, element in enumerate(hdf_data["element"]): # Get row count for the element @@ -751,14 +753,11 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Flatten out array processed_data = data.ravel() - # Make it 2D - processed_data = processed_data[np.newaxis, :] - self._store_data( kind="vector", meta_md5=meta_md5, element=[element], # convert to list - data=processed_data, + data=processed_data[:, np.newaxis], # convert to 2D column_headers=col_names, ) -- 2.52.0 From 2fafbbb1f27346c0ec235d1556dabbf8c41857fd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Feb 2023 11:55:36 +0100 Subject: [PATCH 061/122] update: make matrix store as (x, y, t) for HDF5FeatureStorage --- junifer/storage/hdf5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 3d51262af..46e8d7fa6 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -353,8 +353,8 @@ class HDF5FeatureStorage(BaseFeatureStorage): element_idx_dict = defaultdict(list) if hdf_data["kind"] == "matrix": - # Get row count - n_rows, n_cols = hdf_data["data"][0, :, :].shape + # Get row and column count + n_rows, n_cols = hdf_data["data"][:, :, 0].shape for element in hdf_data["element"]: for key, val in element.items(): element_idx_dict[key].extend([val] * n_rows) @@ -710,7 +710,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): kind="matrix", meta_md5=meta_md5, element=[element], # convert to list - data=data[np.newaxis, :, :], # convert to 3D + data=data[:, :, np.newaxis], # convert to 3D column_headers=col_names, row_headers=row_names, matrix_kind=matrix_kind, -- 2.52.0 From 8444bc9e40cb1b461e372ca5f91fdbecf62ba452 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Feb 2023 11:55:47 +0100 Subject: [PATCH 062/122] update: concatenate element data along last axis in HDF5FeatureStorage --- junifer/storage/hdf5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 46e8d7fa6..bac499f35 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -595,7 +595,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): { "element": stored_data["element"] + element, "data": np.concatenate( - (stored_data["data"], data), axis=0 + (stored_data["data"], data), axis=-1 ), # for serialization / deserialization of storage type "kind": kind, -- 2.52.0 From 5754b4dad2f3bd6b58f15266cf28c42bc4e802ff Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Feb 2023 15:28:41 +0100 Subject: [PATCH 063/122] update: add forked h5io as a submodule --- .gitmodules | 3 +++ junifer/external/h5io | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 junifer/external/h5io diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..0c541549d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "junifer/external/h5io"] + path = junifer/external/h5io + url = https://github.com/juaml/h5io diff --git a/junifer/external/h5io b/junifer/external/h5io new file mode 160000 index 000000000..880d15347 --- /dev/null +++ b/junifer/external/h5io @@ -0,0 +1 @@ +Subproject commit 880d15347d55682f2a322fe5ca420f042213bcb4 -- 2.52.0 From 89ebd03c80b317725775a5949ead6e5692c85d1a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Feb 2023 16:02:15 +0100 Subject: [PATCH 064/122] chore: update h5io submodule commit --- junifer/external/h5io | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/external/h5io b/junifer/external/h5io index 880d15347..845b12b70 160000 --- a/junifer/external/h5io +++ b/junifer/external/h5io @@ -1 +1 @@ -Subproject commit 880d15347d55682f2a322fe5ca420f042213bcb4 +Subproject commit 845b12b70a073b5032cb8856cb8370763cf294dd -- 2.52.0 From 066208f84284e14675ed211a9e0485a33fec33d0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 12:03:40 +0100 Subject: [PATCH 065/122] update: replace h5io froom pip with forked h5io --- junifer/external/h5io | 2 +- junifer/storage/hdf5.py | 2 +- pyproject.toml | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/junifer/external/h5io b/junifer/external/h5io index 845b12b70..9642fb63e 160000 --- a/junifer/external/h5io +++ b/junifer/external/h5io @@ -1 +1 @@ -Subproject commit 845b12b70a073b5032cb8856cb8370763cf294dd +Subproject commit 9642fb63e107da4d95b359de272739a68f18a868 diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index bac499f35..437c40559 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -11,7 +11,7 @@ from typing import Any, Dict, Iterable, List, Optional, Union import numpy as np import pandas as pd -from h5io import read_hdf5, write_hdf5 +from ..external.h5io.h5io import ChunkedArray, read_hdf5, write_hdf5 from tqdm import tqdm from ..api.decorators import register_storage diff --git a/pyproject.toml b/pyproject.toml index 66dfccfee..635f1781e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ dependencies = [ "sqlalchemy>=1.4.27,<= 1.5.0", "pyyaml>=5.1.2,<7.0", "importlib_metadata; python_version < '3.10'", - "h5io>=0.1.7,<=0.2.0", ] dynamic = ["version"] -- 2.52.0 From 1e73b310dbfd8c97f44fae996e5d1ba75c0a46e1 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 12:04:41 +0100 Subject: [PATCH 066/122] refactor: use two-pass strategy to collect features to avoid memory spikes --- junifer/storage/hdf5.py | 96 ++++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 21 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 437c40559..3ebdcc64b 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -811,29 +811,29 @@ class HDF5FeatureStorage(BaseFeatureStorage): klass=NotImplementedError, ) - logger.info( - "Collecting data from " - f"{self.uri.parent}/*{self.uri.name}" # type: ignore - ) - - # Create new storage instance - out_storage = HDF5FeatureStorage(uri=self.uri, overwrite="update") - # Glob files globbed_files = self.uri.parent.glob( # type: ignore f"*{self.uri.name}" # type: ignore ) - # Run loop to aggregate - for file in tqdm(globbed_files, desc="file"): - logger.debug(f"Reading HDF5 file: {file} ...") - # Create new storage instance to load data - in_storage = HDF5FeatureStorage(uri=file) + # Create new storage instance + out_storage = HDF5FeatureStorage(uri=self.uri, overwrite="update") + # Run loop to collect metdata + logger.info( + "Collecting metadata from " + f"{self.uri.parent}/*{self.uri.name}" # type: ignore + ) + # Collect element files per feature MD5 + elements_per_feature_md5 = defaultdict(list) + for file_ in tqdm(globbed_files, desc="file-metadata"): + logger.debug(f"Reading HDF5 file: {file_} ...") + # Create new storage instance to load metadata + in_storage = HDF5FeatureStorage(uri=file_) # Load metadata from new instance in_metadata = in_storage._read_metadata() - logger.info(f"Updating HDF5 metadata with metadata from: {file}") + logger.info(f"Updating HDF5 metadata with metadata from: {file_}") # Load metadata; empty dictionary if first entry; # can be replaced with store_metadata() if run on a loop # for the metadata entries from in_storage @@ -849,11 +849,65 @@ class HDF5FeatureStorage(BaseFeatureStorage): processed_data=out_metadata, title="meta", ) + # Update element files for found MD5s + for feature_md5 in in_metadata.keys(): + elements_per_feature_md5[feature_md5].append(file_) - for meta_md5 in tqdm(in_metadata.keys(), desc="feature"): - # Load data from new instance - logger.debug(f"Collecting feature MD5: {meta_md5} ...") - in_data = in_storage._read_data(md5=meta_md5) - logger.info(f"Updating HDF5 data with data from: {meta_md5}") - # Save data - out_storage._store_data(meta_md5=meta_md5, **in_data) + # Run loop to collect data per feature per file + logger.info( + "Collecting data from " + f"{self.uri.parent}/*{self.uri.name}" # type: ignore + ) + for feature_md5, element_files in tqdm(elements_per_feature_md5.items(), desc="feature"): + element_count = len(element_files) + # Chunk size for collecting + chunk_size = min(100, element_count) + # Operate on chunks + for chunk_idx, chunk_start in tqdm(enumerate(range(0, element_count, chunk_size)), desc="chunk"): + # Store the chunk files' data + stored_data_for_chunk: List[Dict[str, Any]] = [] + # Read the files of a chunk + for i in tqdm(range(chunk_start, chunk_start + chunk_size), desc="file-data"): + file_ = element_files[i] + logger.debug( + f"Reading feature MD5: '{feature_md5}' " + f"from HDF5 file: {file_} ..." + ) + # Read from HDF5 and collect data + stored_data_for_chunk.append( + read_hdf5( + fname=str(file_), + title=feature_md5, + slash="ignore", + ) + ) + + # Concatenate the features data for a chunk + features_data = np.concatenate( + [x["data"] for x in stored_data_for_chunk], axis=-1 + ) + # Make dictionary to write the collected data; + # first the static data then the dynamic data + data_to_write = { + key: val for key, val in stored_data_for_chunk[0].items() + if key not in ("data", "element") + } + # Join the features element for a chunk + data_to_write["element"] = [x["element"] for x in stored_data_for_chunk] + # Write data in chunks to avoid memory usage spikes + data_to_write["data"] = ChunkedArray( + data=features_data, + shape=(features_data.shape[0], features_data.shape[1], element_count), + chunk_size=(features_data.shape[0], features_data.shape[1], chunk_size), + n_chunk=chunk_idx, + ) + # Write to HDF5 + write_hdf5( + fname=str(self.uri.resolve()), # type: ignore + data=data_to_write, + overwrite=self.overwrite, # type: ignore + compression=0, + title=feature_md5, + slash="error", + use_json=False, + ) -- 2.52.0 From 8a620ec134694214c0212d129323a49400cdded2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 12:05:49 +0100 Subject: [PATCH 067/122] update: stringify Path objects for read_hdf5() and write_hdf5() in HDF5FeatureStorage --- junifer/storage/hdf5.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 3ebdcc64b..dc1d22222 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -190,7 +190,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): """ # Read metadata metadata = read_hdf5( - fname=self.uri, + fname=str(self.uri.resolve()), # type: ignore title="meta", slash="ignore", ) @@ -292,7 +292,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): else: # Read metadata metadata = read_hdf5( - fname=self.uri, + fname=str(self.uri.resolve()), # type: ignore title="meta", slash="ignore", ) @@ -341,7 +341,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Read data from HDF5 hdf_data = read_hdf5( - fname=self.uri, + fname=str(self.uri.resolve()), # type: ignore title=md5, slash="ignore", ) @@ -845,7 +845,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): out_metadata.update(in_metadata) # Save metadata out_storage._write_processed_data( - fname=self.uri.resolve(), # type: ignore + fname=str(self.uri.resolve()), # type: ignore processed_data=out_metadata, title="meta", ) -- 2.52.0 From 2c464290bfb41455c06e7d574a5cb322c676cda2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 15:39:20 +0100 Subject: [PATCH 068/122] chore: add junifer/external/h5io to isort exclusion list --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 47b352ada..4425b6011 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,7 @@ commands = [isort] skip = __init__.py + junifer/external/h5io profile = black line_length = 79 lines_after_imports = 2 -- 2.52.0 From 9258da4f3725be7dc06ac21bc50d98efa9839d5e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 15:39:50 +0100 Subject: [PATCH 069/122] chore: add junifer/external/h5io to flake8 exclusion list --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 4425b6011..aa6d47fd0 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,7 @@ known_third_party = [flake8] exclude = __init__.py + junifer/external/h5io max-line-length = 79 extend-ignore = ; Use of `functools.lru_cache` or `functools.cache` on methods can lead to -- 2.52.0 From 9b36833875f67b2bae89eb18009da694332a3316 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 15:40:03 +0100 Subject: [PATCH 070/122] chore: add junifer/external/h5io to coverage exclusion list --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index aa6d47fd0..67363fe5a 100644 --- a/tox.ini +++ b/tox.ini @@ -139,6 +139,7 @@ omit = */_version.py */tests/* */junifer/configs/* + */junifer/external/* parallel = false [coverage:report] -- 2.52.0 From 382af414291b46925b36be98422210076e219c91 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 15:40:12 +0100 Subject: [PATCH 071/122] chore: add junifer/external/h5io to codespell exclusion list --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 67363fe5a..a90e129ea 100644 --- a/tox.ini +++ b/tox.ini @@ -153,7 +153,7 @@ exclude_lines = precision = 2 [codespell] -skip = docs/auto_*,*.html,.git/,*.pyc,docs/_build +skip = docs/auto_*,*.html,.git/,*.pyc,docs/_build,junifer/external/h5io/ count = quiet-level = 3 ignore-words = ignore_words.txt -- 2.52.0 From 74894335378060a1de44e41e1925e5e3e8490fef Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 15:40:24 +0100 Subject: [PATCH 072/122] chore: add junifer/external/h5io to black exclusion list --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 635f1781e..1aadc9f80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,3 +86,8 @@ write_to = "junifer/_version.py" [tool.black] line-length = 79 target-version = ["py38"] +extend-exclude = """ +( + junifer/external/h5io +) +""" -- 2.52.0 From fcffef6cc945fe502f38e3bd45daa1a39b5c2713 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 15:40:43 +0100 Subject: [PATCH 073/122] chore: lint --- junifer/storage/hdf5.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index dc1d22222..75b006fd8 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -11,10 +11,10 @@ from typing import Any, Dict, Iterable, List, Optional, Union import numpy as np import pandas as pd -from ..external.h5io.h5io import ChunkedArray, read_hdf5, write_hdf5 from tqdm import tqdm from ..api.decorators import register_storage +from ..external.h5io.h5io import ChunkedArray, read_hdf5, write_hdf5 from ..utils import logger, raise_error from .base import BaseFeatureStorage from .utils import element_to_prefix @@ -819,7 +819,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Create new storage instance out_storage = HDF5FeatureStorage(uri=self.uri, overwrite="update") - # Run loop to collect metdata + # Run loop to collect metadata logger.info( "Collecting metadata from " f"{self.uri.parent}/*{self.uri.name}" # type: ignore @@ -858,16 +858,23 @@ class HDF5FeatureStorage(BaseFeatureStorage): "Collecting data from " f"{self.uri.parent}/*{self.uri.name}" # type: ignore ) - for feature_md5, element_files in tqdm(elements_per_feature_md5.items(), desc="feature"): + for feature_md5, element_files in tqdm( + elements_per_feature_md5.items(), desc="feature" + ): element_count = len(element_files) # Chunk size for collecting chunk_size = min(100, element_count) # Operate on chunks - for chunk_idx, chunk_start in tqdm(enumerate(range(0, element_count, chunk_size)), desc="chunk"): + for chunk_idx, chunk_start in tqdm( + enumerate(range(0, element_count, chunk_size)), desc="chunk" + ): # Store the chunk files' data stored_data_for_chunk: List[Dict[str, Any]] = [] # Read the files of a chunk - for i in tqdm(range(chunk_start, chunk_start + chunk_size), desc="file-data"): + for i in tqdm( + range(chunk_start, chunk_start + chunk_size), + desc="file-data", + ): file_ = element_files[i] logger.debug( f"Reading feature MD5: '{feature_md5}' " @@ -889,16 +896,27 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Make dictionary to write the collected data; # first the static data then the dynamic data data_to_write = { - key: val for key, val in stored_data_for_chunk[0].items() + key: val + for key, val in stored_data_for_chunk[0].items() if key not in ("data", "element") } # Join the features element for a chunk - data_to_write["element"] = [x["element"] for x in stored_data_for_chunk] + data_to_write["element"] = [ + x["element"] for x in stored_data_for_chunk + ] # Write data in chunks to avoid memory usage spikes data_to_write["data"] = ChunkedArray( data=features_data, - shape=(features_data.shape[0], features_data.shape[1], element_count), - chunk_size=(features_data.shape[0], features_data.shape[1], chunk_size), + shape=( + features_data.shape[0], + features_data.shape[1], + element_count, + ), + chunk_size=( + features_data.shape[0], + features_data.shape[1], + chunk_size, + ), n_chunk=chunk_idx, ) # Write to HDF5 -- 2.52.0 From 63de952995adf42c76b9b870550698de4de763a1 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Feb 2023 15:41:24 +0100 Subject: [PATCH 074/122] docs: add HDF5FeatureStorage in storage.rst --- docs/understanding/storage.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/understanding/storage.rst b/docs/understanding/storage.rst index 5c8d68495..fb637c91b 100644 --- a/docs/understanding/storage.rst +++ b/docs/understanding/storage.rst @@ -67,4 +67,8 @@ Currently supported storage interfaces * - :class:`junifer.storage.SQLiteFeatureStorage` - ``.sqlite`` - SQLite - - ``matrix``, ``table``, ``timeseries`` + - ``matrix``, ``vector``, ``timeseries`` + * - :class:`junifer.storage.HDF5FeatureStorage` + - ``.hdf5`` + - HDF5 + - ``matrix``, ``vector``, ``timeseries`` -- 2.52.0 From 42c2f266e7971e803f1236cea12977fd0a5674f8 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 27 Feb 2023 14:36:27 +0100 Subject: [PATCH 075/122] fix: bypass loading issues using JSON for encode / decode in HDF5FeatureStorage --- junifer/storage/hdf5.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 75b006fd8..1fd07aa9c 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -356,6 +356,9 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Get row and column count n_rows, n_cols = hdf_data["data"][:, :, 0].shape for element in hdf_data["element"]: + # Bypass for when h5io stores without JSON encode / decode + if isinstance(element, list): + element = element[0] for key, val in element.items(): element_idx_dict[key].extend([val] * n_rows) # Add extra column for row headers @@ -366,12 +369,18 @@ class HDF5FeatureStorage(BaseFeatureStorage): reshaped_data = hdf_data["data"].reshape(-1, n_cols) elif hdf_data["kind"] == "vector": for element in hdf_data["element"]: + # Bypass for when h5io stores without JSON encode / decode + if isinstance(element, list): + element = element[0] for key, val in element.items(): element_idx_dict[key].append(val) # Convert data to proper 2D reshaped_data = hdf_data["data"].T elif hdf_data["kind"] == "timeseries": for idx, element in enumerate(hdf_data["element"]): + # Bypass for when h5io stores without JSON encode / decode + if isinstance(element, list): + element = element[0] # Get row count for the element n_rows, _ = hdf_data["data"][:, :, idx].shape for key, val in element.items(): -- 2.52.0 From cb977aa79f2992a9d058856366f0902187b55f32 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 28 Feb 2023 13:01:13 +0100 Subject: [PATCH 076/122] Revert "fix: bypass loading issues using JSON for encode / decode in HDF5FeatureStorage" This reverts commit 118b3a4ed4f0f6e9e0b415665e9753bbc9c76486. --- junifer/storage/hdf5.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 1fd07aa9c..75b006fd8 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -356,9 +356,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Get row and column count n_rows, n_cols = hdf_data["data"][:, :, 0].shape for element in hdf_data["element"]: - # Bypass for when h5io stores without JSON encode / decode - if isinstance(element, list): - element = element[0] for key, val in element.items(): element_idx_dict[key].extend([val] * n_rows) # Add extra column for row headers @@ -369,18 +366,12 @@ class HDF5FeatureStorage(BaseFeatureStorage): reshaped_data = hdf_data["data"].reshape(-1, n_cols) elif hdf_data["kind"] == "vector": for element in hdf_data["element"]: - # Bypass for when h5io stores without JSON encode / decode - if isinstance(element, list): - element = element[0] for key, val in element.items(): element_idx_dict[key].append(val) # Convert data to proper 2D reshaped_data = hdf_data["data"].T elif hdf_data["kind"] == "timeseries": for idx, element in enumerate(hdf_data["element"]): - # Bypass for when h5io stores without JSON encode / decode - if isinstance(element, list): - element = element[0] # Get row count for the element n_rows, _ = hdf_data["data"][:, :, idx].shape for key, val in element.items(): -- 2.52.0 From 35778ce3f191e134e5b761c3e01e1981b4a1121b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 28 Feb 2023 13:03:09 +0100 Subject: [PATCH 077/122] update: make element key in storage always store list of dict in HDF5FeatureStorage --- junifer/storage/hdf5.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 75b006fd8..b10b6f557 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -6,6 +6,7 @@ from collections import defaultdict +from functools import reduce from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Union @@ -901,9 +902,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): if key not in ("data", "element") } # Join the features element for a chunk - data_to_write["element"] = [ - x["element"] for x in stored_data_for_chunk - ] # Write data in chunks to avoid memory usage spikes data_to_write["data"] = ChunkedArray( data=features_data, @@ -918,6 +916,10 @@ class HDF5FeatureStorage(BaseFeatureStorage): chunk_size, ), n_chunk=chunk_idx, + data_to_write["element"] = reduce( + lambda acc, elem: acc + elem, + [x["element"] for x in stored_data_for_chunk], + [], ) # Write to HDF5 write_hdf5( -- 2.52.0 From 464035536b072ba87df4682a984f7a348db6d8a5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 28 Feb 2023 13:04:06 +0100 Subject: [PATCH 078/122] update: make 2D data storage work for ChunkedArray in HDF5FeatureStorage --- junifer/storage/hdf5.py | 43 +++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index b10b6f557..a347eda93 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -902,25 +902,40 @@ class HDF5FeatureStorage(BaseFeatureStorage): if key not in ("data", "element") } # Join the features element for a chunk - # Write data in chunks to avoid memory usage spikes - data_to_write["data"] = ChunkedArray( - data=features_data, - shape=( - features_data.shape[0], - features_data.shape[1], - element_count, - ), - chunk_size=( - features_data.shape[0], - features_data.shape[1], - chunk_size, - ), - n_chunk=chunk_idx, data_to_write["element"] = reduce( lambda acc, elem: acc + elem, [x["element"] for x in stored_data_for_chunk], [], ) + # Write data in chunks to avoid memory usage spikes + if features_data.ndim == 2: + data_to_write["data"] = ChunkedArray( + data=features_data, + shape=( + features_data.shape[0], + element_count, + ), + chunk_size=( + features_data.shape[0], + chunk_size, + ), + n_chunk=chunk_idx, + ) + elif features_data.ndim == 3: + data_to_write["data"] = ChunkedArray( + data=features_data, + shape=( + features_data.shape[0], + features_data.shape[1], + element_count, + ), + chunk_size=( + features_data.shape[0], + features_data.shape[1], + chunk_size, + ), + n_chunk=chunk_idx, + ) # Write to HDF5 write_hdf5( fname=str(self.uri.resolve()), # type: ignore -- 2.52.0 From 2d22c7e9f60e62c882883118c8a4714ff6d1ce77 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 28 Feb 2023 13:07:28 +0100 Subject: [PATCH 079/122] chore: updated commit checksum for h5io --- junifer/external/h5io | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/external/h5io b/junifer/external/h5io index 9642fb63e..1d2ba9925 160000 --- a/junifer/external/h5io +++ b/junifer/external/h5io @@ -1 +1 @@ -Subproject commit 9642fb63e107da4d95b359de272739a68f18a868 +Subproject commit 1d2ba9925c12f5e3c191eca888b829e43c206658 -- 2.52.0 From 1714364e1730c42071be4efc3a3da13cf960fa13 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 28 Feb 2023 13:22:11 +0100 Subject: [PATCH 080/122] update: enable submodule checkout in ci.yml --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bb84762f..c03cf63a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,10 @@ jobs: run: | git config --global user.email "runner@github.com" git config --global user.name "GitHub Runner" - - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: -- 2.52.0 From 6585b8c4d3aa3faf9b16fb0f5141f3bf91f5b20b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 28 Feb 2023 14:21:42 +0100 Subject: [PATCH 081/122] chore: update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c03cf63a1..2d99e819c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: - submodules: 'recursive' + submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: -- 2.52.0 From dfd16eb639c9d5cb29fa78e89486ae8b910591da Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 12:05:10 +0100 Subject: [PATCH 082/122] update: add submodules init for docs-preview.yml --- .github/workflows/docs-preview.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index 9025eb8af..f92fb18e6 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -24,6 +24,8 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v3 + with: + submodules: true - name: Set up Python 3.10 uses: actions/setup-python@v4 with: -- 2.52.0 From d298abbd511c54a6edf6c59a71f254652081db71 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 12:05:30 +0100 Subject: [PATCH 083/122] chore: remove unused strategy in docs-preview.yml --- .github/workflows/docs-preview.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index f92fb18e6..148061147 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -18,9 +18,6 @@ concurrency: preview-${{ github.ref }} jobs: build-docs: runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - name: Checkout source uses: actions/checkout@v3 -- 2.52.0 From 32ed1d09a15884bb925154fd15dd51296a36a616 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 12:05:51 +0100 Subject: [PATCH 084/122] chore: file ending with newline for docs-preview.yml --- .github/workflows/docs-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index 148061147..8c98b4c35 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -50,4 +50,4 @@ jobs: if: github.event_name == 'pull_request' uses: rossjrw/pr-preview-action@v1 with: - source-dir: docs/_build/main \ No newline at end of file + source-dir: docs/_build/main -- 2.52.0 From 6b7aa2d246aeaf8c932f37379f5e61c578bf1dd1 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 12:43:18 +0100 Subject: [PATCH 085/122] update: add submodules init in docs.yml --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e7ed4c66a..88b0e533f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,6 +21,7 @@ jobs: with: # require all of history to see all tagged versions' docs fetch-depth: 0 + submodules: true - name: Set up Python 3.10 uses: actions/setup-python@v4 with: -- 2.52.0 From a05efce080a2460ed34f039c41a21a954d0d442f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 12:47:05 +0100 Subject: [PATCH 086/122] chore: remove unused strategy in docs.yml --- .github/workflows/docs.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 88b0e533f..2b27452c1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,9 +12,6 @@ on: jobs: build-docs: runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - name: Checkout source uses: actions/checkout@v3 -- 2.52.0 From dc76e1e1536baa911aa1a08fc0d31ff6be2f9930 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 12:49:39 +0100 Subject: [PATCH 087/122] update: change recursive to true for submodules in ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d99e819c..31021044f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: - submodules: recursive + submodules: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: -- 2.52.0 From 74f9c0eddf031aa27c528d6e8789eb0284e6b361 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 16:46:58 +0100 Subject: [PATCH 088/122] chore: add h5py as a dependency in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 1aadc9f80..5de186e71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "sqlalchemy>=1.4.27,<= 1.5.0", "pyyaml>=5.1.2,<7.0", "importlib_metadata; python_version < '3.10'", + "h5py>=3.8.0,<3.9", ] dynamic = ["version"] -- 2.52.0 From 7153e47b1d439f75196d5e4cd1738118d570c2d6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 16:47:27 +0100 Subject: [PATCH 089/122] update: add h5io as a package from git submodule --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5de186e71..c9b507e19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ docs = [ ################ [tool.setuptools] -packages = ["junifer"] +packages = ["junifer", "junifer.external.h5io.h5io"] [tool.setuptools_scm] version_scheme = "guess-next-dev" -- 2.52.0 From 97e56bc42aa0bda1eca9989d1f6eca28f2f482c3 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 16:56:40 +0100 Subject: [PATCH 090/122] update: add chunk_size as optional constructor parameter for HDF5FeatureStorage --- junifer/storage/hdf5.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index a347eda93..fdb9dc885 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -43,6 +43,10 @@ class HDF5FeatureStorage(BaseFeatureStorage): force_float32 : bool, optional Whether to force casting of numpy.ndarray values to float32 if float64 values are found (default True). + chunk_size : int, optional + The chunk size to use when collecting data from element files in + :meth:`junifer.storage.HDF5FeatureStorage.collect`. If the file count + is smaller than the value, the minimum is used (default 100). **kwargs : dict The keyword arguments passed to the superclass. @@ -59,6 +63,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): overwrite: Union[bool, str] = "update", compression: int = 7, force_float32: bool = True, + chunk_size: int = 100, **kwargs: str, ) -> None: # Convert str to Path @@ -86,6 +91,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): self.overwrite = overwrite self.compression = compression self.force_float32 = force_float32 + self.chunk_size = chunk_size def get_valid_inputs(self) -> List[str]: """Get valid storage types for input. @@ -864,7 +870,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): ): element_count = len(element_files) # Chunk size for collecting - chunk_size = min(100, element_count) + chunk_size = min(self.chunk_size, element_count) # Operate on chunks for chunk_idx, chunk_start in tqdm( enumerate(range(0, element_count, chunk_size)), desc="chunk" -- 2.52.0 From 00d33315926f4fe98ebae4b57255cef707ee8922 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 18:14:58 +0100 Subject: [PATCH 091/122] update: add pytest config in pyproject.toml and ignore h5io tests --- pyproject.toml | 4 ++++ tox.ini | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c9b507e19..3327243ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,3 +92,7 @@ extend-exclude = """ junifer/external/h5io ) """ + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "--ignore=junifer/external/h5io -vv" diff --git a/tox.ini b/tox.ini index a90e129ea..e4e7ace7e 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ passenv = deps = pytest commands = - pytest -vv + pytest [testenv:coverage] skip_install = false @@ -57,7 +57,7 @@ deps = pytest pytest-cov commands = - pytest --cov={envsitepackagesdir}/junifer --cov-report=xml -vv --cov-report=term + pytest --cov={envsitepackagesdir}/junifer --cov-report=xml --cov-report=term [testenv:codespell] skip_install = true -- 2.52.0 From ddf0d0e2810309a0c6b5b86c559a04d778b51c4c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 1 Mar 2023 18:15:28 +0100 Subject: [PATCH 092/122] update: ignore h5io for coverage --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e4e7ace7e..0b6688fa0 100644 --- a/tox.ini +++ b/tox.ini @@ -139,7 +139,7 @@ omit = */_version.py */tests/* */junifer/configs/* - */junifer/external/* + */junifer/external/h5io/* parallel = false [coverage:report] -- 2.52.0 From 784d0fadb913460fd169ede14f46b36d67067ece Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 3 Mar 2023 15:34:25 +0100 Subject: [PATCH 093/122] refactor: reduce nesting level for parameter check in HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 204 +++++++++++++++++++--------------------- 1 file changed, 99 insertions(+), 105 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index fdb9dc885..319f11345 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -295,122 +295,116 @@ class HDF5FeatureStorage(BaseFeatureStorage): "must be specified." ) ) - # Parameter check pass - else: - # Read metadata - metadata = read_hdf5( - fname=str(self.uri.resolve()), # type: ignore - title="meta", - slash="ignore", + # Parameter check pass; read metadata + metadata = read_hdf5( + fname=str(self.uri.resolve()), # type: ignore + title="meta", + slash="ignore", + ) + # Initialize MD5 variable + md5: str = "" + + # Consider feature_md5 + if feature_md5 and not feature_name: + logger.debug( + f"Validating feature MD5 '{feature_md5}' in metadata " + f"for: {self.uri.resolve()} ..." # type: ignore ) - # Initialize MD5 variable - md5: str = "" + # Validate MD5 + if feature_md5 in metadata: + md5 = feature_md5 # type: ignore + else: + raise_error(msg=f"Feature MD5 '{feature_md5}' not found") - # Consider feature_md5 - if feature_md5 and not feature_name: - logger.debug( - f"Validating feature MD5 '{feature_md5}' in metadata " - f"for: {self.uri.resolve()} ..." # type: ignore - ) - # Validate MD5 - if feature_md5 in metadata: - md5 = feature_md5 # type: ignore - else: - raise_error(msg=f"Feature MD5 '{feature_md5}' not found") + # Consider feature_name + elif not feature_md5 and feature_name: + logger.debug( + f"Validating feature name '{feature_name}' in metadata " + f"for: {self.uri.resolve()} ..." # type: ignore + ) + # Retrieve MD5 for feature_name + # Implicit counter for duplicate feature_name with different + # MD5; happens when something is wrong with marker computation + feature_name_duplicates_with_different_md5 = [] + for md5, meta in metadata.items(): + if meta["name"] == feature_name: + feature_name_duplicates_with_different_md5.append(md5) - # Consider feature_name - elif not feature_md5 and feature_name: - logger.debug( - f"Validating feature name '{feature_name}' in metadata " - f"for: {self.uri.resolve()} ..." # type: ignore - ) - # Retrieve MD5 for feature_name - # Implicit counter for duplicate feature_name with different - # MD5; happens when something is wrong with marker computation - feature_name_duplicates_with_different_md5 = [] - for md5, meta in metadata.items(): - if meta["name"] == feature_name: - feature_name_duplicates_with_different_md5.append(md5) - - # Check for no / duplicate feature_name - if len(feature_name_duplicates_with_different_md5) == 0: - raise_error(msg=f"Feature '{feature_name}' not found") - elif len(feature_name_duplicates_with_different_md5) > 1: - raise_error( - msg=( - "More than one feature with name " - f"'{feature_name}' found. You can bypass this " - "issue by specifying a `feature_md5`." - ) + # Check for no / duplicate feature_name + if len(feature_name_duplicates_with_different_md5) == 0: + raise_error(msg=f"Feature '{feature_name}' not found") + elif len(feature_name_duplicates_with_different_md5) > 1: + raise_error( + msg=( + "More than one feature with name " + f"'{feature_name}' found. You can bypass this " + "issue by specifying a `feature_md5`." ) + ) - md5 = feature_name_duplicates_with_different_md5[0] + md5 = feature_name_duplicates_with_different_md5[0] - # Read data from HDF5 - hdf_data = read_hdf5( - fname=str(self.uri.resolve()), # type: ignore - title=md5, - slash="ignore", - ) - reshaped_data = hdf_data["data"] + # Read data from HDF5 + hdf_data = read_hdf5( + fname=str(self.uri.resolve()), # type: ignore + title=md5, + slash="ignore", + ) + reshaped_data = hdf_data["data"] - # Generate index for the data - logger.debug(f"Generating pandas.MultiIndex for {md5} ...") - # Create dictionary for aggregating index data - element_idx_dict = defaultdict(list) + # Generate index for the data + logger.debug(f"Generating pandas.MultiIndex for {md5} ...") + # Create dictionary for aggregating index data + element_idx_dict = defaultdict(list) - if hdf_data["kind"] == "matrix": - # Get row and column count - n_rows, n_cols = hdf_data["data"][:, :, 0].shape - for element in hdf_data["element"]: - for key, val in element.items(): - element_idx_dict[key].extend([val] * n_rows) - # Add extra column for row headers - element_idx_dict[ - hdf_data["row_header_column_name"] - ].extend(hdf_data["row_headers"]) - # Convert data from 3D to 2D - reshaped_data = hdf_data["data"].reshape(-1, n_cols) - elif hdf_data["kind"] == "vector": - for element in hdf_data["element"]: - for key, val in element.items(): - element_idx_dict[key].append(val) - # Convert data to proper 2D - reshaped_data = hdf_data["data"].T - elif hdf_data["kind"] == "timeseries": - for idx, element in enumerate(hdf_data["element"]): - # Get row count for the element - n_rows, _ = hdf_data["data"][:, :, idx].shape - for key, val in element.items(): - element_idx_dict[key].extend([val] * n_rows) - # Add extra column for timepoints - element_idx_dict[ - hdf_data["row_header_column_name"] - ].extend(np.arange(n_rows)) - # Convert data from 3D to 2D - reshaped_data = hdf_data["data"].reshape(-1, 1) + if hdf_data["kind"] == "matrix": + # Get row and column count + n_rows, n_cols = hdf_data["data"][:, :, 0].shape + for element in hdf_data["element"]: + for key, val in element.items(): + element_idx_dict[key].extend([val] * n_rows) + # Add extra column for row headers + element_idx_dict[hdf_data["row_header_column_name"]].extend( + hdf_data["row_headers"] + ) + # Convert data from 3D to 2D + reshaped_data = hdf_data["data"].reshape(-1, n_cols) + elif hdf_data["kind"] == "vector": + for element in hdf_data["element"]: + for key, val in element.items(): + element_idx_dict[key].append(val) + # Convert data to proper 2D + reshaped_data = hdf_data["data"].T + elif hdf_data["kind"] == "timeseries": + for idx, element in enumerate(hdf_data["element"]): + # Get row count for the element + n_rows, _ = hdf_data["data"][:, :, idx].shape + for key, val in element.items(): + element_idx_dict[key].extend([val] * n_rows) + # Add extra column for timepoints + element_idx_dict[hdf_data["row_header_column_name"]].extend( + np.arange(n_rows) + ) + # Convert data from 3D to 2D + reshaped_data = hdf_data["data"].reshape(-1, 1) - # Create dataframe for index - idx_df = pd.DataFrame(data=element_idx_dict) - # Create multiindex from dataframe - hdf_data_idx = pd.MultiIndex.from_frame(df=idx_df) - logger.debug(f"Generated pandas.MultiIndex for {md5} ...") + # Create dataframe for index + idx_df = pd.DataFrame(data=element_idx_dict) + # Create multiindex from dataframe + hdf_data_idx = pd.MultiIndex.from_frame(df=idx_df) + logger.debug(f"Generated pandas.MultiIndex for {md5} ...") - # Convert to DataFrame - logger.debug( - f"Converting HDF5 data for {md5} to pandas.DataFrame ..." - ) - df = pd.DataFrame( - data=reshaped_data, - index=hdf_data_idx, - columns=hdf_data["column_headers"], - dtype=hdf_data["data"].dtype, - ) - logger.debug( - f"Converted HDF5 data for {md5} to pandas.DataFrame ..." - ) + # Convert to DataFrame + logger.debug(f"Converting HDF5 data for {md5} to pandas.DataFrame ...") + df = pd.DataFrame( + data=reshaped_data, + index=hdf_data_idx, + columns=hdf_data["column_headers"], + dtype=hdf_data["data"].dtype, + ) + logger.debug(f"Converted HDF5 data for {md5} to pandas.DataFrame ...") - return df + return df def _write_processed_data( self, fname: str, processed_data: Dict[str, Any], title: str -- 2.52.0 From 4456fc6290c11d6eb5091b06ad19190de0840f22 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Mar 2023 10:53:03 +0100 Subject: [PATCH 094/122] chore: simplify parameter check conditional in HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 319f11345..11652aa02 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -305,7 +305,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): md5: str = "" # Consider feature_md5 - if feature_md5 and not feature_name: + if feature_md5: logger.debug( f"Validating feature MD5 '{feature_md5}' in metadata " f"for: {self.uri.resolve()} ..." # type: ignore @@ -317,7 +317,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): raise_error(msg=f"Feature MD5 '{feature_md5}' not found") # Consider feature_name - elif not feature_md5 and feature_name: + elif feature_name: logger.debug( f"Validating feature name '{feature_name}' in metadata " f"for: {self.uri.resolve()} ..." # type: ignore -- 2.52.0 From 5353dfad714ed7220c587bc3d3de57c6491eb0f5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Mar 2023 13:54:45 +0100 Subject: [PATCH 095/122] update: make matrix as one row per element in HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 11652aa02..8f2ee2604 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -7,6 +7,7 @@ from collections import defaultdict from functools import reduce +from itertools import product from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Union @@ -360,31 +361,41 @@ class HDF5FeatureStorage(BaseFeatureStorage): if hdf_data["kind"] == "matrix": # Get row and column count n_rows, n_cols = hdf_data["data"][:, :, 0].shape + # Set rows for the index for element in hdf_data["element"]: for key, val in element.items(): - element_idx_dict[key].extend([val] * n_rows) - # Add extra column for row headers - element_idx_dict[hdf_data["row_header_column_name"]].extend( - hdf_data["row_headers"] - ) + # Only one row per element + element_idx_dict[key].append(val) + # Set column headers for dataframe + columns = list( + product(hdf_data["row_headers"], hdf_data["column_headers"]) + ) # Convert data from 3D to 2D - reshaped_data = hdf_data["data"].reshape(-1, n_cols) + reshaped_data = hdf_data["data"].reshape( + len(hdf_data["element"]), n_rows * n_cols + ) elif hdf_data["kind"] == "vector": + # Set rows for the index for element in hdf_data["element"]: for key, val in element.items(): element_idx_dict[key].append(val) + # Set column headers for dataframe + columns = hdf_data["column_headers"] # Convert data to proper 2D reshaped_data = hdf_data["data"].T elif hdf_data["kind"] == "timeseries": for idx, element in enumerate(hdf_data["element"]): # Get row count for the element n_rows, _ = hdf_data["data"][:, :, idx].shape + # Set rows for the index for key, val in element.items(): element_idx_dict[key].extend([val] * n_rows) # Add extra column for timepoints element_idx_dict[hdf_data["row_header_column_name"]].extend( np.arange(n_rows) ) + # Set column headers for dataframe + columns = hdf_data["column_headers"] # Convert data from 3D to 2D reshaped_data = hdf_data["data"].reshape(-1, 1) @@ -399,7 +410,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): df = pd.DataFrame( data=reshaped_data, index=hdf_data_idx, - columns=hdf_data["column_headers"], + columns=columns, # type: ignore dtype=hdf_data["data"].dtype, ) logger.debug(f"Converted HDF5 data for {md5} to pandas.DataFrame ...") -- 2.52.0 From 4eeaecae3bb415f070da2ef9abb2ab2999b7432b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Mar 2023 15:06:56 +0100 Subject: [PATCH 096/122] update: fix store_matrix tests for HDF5FeatureStorage --- junifer/storage/tests/test_hdf5.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index 7c514e66b..69b4098bb 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -4,6 +4,7 @@ # Federico Raimondo # License: AGPL +from itertools import product from pathlib import Path import numpy as np @@ -428,11 +429,11 @@ def test_store_matrix(tmp_path: Path) -> None: # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check shape of dataframe - assert read_df.shape == (4, 3) + assert read_df.shape == (1, 12) # Check data of dataframe - assert_array_equal(read_df.values, data) + assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == col_headers + assert read_df.columns.to_list() == list(product(row_headers, col_headers)) # Diagonal validation error with pytest.raises(ValueError, match="cannot be False"): @@ -530,11 +531,11 @@ def test_store_matrix_without_headers(tmp_path: Path) -> None: # Read the dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check shape of dataframe - assert read_df.shape == (4, 3) + assert read_df.shape == (1, 12) # Check data of dataframe - assert_array_equal(read_df.values, data) + assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == ["c0", "c1", "c2"] + assert read_df.columns.to_list() == list(product(["r0", "r1", "r2", "r3"], ["c0", "c1", "c2"])) def test_store_upper_triangular_matrix(tmp_path: Path) -> None: @@ -585,9 +586,9 @@ def test_store_upper_triangular_matrix(tmp_path: Path) -> None: # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data) + assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == col_headers + assert read_df.columns.to_list() == list(product(row_headers, col_headers)) def test_store_upper_triangular_matrix_without_diagonal( @@ -641,9 +642,9 @@ def test_store_upper_triangular_matrix_without_diagonal( # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data) + assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == col_headers + assert read_df.columns.to_list() == list(product(row_headers, col_headers)) def test_store_lower_triangular_matrix(tmp_path: Path) -> None: @@ -694,9 +695,9 @@ def test_store_lower_triangular_matrix(tmp_path: Path) -> None: # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data) + assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == col_headers + assert read_df.columns.to_list() == list(product(row_headers, col_headers)) def test_store_lower_triangular_matrix_without_diagonal( @@ -750,9 +751,9 @@ def test_store_lower_triangular_matrix_without_diagonal( # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data) + assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == col_headers + assert read_df.columns.to_list() == list(product(row_headers, col_headers)) def test_store_vector(tmp_path: Path) -> None: -- 2.52.0 From 418ea7411109b4fc239e2393fa5ecaa960642e78 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 8 Mar 2023 15:12:05 +0100 Subject: [PATCH 097/122] chore: lint --- junifer/storage/tests/test_hdf5.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index 69b4098bb..2f3e16730 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -535,7 +535,9 @@ def test_store_matrix_without_headers(tmp_path: Path) -> None: # Check data of dataframe assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == list(product(["r0", "r1", "r2", "r3"], ["c0", "c1", "c2"])) + assert read_df.columns.to_list() == list( + product(["r0", "r1", "r2", "r3"], ["c0", "c1", "c2"]) + ) def test_store_upper_triangular_matrix(tmp_path: Path) -> None: -- 2.52.0 From 18643864cdc6ffb89fb93acd0435ae2ddc1eba60 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 13 Mar 2023 10:07:01 +0100 Subject: [PATCH 098/122] update: remove **kwargs from HDF5FeatureStorage constructor --- junifer/storage/hdf5.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 8f2ee2604..0ff5d78d0 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -48,8 +48,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): The chunk size to use when collecting data from element files in :meth:`junifer.storage.HDF5FeatureStorage.collect`. If the file count is smaller than the value, the minimum is used (default 100). - **kwargs : dict - The keyword arguments passed to the superclass. See Also -------- @@ -65,7 +63,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): compression: int = 7, force_float32: bool = True, chunk_size: int = 100, - **kwargs: str, ) -> None: # Convert str to Path if not isinstance(uri, Path): @@ -86,7 +83,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): uri=uri, storage_types=storage_types, single_output=single_output, - **kwargs, ) self.overwrite = overwrite -- 2.52.0 From 07c769e403c89ecdc09befb1924e17d615d67f07 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 13 Mar 2023 10:18:49 +0100 Subject: [PATCH 099/122] refactor: simplify logic for chunked array store in HDF5FeatureStorage --- junifer/storage/hdf5.py | 46 ++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 0ff5d78d0..1ef29e74a 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -915,34 +915,24 @@ class HDF5FeatureStorage(BaseFeatureStorage): [], ) # Write data in chunks to avoid memory usage spikes - if features_data.ndim == 2: - data_to_write["data"] = ChunkedArray( - data=features_data, - shape=( - features_data.shape[0], - element_count, - ), - chunk_size=( - features_data.shape[0], - chunk_size, - ), - n_chunk=chunk_idx, - ) - elif features_data.ndim == 3: - data_to_write["data"] = ChunkedArray( - data=features_data, - shape=( - features_data.shape[0], - features_data.shape[1], - element_count, - ), - chunk_size=( - features_data.shape[0], - features_data.shape[1], - chunk_size, - ), - n_chunk=chunk_idx, - ) + # Start with the case for 2D + array_shape = [features_data.shape[0]] + array_chunk_size = [features_data.shape[0]] + # Append second dimension for 3D + if features_data.ndim == 3: + array_shape.append(features_data.shape[1]) + array_chunk_size.append(features_data.shape[1]) + # Append final dimension of element count + array_shape.append(element_count) + # Append final dimension of chunk size + array_chunk_size.append(chunk_size) + # Write chunked array + data_to_write["data"] = ChunkedArray( + data=features_data, + shape=tuple(array_shape), + chunk_size=tuple(array_chunk_size), + n_chunk=chunk_idx, + ) # Write to HDF5 write_hdf5( fname=str(self.uri.resolve()), # type: ignore -- 2.52.0 From d66ad6a7255c741140e0c7d67680c5b9310e8f02 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 10:15:50 +0100 Subject: [PATCH 100/122] chore: correct type annotation for matrix_kind in store_matrix() of storage-like objects --- junifer/storage/base.py | 2 +- junifer/storage/hdf5.py | 2 +- junifer/storage/sqlite.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/storage/base.py b/junifer/storage/base.py index 5d2aba2e4..04cdcb71f 100644 --- a/junifer/storage/base.py +++ b/junifer/storage/base.py @@ -190,7 +190,7 @@ class BaseFeatureStorage(ABC): data: np.ndarray, col_names: Optional[Iterable[str]] = None, row_names: Optional[Iterable[str]] = None, - matrix_kind: Optional[str] = "full", + matrix_kind: str = "full", diagonal: bool = True, ) -> None: """Store matrix. diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 1ef29e74a..d4411b250 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -636,7 +636,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): data: np.ndarray, col_names: Optional[Iterable[str]] = None, row_names: Optional[Iterable[str]] = None, - matrix_kind: Optional[str] = "full", + matrix_kind: str = "full", diagonal: bool = True, row_header_col_name: str = "ROI", ) -> None: diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 7f6b83eb6..b48e4d77b 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -400,7 +400,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): data: np.ndarray, col_names: Optional[List[str]] = None, row_names: Optional[List[str]] = None, - matrix_kind: Optional[str] = "full", + matrix_kind: str = "full", diagonal: bool = True, ) -> None: """Store matrix. -- 2.52.0 From 357bfeae0d3465bcd509bd3d077c17426c6ee87e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 10:20:52 +0100 Subject: [PATCH 101/122] update: add function for store_matrix() checks for storage-like objects --- junifer/storage/utils.py | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py index 05de657ba..9b8a8378b 100644 --- a/junifer/storage/utils.py +++ b/junifer/storage/utils.py @@ -139,3 +139,62 @@ def element_to_prefix(element: Dict) -> str: logger.debug(f"Converted prefix: {prefix}") return f"{prefix}_" + + +def store_matrix_checks( + matrix_kind: str, + diagonal: bool, + data_shape: Tuple[int, int], + row_names_len: int, + col_names_len: int, +) -> None: + """Run parameter checks for store_matrix() methods. + + Parameters + ---------- + matrix_kind : {"triu", "tril", "full"} + The kind of matrix: + + * ``triu`` : store upper triangular only + * ``tril`` : store lower triangular + * ``full`` : full matrix + + diagonal : bool + Whether to store the diagonal. If ``matrix_kind`` is "full", + setting this to False will raise an error. + data_shape : tuple of int and int + The shape of the matrix data to store. + row_names_len : int + The length of row labels. + col_names_len : int + The length of column labels. + + """ + # Matrix kind validation + if matrix_kind not in ("triu", "tril", "full"): + raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError) + # Diagonal validation + if diagonal is False and matrix_kind not in ["triu", "tril"]: + raise_error( + msg="Diagonal cannot be False if kind is not full", + klass=ValueError, + ) + # Matrix kind and shape validation + if matrix_kind in ["triu", "tril"]: + if data_shape[0] != data_shape[1]: + raise_error( + "Cannot store a non-square matrix as a triangular matrix", + klass=ValueError, + ) + # Row label validation + if row_names_len != data_shape[0]: # type: ignore + raise_error( + msg="Number of row names does not match number of rows", + klass=ValueError, + ) + # Column label validation + if col_names_len != data_shape[1]: # type: ignore + raise_error( + msg="Number of column names does not match number of columns", + klass=ValueError, + ) -- 2.52.0 From 9d8db1a5a9a09089365412d41072631545188b88 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 10:21:30 +0100 Subject: [PATCH 102/122] update: add tests for function for store_matrix() checks for storage-like objects --- junifer/storage/tests/test_utils.py | 85 ++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/junifer/storage/tests/test_utils.py b/junifer/storage/tests/test_utils.py index 35a6f9c4b..6a8daa087 100644 --- a/junifer/storage/tests/test_utils.py +++ b/junifer/storage/tests/test_utils.py @@ -4,7 +4,7 @@ # Synchon Mandal # License: AGPL -from typing import Dict, List +from typing import Dict, List, Tuple, Union import pytest @@ -12,6 +12,7 @@ from junifer.storage.utils import ( element_to_prefix, get_dependency_version, process_meta, + store_matrix_checks, ) @@ -245,3 +246,85 @@ def test_element_to_prefix_invalid_type() -> None: element = 2.3 with pytest.raises(ValueError, match=r"must be a dict"): element_to_prefix(element) # type: ignore + + +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + { + "matrix_kind": "half", + "diagonal": True, + "data_shape": (1, 1), + "row_names_len": 1, + "col_names_len": 1, + }, + "Invalid kind", + ), + ( + { + "matrix_kind": "full", + "diagonal": False, + "data_shape": (1, 1), + "row_names_len": 1, + "col_names_len": 1, + }, + "Diagonal cannot", + ), + ( + { + "matrix_kind": "triu", + "diagonal": False, + "data_shape": (2, 1), + "row_names_len": 2, + "col_names_len": 1, + }, + "Cannot store a non-square", + ), + ( + { + "matrix_kind": "tril", + "diagonal": False, + "data_shape": (1, 2), + "row_names_len": 1, + "col_names_len": 2, + }, + "Cannot store a non-square", + ), + ( + { + "matrix_kind": "full", + "diagonal": True, + "data_shape": (2, 2), + "row_names_len": 1, + "col_names_len": 2, + }, + "Number of row names", + ), + ( + { + "matrix_kind": "full", + "diagonal": True, + "data_shape": (2, 2), + "row_names_len": 2, + "col_names_len": 1, + }, + "Number of column names", + ), + ], +) +def test_store_matrix_checks( + params: Dict[str, Union[str, bool, Tuple[int, int], int]], err_msg: str +) -> None: + """Test matrix storing parameter checks. + + Parameters + ---------- + params : dict + The parametrized parameters for the function. + err_msg : str + The parametrized substring of expected error message. + + """ + with pytest.raises(ValueError, match=f"{err_msg}"): + store_matrix_checks(**params) # type: ignore -- 2.52.0 From 1d19bebb9212a1b0fe538902afefaf371e51fc05 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 10:53:32 +0100 Subject: [PATCH 103/122] update: use store_matrix_checks() for HDF5FeatureStorage.store_matrix() --- junifer/storage/hdf5.py | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index d4411b250..9f0d6d096 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -19,7 +19,7 @@ from ..api.decorators import register_storage from ..external.h5io.h5io import ChunkedArray, read_hdf5, write_hdf5 from ..utils import logger, raise_error from .base import BaseFeatureStorage -from .utils import element_to_prefix +from .utils import element_to_prefix, store_matrix_checks @register_storage @@ -681,38 +681,20 @@ class HDF5FeatureStorage(BaseFeatureStorage): match data column count. """ - # Matrix kind validation - if matrix_kind not in ("triu", "tril", "full"): - raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError) - # Diagonal validation - if diagonal is False and matrix_kind not in ["triu", "tril"]: - raise_error( - msg="Diagonal cannot be False if kind is not full", - klass=ValueError, - ) - # Matrix kind and shape validation - if matrix_kind in ["triu", "tril"]: - if data.shape[0] != data.shape[1]: - raise_error( - "Cannot store a non-square matrix as a triangular matrix", - klass=ValueError, - ) - # Row data and label validation + # Row data validation if row_names is None: row_names = [f"r{i}" for i in range(data.shape[0])] - elif len(row_names) != data.shape[0]: # type: ignore - raise_error( - msg="Number of row names does not match number of rows", - klass=ValueError, - ) - # Column data and label validation + # Column data validation if col_names is None: col_names = [f"c{i}" for i in range(data.shape[1])] - elif len(col_names) != data.shape[1]: # type: ignore - raise_error( - msg="Number of column names does not match number of columns", - klass=ValueError, - ) + # Parameter checks + store_matrix_checks( + matrix_kind=matrix_kind, + diagonal=diagonal, + data_shape=data.shape, + row_names_len=len(row_names), # type: ignore + col_names_len=len(col_names), # type: ignore + ) # Store self._store_data( kind="matrix", -- 2.52.0 From e88532eb7f81e121a3ece13db88340c8bf8a8462 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 11:50:16 +0100 Subject: [PATCH 104/122] update: use store_matrix_checks() for SQLiteFeatureStorage.store_matrix() --- junifer/storage/sqlite.py | 50 +++++++++++++-------------------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index b48e4d77b..13a378b24 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -18,7 +18,7 @@ from tqdm import tqdm from ..api.decorators import register_storage from ..utils import logger, raise_error, warn_with_log from .pandas_base import PandasBaseFeatureStorage -from .utils import element_to_prefix +from .utils import element_to_prefix, store_matrix_checks if TYPE_CHECKING: @@ -432,48 +432,32 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): this to False will raise an error (default True). """ - if diagonal is False and matrix_kind not in ["triu", "tril"]: - raise_error( - msg="Diagonal cannot be False if kind is not full", - klass=ValueError, - ) - - if matrix_kind in ["triu", "tril"]: - if data.shape[0] != data.shape[1]: - raise_error( - "Cannot store a non-square matrix as a triangular matrix", - klass=ValueError, - ) - + # Row data validation + if row_names is None: + row_names = [f"r{i}" for i in range(data.shape[0])] + # Column data validation + if col_names is None: + col_names = [f"c{i}" for i in range(data.shape[1])] + # Parameter checks + store_matrix_checks( + matrix_kind=matrix_kind, + diagonal=diagonal, + data_shape=data.shape, + row_names_len=len(row_names), # type: ignore + col_names_len=len(col_names), # type: ignore + ) + # Data storage manipulations if matrix_kind == "triu": k = 0 if diagonal is True else 1 data_idx = np.triu_indices(data.shape[0], k=k) elif matrix_kind == "tril": k = 0 if diagonal is True else -1 data_idx = np.tril_indices(data.shape[0], k=k) - elif matrix_kind == "full": + else: # full data_idx = ( np.repeat(np.arange(data.shape[0]), data.shape[1]), np.tile(np.arange(data.shape[1]), data.shape[0]), ) - else: - raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError) - - if row_names is None: - row_names = [f"r{i}" for i in range(data.shape[0])] - elif len(row_names) != data.shape[0]: - raise_error( - msg="Number of row names does not match number of rows", - klass=ValueError, - ) - - if col_names is None: - col_names = [f"c{i}" for i in range(data.shape[1])] - elif len(col_names) != data.shape[1]: - raise_error( - msg="Number of column names does not match number of columns", - klass=ValueError, - ) # Subset data flat_data = data[data_idx] -- 2.52.0 From 36df6585061ba3b24e50628d816e7e6fc53f5fe3 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 13:49:24 +0100 Subject: [PATCH 105/122] update: add function for matrix to vector for storage-like objects --- junifer/storage/utils.py | 62 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py index 9b8a8378b..ef8d2d77f 100644 --- a/junifer/storage/utils.py +++ b/junifer/storage/utils.py @@ -7,7 +7,9 @@ import hashlib import json from importlib.metadata import PackageNotFoundError, version -from typing import Dict, Tuple +from typing import Dict, Iterable, List, Tuple + +import numpy as np from ..utils.logging import logger, raise_error @@ -198,3 +200,61 @@ def store_matrix_checks( msg="Number of column names does not match number of columns", klass=ValueError, ) + + +def matrix_to_vector( + data: np.ndarray, + col_names: Iterable[str], + row_names: Iterable[str], + matrix_kind: str, + diagonal: bool, +) -> Tuple[np.ndarray, List[str]]: + """Convert matrix to vector based on parameters. + + Parameters + ---------- + data : 2D numpy.ndarray + The matrix data to store. + col_names : list or tuple of str + The column labels. + row_names : list or tuple of str + The row labels. + matrix_kind : str + The kind of matrix: + + * ``triu`` : store upper triangular only + * ``tril`` : store lower triangular + * ``full`` : full matrix + + diagonal : bool + Whether to store the diagonal. + + Returns + ------- + 1D numpy.ndarray + The vector data. + list of str + The column labels. + + """ + # Prepare data indexing based on matrix kind + if matrix_kind == "triu": + k = 0 if diagonal is True else 1 + data_idx = np.triu_indices(data.shape[0], k=k) + elif matrix_kind == "tril": + k = 0 if diagonal is True else -1 + data_idx = np.tril_indices(data.shape[0], k=k) + else: # full + data_idx = ( + np.repeat(np.arange(data.shape[0]), data.shape[1]), + np.tile(np.arange(data.shape[1]), data.shape[0]), + ) + # Subset data as 1D + flat_data = data[data_idx] + # Generate flat 1D row X column names + columns = [ + f"{row_names[i]}~{col_names[j]}" # type: ignore + for i, j in zip(data_idx[0], data_idx[1]) + ] + + return flat_data[np.newaxis, :], columns -- 2.52.0 From 27e2b5f3f41c60ebc76b28a194d6d75dc844dd84 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 13:50:03 +0100 Subject: [PATCH 106/122] update: add tests for function for matrix to vector for storage-like objects --- junifer/storage/tests/test_utils.py | 91 ++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/junifer/storage/tests/test_utils.py b/junifer/storage/tests/test_utils.py index 6a8daa087..6ddf389cc 100644 --- a/junifer/storage/tests/test_utils.py +++ b/junifer/storage/tests/test_utils.py @@ -4,13 +4,16 @@ # Synchon Mandal # License: AGPL -from typing import Dict, List, Tuple, Union +from typing import Dict, Iterable, List, Tuple, Union +import numpy as np import pytest +from numpy.testing import assert_array_equal from junifer.storage.utils import ( element_to_prefix, get_dependency_version, + matrix_to_vector, process_meta, store_matrix_checks, ) @@ -328,3 +331,89 @@ def test_store_matrix_checks( """ with pytest.raises(ValueError, match=f"{err_msg}"): store_matrix_checks(**params) # type: ignore + + +@pytest.mark.parametrize( + "params, expected_data, expected_columns", + [ + ( + { + "data": np.arange(9).reshape(3, 3), + "col_names": ["c0", "c1", "c2"], + "row_names": ("r0", "r1", "r2"), + "matrix_kind": "triu", + "diagonal": True, + }, + np.array([[0, 1, 2, 4, 5, 8]]), + ["r0~c0", "r0~c1", "r0~c2", "r1~c1", "r1~c2", "r2~c2"], + ), + ( + { + "data": np.arange(9).reshape(3, 3), + "col_names": ("c0", "c1", "c2"), + "row_names": ["r0", "r1", "r2"], + "matrix_kind": "triu", + "diagonal": False, + }, + np.array([[1, 2, 5]]), + ["r0~c1", "r0~c2", "r1~c2"], + ), + ( + { + "data": np.arange(9).reshape(3, 3), + "col_names": ("c0", "c1", "c2"), + "row_names": ["r0", "r1", "r2"], + "matrix_kind": "tril", + "diagonal": True, + }, + np.array([[0, 3, 4, 6, 7, 8]]), + ["r0~c0", "r1~c0", "r1~c1", "r2~c0", "r2~c1", "r2~c2"], + ), + ( + { + "data": np.arange(9).reshape(3, 3), + "col_names": ("c0", "c1", "c2"), + "row_names": ["r0", "r1", "r2"], + "matrix_kind": "tril", + "diagonal": False, + }, + np.array([[3, 6, 7]]), + ["r1~c0", "r2~c0", "r2~c1"], + ), + ( + { + "data": np.arange(9).reshape(3, 3), + "col_names": ("c0", "c1", "c2"), + "row_names": ["r0", "r1", "r2"], + "matrix_kind": "full", + "diagonal": False, + }, + np.arange(9).reshape(1, -1), + [ + f"{r}~{c}" + for r in ["r0", "r1", "r2"] + for c in ["c0", "c1", "c2"] + ], + ), + ], +) +def test_matrix_to_vector( + params: Dict[str, Union[np.ndarray, Iterable[str], str, bool]], + expected_data: np.ndarray, + expected_columns: List[str], +) -> None: + """Test matrix to vector. + + Parameters + ---------- + params : dict + The parametrized parameters for the function. + expected_data : np.ndarray + The parametrized vector data to expect. + expected_columns : np.ndarray + The parametrized columns labels to expect. + + """ + data, columns = matrix_to_vector(**params) # type: ignore + assert_array_equal(data, expected_data) + assert columns == expected_columns -- 2.52.0 From 473fda41c9128fd65bf6db75a99d23943ac941d2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 13:52:22 +0100 Subject: [PATCH 107/122] update: use matrix_to_vector() for SQLiteFeatureStorage.store_matrix() --- junifer/storage/sqlite.py | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 13a378b24..fd2f9cc49 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -18,7 +18,7 @@ from tqdm import tqdm from ..api.decorators import register_storage from ..utils import logger, raise_error, warn_with_log from .pandas_base import PandasBaseFeatureStorage -from .utils import element_to_prefix, store_matrix_checks +from .utils import element_to_prefix, matrix_to_vector, store_matrix_checks if TYPE_CHECKING: @@ -446,26 +446,14 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): row_names_len=len(row_names), # type: ignore col_names_len=len(col_names), # type: ignore ) - # Data storage manipulations - if matrix_kind == "triu": - k = 0 if diagonal is True else 1 - data_idx = np.triu_indices(data.shape[0], k=k) - elif matrix_kind == "tril": - k = 0 if diagonal is True else -1 - data_idx = np.tril_indices(data.shape[0], k=k) - else: # full - data_idx = ( - np.repeat(np.arange(data.shape[0]), data.shape[1]), - np.tile(np.arange(data.shape[1]), data.shape[0]), - ) - - # Subset data - flat_data = data[data_idx] - # Generate flat 1D row X column names - columns = [ - f"{row_names[i]}~{col_names[j]}" - for i, j in zip(data_idx[0], data_idx[1]) - ] + # Matrix to vector conversion + flat_data, columns = matrix_to_vector( + data=data, + col_names=col_names, + row_names=row_names, + matrix_kind=matrix_kind, + diagonal=diagonal, + ) # Convert element metadata to index n_rows = 1 @@ -473,7 +461,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): element=element, n_rows=n_rows, rows_col_name=None ) # Prepare new dataframe - data_df = pd.DataFrame(flat_data[None, :], columns=columns, index=idx) + data_df = pd.DataFrame(flat_data, columns=columns, index=idx) # SQLite's SQLITE_MAX_COLUMN is 2000, so if more than that, # convert it to long format -- 2.52.0 From cc0e20e240ce807ac2bf5c87377a7d8adc31a2c4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 14:58:15 +0100 Subject: [PATCH 108/122] update: use matrix_to_vector() for HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 9f0d6d096..c1c0b4875 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -7,7 +7,6 @@ from collections import defaultdict from functools import reduce -from itertools import product from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Union @@ -19,7 +18,7 @@ from ..api.decorators import register_storage from ..external.h5io.h5io import ChunkedArray, read_hdf5, write_hdf5 from ..utils import logger, raise_error from .base import BaseFeatureStorage -from .utils import element_to_prefix, store_matrix_checks +from .utils import element_to_prefix, matrix_to_vector, store_matrix_checks @register_storage @@ -355,21 +354,35 @@ class HDF5FeatureStorage(BaseFeatureStorage): element_idx_dict = defaultdict(list) if hdf_data["kind"] == "matrix": - # Get row and column count - n_rows, n_cols = hdf_data["data"][:, :, 0].shape + # Initialize list to store element-wise flatttened data + flattened_data_list = [] # Set rows for the index - for element in hdf_data["element"]: + for idx, element in enumerate(hdf_data["element"]): for key, val in element.items(): # Only one row per element element_idx_dict[key].append(val) - # Set column headers for dataframe - columns = list( - product(hdf_data["row_headers"], hdf_data["column_headers"]) - ) - # Convert data from 3D to 2D - reshaped_data = hdf_data["data"].reshape( - len(hdf_data["element"]), n_rows * n_cols - ) + # Flatten data for element + # Get columns for first iteration only + if idx == 0: + flat_data, columns = matrix_to_vector( + data=hdf_data["data"][:, :, idx], + col_names=hdf_data["column_headers"], + row_names=hdf_data["row_headers"], + matrix_kind=hdf_data["matrix_kind"], + diagonal=bool(hdf_data["diagonal"]), + ) + else: + flat_data, _ = matrix_to_vector( + data=hdf_data["data"][:, :, idx], + col_names=hdf_data["column_headers"], + row_names=hdf_data["row_headers"], + matrix_kind=hdf_data["matrix_kind"], + diagonal=bool(hdf_data["diagonal"]), + ) + flattened_data_list.append(flat_data) + + # Vertically stack data + reshaped_data = np.vstack(flattened_data_list) elif hdf_data["kind"] == "vector": # Set rows for the index for element in hdf_data["element"]: -- 2.52.0 From 63ed17a54f8f186b6f28863a671b655dcfc67b8f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 14:59:05 +0100 Subject: [PATCH 109/122] update: adapt read_df() tests of matrix for HDF5FeatureStorage --- junifer/storage/tests/test_hdf5.py | 95 ++++++++++-------------------- 1 file changed, 30 insertions(+), 65 deletions(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index 2f3e16730..a78fb705d 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -4,7 +4,6 @@ # Federico Raimondo # License: AGPL -from itertools import product from pathlib import Path import numpy as np @@ -433,59 +432,9 @@ def test_store_matrix(tmp_path: Path) -> None: # Check data of dataframe assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == list(product(row_headers, col_headers)) - - # Diagonal validation error - with pytest.raises(ValueError, match="cannot be False"): - storage.store_matrix( - meta_md5=meta_md5, - element=element_to_store, - data=data, - row_names=row_headers, - col_names=col_headers, - matrix_kind="full", - diagonal=False, - ) - # Matrix kind and shape validation - with pytest.raises(ValueError, match="non-square"): - storage.store_matrix( - meta_md5=meta_md5, - element=element_to_store, - data=data, - row_names=row_headers, - col_names=col_headers, - matrix_kind="triu", - ) - # Matrix kind validation - with pytest.raises(ValueError, match="Invalid kind"): - storage.store_matrix( - meta_md5=meta_md5, - element=element_to_store, - data=data, - row_names=row_headers, - col_names=col_headers, - matrix_kind="wrong", - ) - # Row data and label validation - with pytest.raises(ValueError, match="of row names"): - storage.store_matrix( - meta_md5=meta_md5, - element=element_to_store, - data=data, - row_names=["row1", "row2", "row3"], - col_names=col_headers, - matrix_kind="full", - ) - # Column data and label validation - with pytest.raises(ValueError, match="of column names"): - storage.store_matrix( - meta_md5=meta_md5, - element=element_to_store, - data=data, - row_names=row_headers, - col_names=["col1", "col2", "col3", "col4"], - matrix_kind="full", - ) + assert read_df.columns.to_list() == [ + f"{row}~{col}" for row in row_headers for col in col_headers + ] def test_store_matrix_without_headers(tmp_path: Path) -> None: @@ -535,9 +484,11 @@ def test_store_matrix_without_headers(tmp_path: Path) -> None: # Check data of dataframe assert_array_equal(read_df.values, data.reshape(1, -1)) # Check column headers - assert read_df.columns.to_list() == list( - product(["r0", "r1", "r2", "r3"], ["c0", "c1", "c2"]) - ) + assert read_df.columns.to_list() == [ + f"{row}~{col}" + for row in ["r0", "r1", "r2", "r3"] + for col in ["c0", "c1", "c2"] + ] def test_store_upper_triangular_matrix(tmp_path: Path) -> None: @@ -588,9 +539,16 @@ def test_store_upper_triangular_matrix(tmp_path: Path) -> None: # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data.reshape(1, -1)) + assert_array_equal(read_df.values, np.array([[1, 2, 3, 22, 33, 333]])) # Check column headers - assert read_df.columns.to_list() == list(product(row_headers, col_headers)) + assert read_df.columns.to_list() == [ + "row1~col1", + "row1~col2", + "row1~col3", + "row2~col2", + "row2~col3", + "row3~col3", + ] def test_store_upper_triangular_matrix_without_diagonal( @@ -644,9 +602,9 @@ def test_store_upper_triangular_matrix_without_diagonal( # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data.reshape(1, -1)) + assert_array_equal(read_df.values, np.array([[2, 3, 33]])) # Check column headers - assert read_df.columns.to_list() == list(product(row_headers, col_headers)) + assert read_df.columns.to_list() == ["row1~col2", "row1~col3", "row2~col3"] def test_store_lower_triangular_matrix(tmp_path: Path) -> None: @@ -697,9 +655,16 @@ def test_store_lower_triangular_matrix(tmp_path: Path) -> None: # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data.reshape(1, -1)) + assert_array_equal(read_df.values, np.array([[1, 11, 22, 111, 222, 333]])) # Check column headers - assert read_df.columns.to_list() == list(product(row_headers, col_headers)) + assert read_df.columns.to_list() == [ + "row1~col1", + "row2~col1", + "row2~col2", + "row3~col1", + "row3~col2", + "row3~col3", + ] def test_store_lower_triangular_matrix_without_diagonal( @@ -753,9 +718,9 @@ def test_store_lower_triangular_matrix_without_diagonal( # Read into dataframe read_df = storage.read_df(feature_md5=meta_md5) # Check data of dataframe - assert_array_equal(read_df.values, data.reshape(1, -1)) + assert_array_equal(read_df.values, np.array([[11, 111, 222]])) # Check column headers - assert read_df.columns.to_list() == list(product(row_headers, col_headers)) + assert read_df.columns.to_list() == ["row2~col1", "row3~col1", "row3~col2"] def test_store_vector(tmp_path: Path) -> None: -- 2.52.0 From 4dbd53ab73c97082884e68b1808ef870ce2ecf19 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 15:03:33 +0100 Subject: [PATCH 110/122] chore: codespell fix --- junifer/tests/test_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/tests/test_stats.py b/junifer/tests/test_stats.py index 71b660698..ef36b1071 100644 --- a/junifer/tests/test_stats.py +++ b/junifer/tests/test_stats.py @@ -27,7 +27,7 @@ def test_get_aggfunc_by_name(name: str, params: Optional[Dict]) -> None: Parameters ---------- name : str - The paramterized name of the method name. + The parametrized name of the method name. params : dict The parametrized parameters passed to the method. -- 2.52.0 From 6a086c226f9e0a24dbb8c7a954d9d42c5a61327a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:14:36 +0100 Subject: [PATCH 111/122] update: let storage.utils.matrix_to_vector() take 2D and 3D data --- junifer/storage/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py index ef8d2d77f..629b64f20 100644 --- a/junifer/storage/utils.py +++ b/junifer/storage/utils.py @@ -213,8 +213,8 @@ def matrix_to_vector( Parameters ---------- - data : 2D numpy.ndarray - The matrix data to store. + data : 2D / 3D numpy.ndarray + The matrix / tensor data to store / read. col_names : list or tuple of str The column labels. row_names : list or tuple of str @@ -231,8 +231,8 @@ def matrix_to_vector( Returns ------- - 1D numpy.ndarray - The vector data. + 1D / 2D numpy.ndarray + The vector / matrix data. list of str The column labels. @@ -257,4 +257,4 @@ def matrix_to_vector( for i, j in zip(data_idx[0], data_idx[1]) ] - return flat_data[np.newaxis, :], columns + return flat_data, columns -- 2.52.0 From 324e7fef360053e4d974d6a69bacb13b450cbe89 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:15:07 +0100 Subject: [PATCH 112/122] update: adapt tests for storage.utils.matrix_to_vector() --- junifer/storage/tests/test_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/junifer/storage/tests/test_utils.py b/junifer/storage/tests/test_utils.py index 6ddf389cc..8e043b1ec 100644 --- a/junifer/storage/tests/test_utils.py +++ b/junifer/storage/tests/test_utils.py @@ -344,7 +344,7 @@ def test_store_matrix_checks( "matrix_kind": "triu", "diagonal": True, }, - np.array([[0, 1, 2, 4, 5, 8]]), + np.array([0, 1, 2, 4, 5, 8]), ["r0~c0", "r0~c1", "r0~c2", "r1~c1", "r1~c2", "r2~c2"], ), ( @@ -355,7 +355,7 @@ def test_store_matrix_checks( "matrix_kind": "triu", "diagonal": False, }, - np.array([[1, 2, 5]]), + np.array([1, 2, 5]), ["r0~c1", "r0~c2", "r1~c2"], ), ( @@ -366,7 +366,7 @@ def test_store_matrix_checks( "matrix_kind": "tril", "diagonal": True, }, - np.array([[0, 3, 4, 6, 7, 8]]), + np.array([0, 3, 4, 6, 7, 8]), ["r0~c0", "r1~c0", "r1~c1", "r2~c0", "r2~c1", "r2~c2"], ), ( @@ -377,7 +377,7 @@ def test_store_matrix_checks( "matrix_kind": "tril", "diagonal": False, }, - np.array([[3, 6, 7]]), + np.array([3, 6, 7]), ["r1~c0", "r2~c0", "r2~c1"], ), ( @@ -388,7 +388,7 @@ def test_store_matrix_checks( "matrix_kind": "full", "diagonal": False, }, - np.arange(9).reshape(1, -1), + np.arange(9), [ f"{r}~{c}" for r in ["r0", "r1", "r2"] -- 2.52.0 From e82f734584ecfa251501587156db6ec7f9dd6611 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:15:27 +0100 Subject: [PATCH 113/122] update: adapt usage of matrix_to_vector() for SQLiteFeatureStorage --- junifer/storage/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index fd2f9cc49..26bb1aaf2 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -461,7 +461,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): element=element, n_rows=n_rows, rows_col_name=None ) # Prepare new dataframe - data_df = pd.DataFrame(flat_data, columns=columns, index=idx) + data_df = pd.DataFrame(flat_data[np.newaxis, :], columns=columns, index=idx) # SQLite's SQLITE_MAX_COLUMN is 2000, so if more than that, # convert it to long format -- 2.52.0 From b3f908e3e73f2fe69ef37203a47b27dcdd0ea5c7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:16:11 +0100 Subject: [PATCH 114/122] update: simplify usage of matrix_to_vector() in HDF5FeatureStorage --- junifer/storage/hdf5.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index c1c0b4875..0229088e3 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -354,35 +354,21 @@ class HDF5FeatureStorage(BaseFeatureStorage): element_idx_dict = defaultdict(list) if hdf_data["kind"] == "matrix": - # Initialize list to store element-wise flatttened data - flattened_data_list = [] # Set rows for the index for idx, element in enumerate(hdf_data["element"]): for key, val in element.items(): # Only one row per element element_idx_dict[key].append(val) - # Flatten data for element - # Get columns for first iteration only - if idx == 0: - flat_data, columns = matrix_to_vector( - data=hdf_data["data"][:, :, idx], - col_names=hdf_data["column_headers"], - row_names=hdf_data["row_headers"], - matrix_kind=hdf_data["matrix_kind"], - diagonal=bool(hdf_data["diagonal"]), - ) - else: - flat_data, _ = matrix_to_vector( - data=hdf_data["data"][:, :, idx], - col_names=hdf_data["column_headers"], - row_names=hdf_data["row_headers"], - matrix_kind=hdf_data["matrix_kind"], - diagonal=bool(hdf_data["diagonal"]), - ) - flattened_data_list.append(flat_data) - - # Vertically stack data - reshaped_data = np.vstack(flattened_data_list) + # Flatten data and get column headers for dataframe + flat_data, columns = matrix_to_vector( + data=hdf_data["data"], + col_names=hdf_data["column_headers"], + row_names=hdf_data["row_headers"], + matrix_kind=hdf_data["matrix_kind"], + diagonal=bool(hdf_data["diagonal"]), + ) + # Convert data to proper 2D + reshaped_data = flat_data.T elif hdf_data["kind"] == "vector": # Set rows for the index for element in hdf_data["element"]: -- 2.52.0 From 8f2786ef8081c10a34449031bb5dd2c83a0066f3 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:16:45 +0100 Subject: [PATCH 115/122] chore: update nilearn constraints in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3327243ae..6db486d44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "datalad>=0.15.4,<0.18", "pandas>=1.4.0,<1.6", "nibabel>=3.2.0,<4.1", - "nilearn>=0.9.0,<1.0", + "nilearn>=0.9.0,<=0.10.0", "sqlalchemy>=1.4.27,<= 1.5.0", "pyyaml>=5.1.2,<7.0", "importlib_metadata; python_version < '3.10'", -- 2.52.0 From 4b96eea1f205507e0b5aa6e46e0852639dcebeba Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:17:03 +0100 Subject: [PATCH 116/122] chore: update datalad constraints in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6db486d44..c7c05cbef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ classifiers = [ dependencies = [ "click>=8.1.3,<8.2", "numpy>=1.22,<1.24", - "datalad>=0.15.4,<0.18", + "datalad>=0.15.4,<0.19", "pandas>=1.4.0,<1.6", "nibabel>=3.2.0,<4.1", "nilearn>=0.9.0,<=0.10.0", -- 2.52.0 From e4414e5f4863c5854c3c0fe4740ce943364df369 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:17:19 +0100 Subject: [PATCH 117/122] docs: increment docs copyright year to 2023 --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index a11f4ed53..a72c37e38 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ sys.path.append((curdir / "sphinxext").as_posix()) # -- Project information ----------------------------------------------------- project = "junifer" -copyright = "2022, Authors of junifer" +copyright = "2023, Authors of junifer" author = "Fede Raimondo" # -- General configuration --------------------------------------------------- -- 2.52.0 From ad6a14a8042ffc81f1628d00997d375101483868 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:17:44 +0100 Subject: [PATCH 118/122] chore: isort --- junifer/datagrabber/aomic/id1000.py | 2 +- junifer/datagrabber/aomic/piop2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/datagrabber/aomic/id1000.py b/junifer/datagrabber/aomic/id1000.py index 1a1b188e2..428192b46 100644 --- a/junifer/datagrabber/aomic/id1000.py +++ b/junifer/datagrabber/aomic/id1000.py @@ -7,7 +7,7 @@ # License: AGPL from pathlib import Path -from typing import Union, Dict +from typing import Dict, Union from junifer.datagrabber import PatternDataladDataGrabber diff --git a/junifer/datagrabber/aomic/piop2.py b/junifer/datagrabber/aomic/piop2.py index b9fad565e..284d259a3 100644 --- a/junifer/datagrabber/aomic/piop2.py +++ b/junifer/datagrabber/aomic/piop2.py @@ -7,7 +7,7 @@ # License: AGPL from pathlib import Path -from typing import List, Union, Dict +from typing import Dict, List, Union from junifer.datagrabber import PatternDataladDataGrabber -- 2.52.0 From cf22bb1a4c4c3dff057177b099e4cc9faad36bca Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:19:39 +0100 Subject: [PATCH 119/122] chore: flake8 --- junifer/storage/hdf5.py | 2 +- junifer/storage/sqlite.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 0229088e3..e35517c05 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -355,7 +355,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): if hdf_data["kind"] == "matrix": # Set rows for the index - for idx, element in enumerate(hdf_data["element"]): + for element in hdf_data["element"]: for key, val in element.items(): # Only one row per element element_idx_dict[key].append(val) diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 26bb1aaf2..e511c9e49 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -461,7 +461,9 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): element=element, n_rows=n_rows, rows_col_name=None ) # Prepare new dataframe - data_df = pd.DataFrame(flat_data[np.newaxis, :], columns=columns, index=idx) + data_df = pd.DataFrame( + flat_data[np.newaxis, :], columns=columns, index=idx + ) # SQLite's SQLITE_MAX_COLUMN is 2000, so if more than that, # convert it to long format -- 2.52.0 From 75207fad537c9c0702eda96398d4c0edcb58d9c7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 16:53:03 +0100 Subject: [PATCH 120/122] fix: adapt test for dependency version checks --- junifer/storage/tests/test_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/junifer/storage/tests/test_utils.py b/junifer/storage/tests/test_utils.py index 8e043b1ec..06c50f497 100644 --- a/junifer/storage/tests/test_utils.py +++ b/junifer/storage/tests/test_utils.py @@ -24,10 +24,10 @@ from junifer.storage.utils import ( [ ("click", "8.2"), ("numpy", "1.24"), - ("datalad", "0.18"), + ("datalad", "0.19"), ("pandas", "1.6"), ("nibabel", "4.1"), - ("nilearn", "1.0"), + ("nilearn", "0.10.0"), ("sqlalchemy", "1.5.0"), ("pyyaml", "7.0"), ], @@ -44,7 +44,10 @@ def test_get_dependency_version(dependency: str, max_version: str) -> None: """ version = get_dependency_version(dependency) - assert version < max_version + if len(version.split(".")) == 3: # semver + assert int(version.split(".")[1]) < int(max_version.split(".")[1]) + else: + assert version < max_version def test_get_dependency_version_invalid() -> None: -- 2.52.0 From b526fa08eadf2ec5341f5fb86e59f0c1ea951c7b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 18:37:47 +0100 Subject: [PATCH 121/122] fix: add missing constraint for dependency check test --- junifer/storage/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/storage/tests/test_utils.py b/junifer/storage/tests/test_utils.py index 06c50f497..dd7bab3bd 100644 --- a/junifer/storage/tests/test_utils.py +++ b/junifer/storage/tests/test_utils.py @@ -45,7 +45,7 @@ def test_get_dependency_version(dependency: str, max_version: str) -> None: """ version = get_dependency_version(dependency) if len(version.split(".")) == 3: # semver - assert int(version.split(".")[1]) < int(max_version.split(".")[1]) + assert int(version.split(".")[1]) <= int(max_version.split(".")[1]) else: assert version < max_version -- 2.52.0 From bdbd677f29da4a928b557a96e5f483b14fd4e424 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 14 Mar 2023 22:16:42 +0100 Subject: [PATCH 122/122] update: simplify element index creation for HDF5FeatureStorage.read_df() --- junifer/storage/hdf5.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index e35517c05..7ccc1fcac 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -350,15 +350,10 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Generate index for the data logger.debug(f"Generating pandas.MultiIndex for {md5} ...") - # Create dictionary for aggregating index data - element_idx_dict = defaultdict(list) if hdf_data["kind"] == "matrix": - # Set rows for the index - for element in hdf_data["element"]: - for key, val in element.items(): - # Only one row per element - element_idx_dict[key].append(val) + # Set index for element + element_idx = hdf_data["element"] # Flatten data and get column headers for dataframe flat_data, columns = matrix_to_vector( data=hdf_data["data"], @@ -370,23 +365,23 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Convert data to proper 2D reshaped_data = flat_data.T elif hdf_data["kind"] == "vector": - # Set rows for the index - for element in hdf_data["element"]: - for key, val in element.items(): - element_idx_dict[key].append(val) + # Set index for element + element_idx = hdf_data["element"] # Set column headers for dataframe columns = hdf_data["column_headers"] # Convert data to proper 2D reshaped_data = hdf_data["data"].T elif hdf_data["kind"] == "timeseries": + # Create dictionary for aggregating index data + element_idx = defaultdict(list) for idx, element in enumerate(hdf_data["element"]): # Get row count for the element n_rows, _ = hdf_data["data"][:, :, idx].shape # Set rows for the index for key, val in element.items(): - element_idx_dict[key].extend([val] * n_rows) + element_idx[key].extend([val] * n_rows) # Add extra column for timepoints - element_idx_dict[hdf_data["row_header_column_name"]].extend( + element_idx[hdf_data["row_header_column_name"]].extend( np.arange(n_rows) ) # Set column headers for dataframe @@ -395,7 +390,7 @@ class HDF5FeatureStorage(BaseFeatureStorage): reshaped_data = hdf_data["data"].reshape(-1, 1) # Create dataframe for index - idx_df = pd.DataFrame(data=element_idx_dict) + idx_df = pd.DataFrame(data=element_idx) # type: ignore # Create multiindex from dataframe hdf_data_idx = pd.MultiIndex.from_frame(df=idx_df) logger.debug(f"Generated pandas.MultiIndex for {md5} ...") -- 2.52.0