From e3e9db9a1bde9ba4fc9315b4be3a8c39cd2318bd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 15:28:23 +0100 Subject: [PATCH 1/8] chore: update h5io commit head --- junifer/external/h5io | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/external/h5io b/junifer/external/h5io index 1d2ba9925..4413a9fc3 160000 --- a/junifer/external/h5io +++ b/junifer/external/h5io @@ -1 +1 @@ -Subproject commit 1d2ba9925c12f5e3c191eca888b829e43c206658 +Subproject commit 4413a9fc3db2ca8fd0982f86e28b326eee6e0b28 -- 2.52.0 From 5ac5c6429ba386d71fa203a9766db1ae41115fc3 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 15:29:18 +0100 Subject: [PATCH 2/8] refactor: update HDF5FeatureStorage._read_metadata() --- junifer/storage/hdf5.py | 42 +++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 7ccc1fcac..416e000bb 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -151,34 +151,40 @@ class HDF5FeatureStorage(BaseFeatureStorage): Raises ------ - IOError - If HDF5 file or `meta` does not exist. + FileNotFoundError + If HDF5 file does not exist. + RuntimeError + If ``meta`` does not exist in the file. """ # Get correct URI for element; # is different from uri if single_output is False uri = self._fetch_correct_uri_for_io(element=element) - try: - logger.info(f"Loading HDF5 metadata from: {uri}") - metadata = read_hdf5( - fname=uri, - title="meta", - slash="ignore", - ) - except IOError: + # Check if file exists + if not Path(uri).exists(): raise_error( - msg=f"HDF5 file not found at: {uri}", - klass=IOError, + f"HDF5 file not found at: {uri}", + klass=FileNotFoundError, ) - except ValueError: + + # Check if group is found in the storage + if not has_hdf5(fname=uri, title="meta"): raise_error( - msg=f"`meta` not found in: {uri}", - klass=IOError, + f"Invalid junifer HDF5 file at: {uri}", + klass=RuntimeError, ) - else: - logger.info(f"Loaded HDF5 metadata from: {uri}") - return metadata + + # Read metadata + logger.info(f"Loading HDF5 metadata from: {uri}") + metadata = read_hdf5( + fname=uri, + title="meta", + slash="ignore", + ) + logger.info(f"Loaded HDF5 metadata from: {uri}") + + return metadata def list_features(self) -> Dict[str, Dict[str, Any]]: """List the features in the storage. -- 2.52.0 From 5b75f7b757626a1efbda2ab580f706decda7cfbe Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 15:30:02 +0100 Subject: [PATCH 3/8] refactor: update HDF5FeatureStorage._read_data() --- junifer/storage/hdf5.py | 44 +++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 416e000bb..278bf6de7 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -15,7 +15,7 @@ import pandas as pd from tqdm import tqdm from ..api.decorators import register_storage -from ..external.h5io.h5io import ChunkedArray, read_hdf5, write_hdf5 +from ..external.h5io.h5io import ChunkedArray, has_hdf5, read_hdf5, write_hdf5 from ..utils import logger, raise_error from .base import BaseFeatureStorage from .utils import element_to_prefix, matrix_to_vector, store_matrix_checks @@ -224,34 +224,40 @@ class HDF5FeatureStorage(BaseFeatureStorage): Raises ------ - IOError - If HDF5 file or data does not exist. + FileNotFoundError + If HDF5 file does not exist. + RuntimeError + If the specified ``md5`` does not exist in the file. """ # Get correct URI for element; # is different from uri if single_output is False uri = self._fetch_correct_uri_for_io(element=element) - try: - logger.info(f"Loading HDF5 data for {md5} from: {uri}") - data = read_hdf5( - fname=uri, - title=md5, - slash="ignore", - ) - except IOError: + # Check if file exists + if not Path(uri).exists(): raise_error( - msg=f"HDF5 file not found at: {uri}", - klass=IOError, + f"HDF5 file not found at: {uri}", + klass=FileNotFoundError, ) - except ValueError: + + # Check if group is found in the storage + if not has_hdf5(fname=uri, title=md5): raise_error( - msg=f"`{md5}` not found in: {uri}", - klass=IOError, + f"{md5} not found in HDF5 file at: {uri}", + klass=RuntimeError, ) - else: - logger.info(f"Loaded HDF5 data for {md5} from: {uri}") - return data + + # Read data + logger.info(f"Loading HDF5 data for {md5} from: {uri}") + data = read_hdf5( + fname=uri, + title=md5, + slash="ignore", + ) + logger.info(f"Loaded HDF5 data for {md5} from: {uri}") + + return data def read_df( self, -- 2.52.0 From ee1c62b4e6dec574666e4cec49d82c33ffc11031 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 15:30:57 +0100 Subject: [PATCH 4/8] update: improve HDF5FeatureStorage.store_metadata() --- junifer/storage/hdf5.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 278bf6de7..0af0ccb6a 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -474,11 +474,15 @@ class HDF5FeatureStorage(BaseFeatureStorage): The metadata as a dictionary. """ - # Read metadata; if no file found, create an empty dictionary - try: + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element) + + # Check if file exists, then read metadata else create empty dictionary + if Path(uri).exists(): metadata = self._read_metadata(element=element) - except IOError: - logger.debug(f"Creating new metadata map for {meta_md5} ...") + else: + logger.debug(f"Creating new file at {uri} ...") metadata = {} # Only add entry if MD5 is not present @@ -487,10 +491,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): # Update metadata metadata[meta_md5] = meta - # Get correct URI for element; - # is different from uri if single_output is False - uri = self._fetch_correct_uri_for_io(element=element) - logger.info(f"Writing HDF5 metadata for {meta_md5} to: {uri}") logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") logger.debug( -- 2.52.0 From 61ea1729ab3336c51c701b279835531529b8e9b0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 15:31:33 +0100 Subject: [PATCH 5/8] update: improve HDF5FeatureStorage._store_data() --- junifer/storage/hdf5.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/junifer/storage/hdf5.py b/junifer/storage/hdf5.py index 0af0ccb6a..631e44625 100644 --- a/junifer/storage/hdf5.py +++ b/junifer/storage/hdf5.py @@ -540,10 +540,15 @@ class HDF5FeatureStorage(BaseFeatureStorage): Keyword arguments passed from the calling method. """ - # Read existing data; if no file found, create an empty dictionary - try: + # Get correct URI for element; + # is different from uri if single_output is False + uri = self._fetch_correct_uri_for_io(element=element[0]) + + # Check if MD5 exists, then read data else create empty dictionary + # File should be present here already + if has_hdf5(fname=uri, title=meta_md5): stored_data = self._read_data(md5=meta_md5, element=element[0]) - except IOError: + else: logger.debug(f"Creating new data map for {meta_md5} ...") stored_data = {} @@ -616,10 +621,6 @@ class HDF5FeatureStorage(BaseFeatureStorage): } ) - # Get correct URI for element; - # is different from uri if single_output is False - uri = self._fetch_correct_uri_for_io(element=element[0]) - logger.info(f"Writing HDF5 data for {meta_md5} to: {uri}") logger.debug(f"HDF5 overwrite is set to: {self.overwrite} ...") logger.debug( -- 2.52.0 From 1d69f065efb62f705800806d702a105b50626d08 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 15:31:54 +0100 Subject: [PATCH 6/8] chore: prune tests for HDF5FeatureStorage --- junifer/storage/tests/test_hdf5.py | 81 ------------------------------ 1 file changed, 81 deletions(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index a78fb705d..511b04082 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -36,29 +36,6 @@ def test_single_output(tmp_path: Path) -> None: assert storage.single_output is True -def test_single_output_meta_not_found_error(tmp_path: Path) -> None: - """Test single output metadata not found error. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - uri = tmp_path / "test_single_output_no_meta.hdf5" - storage = HDF5FeatureStorage(uri=uri, single_output=True) - # Store data to create the file - storage._store_data( - kind="vector", - meta_md5="md5", - element=[{"sub": "001"}], - data=np.empty((1, 1)), - ) - # Check metadata error - with pytest.raises(IOError, match="`meta` not found in:"): - storage._read_metadata() - - def test_single_output_file_not_found_error(tmp_path: Path) -> None: """Test single output file not found error. @@ -272,64 +249,6 @@ def test_read_df(tmp_path: Path) -> None: ) -def test_store_data_ignore_duplicate(tmp_path: Path) -> None: - """Test duplicate ignore for data store. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - uri = tmp_path / "test_duplicate_data_store.hdf5" - # Single storage, must be the uri - storage = HDF5FeatureStorage(uri=uri, single_output=True) - # Store data first time - storage._store_data( - kind="vector", - meta_md5="md5", - element=[{"sub": "001"}], - data=np.empty((1, 1)), - ) - # Store data second time, should be ignored - storage._store_data( - kind="vector", - meta_md5="md5", - element=[{"sub": "001"}], - data=np.empty((1, 1)), - ) - - -def test_store_data_incorrect_kwargs(tmp_path: Path) -> None: - """Test incorrect kwargs for data store. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - uri = tmp_path / "test_incorrect_kwargs_data_store.hdf5" - # Single storage, must be the uri - storage = HDF5FeatureStorage(uri=uri, single_output=True) - # Store data first time - storage._store_data( - kind="vector", - meta_md5="md5", - element=[{"sub": "001"}], - data=np.empty((1, 1)), - ) - # Store data second time, should be ignored - with pytest.raises(RuntimeError, match="The additional data for"): - storage._store_data( - kind="vector", - meta_md5="md5", - element=[{"sub": "001"}], - data=np.empty((1, 1)), - col_names="col", - ) - - @pytest.mark.parametrize( "force, dtype", [ -- 2.52.0 From 7f12c2e81952f9d7218917d1b78b48430fae8ce2 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 16:57:37 +0100 Subject: [PATCH 7/8] update: add missing tests for HDF5FeatureStorage --- junifer/storage/tests/test_hdf5.py | 87 ++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/junifer/storage/tests/test_hdf5.py b/junifer/storage/tests/test_hdf5.py index 511b04082..05b29de85 100644 --- a/junifer/storage/tests/test_hdf5.py +++ b/junifer/storage/tests/test_hdf5.py @@ -6,6 +6,7 @@ from pathlib import Path +import h5py import numpy as np import pytest from numpy.testing import assert_array_equal @@ -36,22 +37,6 @@ def test_single_output(tmp_path: Path) -> None: assert storage.single_output is True -def test_single_output_file_not_found_error(tmp_path: Path) -> None: - """Test single output file not found error. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - uri = tmp_path / "test_single_output_no_file.hdf5" - storage = HDF5FeatureStorage(uri=uri, single_output=True) - # Check file error - with pytest.raises(IOError, match="HDF5 file not found at:"): - storage._read_data(md5="md5") - - def test_multi_output_error(tmp_path: Path) -> None: """Test error for multi output. @@ -86,6 +71,76 @@ def test_single_output_parent_path_creation(tmp_path: Path) -> None: assert to_create_hdf5.exists() +def test_read_metadata_file_not_found_error(tmp_path: Path) -> None: + """Test file not found error when reading metadata. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_read_metadata_no_file.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Check file not found error + with pytest.raises(FileNotFoundError, match="HDF5 file not found at:"): + storage._read_metadata() + + +def test_read_metadata_meta_not_found_error(tmp_path: Path) -> None: + """Test meta not found error when reading metadata. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_read_metadata_no_meta.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Create file + with h5py.File(uri, "w") as f: + f.create_dataset("mydataset", (100,), dtype="i") + # Check meta not found error + with pytest.raises(RuntimeError, match="Invalid junifer HDF5 file at:"): + storage._read_metadata() + + +def test_read_data_file_not_found_error(tmp_path: Path) -> None: + """Test file not found error when reading data. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_read_data_no_file.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Check file not found error + with pytest.raises(FileNotFoundError, match="HDF5 file not found at:"): + storage._read_data(md5="md5") + + +def test_read_data_md5_not_found_error(tmp_path: Path) -> None: + """Test meta not found error when reading data. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + uri = tmp_path / "test_read_data_no_md5.hdf5" + storage = HDF5FeatureStorage(uri=uri, single_output=True) + # Create file + with h5py.File(uri, "w") as f: + f.create_dataset("mydataset", (100,), dtype="i") + # Check MD5 not found error + with pytest.raises(RuntimeError, match="not found in HDF5 file at:"): + storage._read_data(md5="md5") + + def test_store_metadata_and_list_features(tmp_path: Path) -> None: """Test metadata store and features listing. -- 2.52.0 From b6bf310fb2c72ac4a60ec32e495c2cb6b6745f6f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 20 Mar 2023 17:01:18 +0100 Subject: [PATCH 8/8] chore: update latest.inc --- docs/changes/latest.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changes/latest.inc b/docs/changes/latest.inc index 007443848..f7e833622 100644 --- a/docs/changes/latest.inc +++ b/docs/changes/latest.inc @@ -61,6 +61,8 @@ Enhancements - Allow for empty parcels in :class:`junifer.markers.ParcelAggregation`, that will result in NaNs (:gh:`194` by `Fede Raimondo`_). +- Improve metadata and data I/O for :class:`junifer.storage.HDF5FeatureStorage` (:gh:`196` by `Synchon Mandal`_). + Bugs ~~~~ -- 2.52.0