diff --git a/junifer/datagrabber/pattern.py b/junifer/datagrabber/pattern.py index ad1690ba5..e32dc45af 100644 --- a/junifer/datagrabber/pattern.py +++ b/junifer/datagrabber/pattern.py @@ -86,8 +86,7 @@ class PatternDataGrabber(BaseDataGrabber): glob_pattern = pattern for t_r in self.replacements: # Replace the first of each with a named group definition - re_pattern = re_pattern.replace( - f"{{{t_r}}}", f"(?P<{t_r}>.*)", 1) + re_pattern = re_pattern.replace(f"{{{t_r}}}", f"(?P<{t_r}>.*)", 1) for t_r in self.replacements: # Replace the second appearance of each with the named group diff --git a/junifer/datagrabber/tests/test_multiple.py b/junifer/datagrabber/tests/test_multiple.py index eedd94965..5d591c23a 100644 --- a/junifer/datagrabber/tests/test_multiple.py +++ b/junifer/datagrabber/tests/test_multiple.py @@ -3,7 +3,8 @@ # Authors: Federico Raimondo # License: AGPL -from junifer.datagrabber import PatternDataladDataGrabber, MultipleDataGrabber +from junifer.datagrabber import MultipleDataGrabber, PatternDataladDataGrabber + _testing_dataset = { "example_bids": { @@ -34,14 +35,16 @@ def test_multiple() -> None: uri=repo_uri, types=["T1w"], patterns=pattern1, - replacements=replacements) + replacements=replacements, + ) dg2 = PatternDataladDataGrabber( rootdir=rootdir, uri=repo_uri, types=["bold"], patterns=pattern2, - replacements=replacements) + replacements=replacements, + ) dg = MultipleDataGrabber([dg1, dg2]) expected_subs = [ @@ -73,14 +76,16 @@ def test_multiple_no_intersection() -> None: uri=repo_uri1, types=["T1w"], patterns=pattern1, - replacements=replacements) + replacements=replacements, + ) dg2 = PatternDataladDataGrabber( rootdir=rootdir, uri=repo_uri2, types=["bold"], patterns=pattern2, - replacements=replacements) + replacements=replacements, + ) dg = MultipleDataGrabber([dg1, dg2]) expected_subs = set() diff --git a/junifer/storage/sqlite.py b/junifer/storage/sqlite.py index 79b9457d7..f15903982 100644 --- a/junifer/storage/sqlite.py +++ b/junifer/storage/sqlite.py @@ -391,7 +391,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): # Process metadata meta_md5, t_meta_row = process_meta(t_meta) # Get sqlalchemy engine - engine = self.get_engine(t_meta) + engine = self.get_engine(meta=t_meta) if meta_md5 not in inspect(engine).get_table_names(): # Convert metadata to dataframe meta_df = self._meta_row(meta=t_meta_row, meta_md5=meta_md5) @@ -553,7 +553,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage): table_name = f"meta_{meta_md5}" t_df = in_storage.read_df(feature_md5=meta_md5) # Save data - out_storage._save_upsert(t_df, table_name, if_exist="nocheck") + out_storage._save_upsert(t_df, table_name, if_exists="nocheck") # TODO: refactor diff --git a/junifer/storage/tests/test_sqlite.py b/junifer/storage/tests/test_sqlite.py index 05f7993f7..aa33b9a8d 100644 --- a/junifer/storage/tests/test_sqlite.py +++ b/junifer/storage/tests/test_sqlite.py @@ -152,7 +152,7 @@ def test_upsert_replace(tmp_path: Path) -> None: # Metadata to store meta = {"element": "test", "version": "0.0.1"} # Save to database - storage.store_df(df1, meta) + storage.store_df(df=df1, meta=meta) # Store metadata table_name = storage.store_metadata(meta) # Read stored table @@ -162,7 +162,7 @@ def test_upsert_replace(tmp_path: Path) -> None: # Check if dataframes are equal assert_frame_equal(df1, c_df1) # Upsert using replace - storage._save_upsert(df2, table_name, if_exist="replace") + storage._save_upsert(df=df2, name=table_name, if_exists="replace") # Read stored table c_df2 = _read_sql( table_name=table_name, uri=uri, index_col=["element", "pk2"] @@ -188,9 +188,9 @@ def test_upsert_ignore(tmp_path: Path) -> None: # Metadata to store meta = {"element": "test", "version": "0.0.1"} # Save to database - storage.store_df(df1, meta) + storage.store_df(df=df1, meta=meta) # Store metadata - table_name = storage.store_metadata(meta) + table_name = storage.store_metadata(meta=meta) # Read stored table c_df1 = _read_sql( table_name=table_name, uri=uri, index_col=["element", "pk2"] @@ -206,7 +206,7 @@ def test_upsert_ignore(tmp_path: Path) -> None: assert_frame_equal(c_dfignore, df_ignore) # Check for error with pytest.raises(ValueError, match=r"already exists"): - storage._save_upsert(df2, table_name, if_exist="fail") + storage._save_upsert(df2, table_name, if_exists="fail") def test_upsert_update(tmp_path: Path) -> None: @@ -375,39 +375,23 @@ def test_store_table(tmp_path: Path) -> None: # Check if dataframes are equal assert_frame_equal(df, c_df) - -def test_store_table_check_warning(tmp_path: Path) -> None: - """Test table store and check warning. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - uri = tmp_path / "test_store_table_check_warning.db" - storage = SQLiteFeatureStorage(uri=uri, single_output=True) - # Metadata to store - meta = {"element": "test", "version": "0.0.1", "marker": {"name": "fc"}} - # Data to store - data = [[1, 10], [2, 20], [3, 300], [4, 40], [5, 50], [6, 600]] + # New data to store + data_new = [[1, 10], [2, 20], [3, 300], [4, 40], [5, 50], [6, 600]] # Convert element to index - idx = element_to_index(meta, n_rows=6, rows_col_name="scan") + idx_new = element_to_index(meta, n_rows=6, rows_col_name="scan") # Create dataframe - df = pd.DataFrame(data, columns=["f1", "f2"], index=idx) + df_new = pd.DataFrame(data_new, columns=["f1", "f2"], index=idx_new) # Check warning with pytest.warns(RuntimeWarning, match=r"Some rows"): storage.store_table( - data, meta, columns=["f1", "f2"], rows_col_name="scan" + data_new, meta, columns=["f1", "f2"], rows_col_name="scan" ) - # Store metadata - table_name = storage.store_metadata(meta) # Read stored table - c_df = _read_sql( + c_df_new = _read_sql( table_name=table_name, uri=uri, index_col=["element", "scan"] ) # Check if dataframes are equal - assert_frame_equal(df, c_df) + assert_frame_equal(df_new, c_df_new) # TODO: can the test be parametrized? diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py index bc0bd8faf..b707e80ea 100644 --- a/junifer/storage/utils.py +++ b/junifer/storage/utils.py @@ -63,21 +63,23 @@ def process_meta(meta: Dict) -> Tuple[str, Dict]: """ if meta is None: raise_error(msg="`meta` must be a dict (currently is None)") + # Copy the metadata + t_meta = meta.copy() # Remove key "element" - element = meta.pop("element", None) + element = t_meta.pop("element", None) if element is None: - if "_element_keys" not in meta: + if "_element_keys" not in t_meta: raise_error( msg="`meta` must contain the key 'element' or '_element_keys'" ) else: if isinstance(element, dict): - meta["_element_keys"] = list(element.keys()) + t_meta["_element_keys"] = list(element.keys()) else: - meta["_element_keys"] = ["element"] + t_meta["_element_keys"] = ["element"] # MD5 hash of the metadata - md5_hash = _meta_hash(meta) - return md5_hash, meta + md5_hash = _meta_hash(t_meta) + return md5_hash, t_meta def element_to_prefix(element: Union[Tuple, Dict, str, int]) -> str: diff --git a/junifer/testing/datagrabbers.py b/junifer/testing/datagrabbers.py index 664a7800a..9d0f29415 100644 --- a/junifer/testing/datagrabbers.py +++ b/junifer/testing/datagrabbers.py @@ -39,8 +39,7 @@ class OasisVBMTestingDatagrabber(BaseDataGrabber): """ out = super().__getitem__(element) i_sub = int(element.split("-")[1]) - 1 - out["VBM_GM"] = { - "path": self._dataset.gray_matter_maps[i_sub]} + out["VBM_GM"] = {"path": self._dataset.gray_matter_maps[i_sub]} # Set the element accordingly out["meta"]["element"] = {"subject": element} return out