junifer/datagrabber/pattern.py

@@ -86,8 +86,7 @@ class PatternDataGrabber(BaseDataGrabber):
         glob_pattern = pattern
         for t_r in self.replacements:
             # Replace the first of each with a named group definition
-            re_pattern = re_pattern.replace(
+            re_pattern = re_pattern.replace(f"{{{t_r}}}", f"(?P<{t_r}>.*)", 1)
-                f"{{{t_r}}}", f"(?P<{t_r}>.*)", 1)
 
         for t_r in self.replacements:
             # Replace the second appearance of each with the named group

junifer/datagrabber/tests/test_multiple.py

@@ -3,7 +3,8 @@
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 # License: AGPL
 
-from junifer.datagrabber import PatternDataladDataGrabber, MultipleDataGrabber
+from junifer.datagrabber import MultipleDataGrabber, PatternDataladDataGrabber
+
 
 _testing_dataset = {
     "example_bids": {
@@ -34,14 +35,16 @@ def test_multiple() -> None:
         uri=repo_uri,
         types=["T1w"],
         patterns=pattern1,
-        replacements=replacements)
+        replacements=replacements,
+    )
 
     dg2 = PatternDataladDataGrabber(
         rootdir=rootdir,
         uri=repo_uri,
         types=["bold"],
         patterns=pattern2,
-        replacements=replacements)
+        replacements=replacements,
+    )
 
     dg = MultipleDataGrabber([dg1, dg2])
     expected_subs = [
@@ -73,14 +76,16 @@ def test_multiple_no_intersection() -> None:
         uri=repo_uri1,
         types=["T1w"],
         patterns=pattern1,
-        replacements=replacements)
+        replacements=replacements,
+    )
 
     dg2 = PatternDataladDataGrabber(
         rootdir=rootdir,
         uri=repo_uri2,
         types=["bold"],
         patterns=pattern2,
-        replacements=replacements)
+        replacements=replacements,
+    )
 
     dg = MultipleDataGrabber([dg1, dg2])
     expected_subs = set()

junifer/storage/sqlite.py

@@ -391,7 +391,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
         # Process metadata
         meta_md5, t_meta_row = process_meta(t_meta)
         # Get sqlalchemy engine
-        engine = self.get_engine(t_meta)
+        engine = self.get_engine(meta=t_meta)
         if meta_md5 not in inspect(engine).get_table_names():
             # Convert metadata to dataframe
             meta_df = self._meta_row(meta=t_meta_row, meta_md5=meta_md5)
@@ -553,7 +553,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
                 table_name = f"meta_{meta_md5}"
                 t_df = in_storage.read_df(feature_md5=meta_md5)
                 # Save data
-                out_storage._save_upsert(t_df, table_name, if_exist="nocheck")
+                out_storage._save_upsert(t_df, table_name, if_exists="nocheck")
 
 
 # TODO: refactor

junifer/storage/tests/test_sqlite.py

@@ -152,7 +152,7 @@ def test_upsert_replace(tmp_path: Path) -> None:
     # Metadata to store
     meta = {"element": "test", "version": "0.0.1"}
     # Save to database
-    storage.store_df(df1, meta)
+    storage.store_df(df=df1, meta=meta)
     # Store metadata
     table_name = storage.store_metadata(meta)
     # Read stored table
@@ -162,7 +162,7 @@ def test_upsert_replace(tmp_path: Path) -> None:
     # Check if dataframes are equal
     assert_frame_equal(df1, c_df1)
     # Upsert using replace
-    storage._save_upsert(df2, table_name, if_exist="replace")
+    storage._save_upsert(df=df2, name=table_name, if_exists="replace")
     # Read stored table
     c_df2 = _read_sql(
         table_name=table_name, uri=uri, index_col=["element", "pk2"]
@@ -188,9 +188,9 @@ def test_upsert_ignore(tmp_path: Path) -> None:
     # Metadata to store
     meta = {"element": "test", "version": "0.0.1"}
     # Save to database
-    storage.store_df(df1, meta)
+    storage.store_df(df=df1, meta=meta)
     # Store metadata
-    table_name = storage.store_metadata(meta)
+    table_name = storage.store_metadata(meta=meta)
     # Read stored table
     c_df1 = _read_sql(
         table_name=table_name, uri=uri, index_col=["element", "pk2"]
@@ -206,7 +206,7 @@ def test_upsert_ignore(tmp_path: Path) -> None:
     assert_frame_equal(c_dfignore, df_ignore)
     # Check for error
     with pytest.raises(ValueError, match=r"already exists"):
-        storage._save_upsert(df2, table_name, if_exist="fail")
+        storage._save_upsert(df2, table_name, if_exists="fail")
 
 
 def test_upsert_update(tmp_path: Path) -> None:
@@ -375,39 +375,23 @@ def test_store_table(tmp_path: Path) -> None:
     # Check if dataframes are equal
     assert_frame_equal(df, c_df)
 
-
+    # New data to store
-def test_store_table_check_warning(tmp_path: Path) -> None:
+    data_new = [[1, 10], [2, 20], [3, 300], [4, 40], [5, 50], [6, 600]]
-    """Test table store and check warning.
-
-    Parameters
-    ----------
-    tmp_path : pathlib.Path
-        The path to the test directory.
-
-    """
-    uri = tmp_path / "test_store_table_check_warning.db"
-    storage = SQLiteFeatureStorage(uri=uri, single_output=True)
-    # Metadata to store
-    meta = {"element": "test", "version": "0.0.1", "marker": {"name": "fc"}}
-    # Data to store
-    data = [[1, 10], [2, 20], [3, 300], [4, 40], [5, 50], [6, 600]]
     # Convert element to index
-    idx = element_to_index(meta, n_rows=6, rows_col_name="scan")
+    idx_new = element_to_index(meta, n_rows=6, rows_col_name="scan")
     # Create dataframe
-    df = pd.DataFrame(data, columns=["f1", "f2"], index=idx)
+    df_new = pd.DataFrame(data_new, columns=["f1", "f2"], index=idx_new)
     # Check warning
     with pytest.warns(RuntimeWarning, match=r"Some rows"):
         storage.store_table(
-            data, meta, columns=["f1", "f2"], rows_col_name="scan"
+            data_new, meta, columns=["f1", "f2"], rows_col_name="scan"
         )
-    # Store metadata
-    table_name = storage.store_metadata(meta)
     # Read stored table
-    c_df = _read_sql(
+    c_df_new = _read_sql(
         table_name=table_name, uri=uri, index_col=["element", "scan"]
     )
     # Check if dataframes are equal
-    assert_frame_equal(df, c_df)
+    assert_frame_equal(df_new, c_df_new)
 
 
 # TODO: can the test be parametrized?

junifer/storage/utils.py

@@ -63,21 +63,23 @@ def process_meta(meta: Dict) -> Tuple[str, Dict]:
     """
     if meta is None:
         raise_error(msg="`meta` must be a dict (currently is None)")
+    # Copy the metadata
+    t_meta = meta.copy()
     # Remove key "element"
-    element = meta.pop("element", None)
+    element = t_meta.pop("element", None)
     if element is None:
-        if "_element_keys" not in meta:
+        if "_element_keys" not in t_meta:
             raise_error(
                 msg="`meta` must contain the key 'element' or '_element_keys'"
             )
     else:
         if isinstance(element, dict):
-            meta["_element_keys"] = list(element.keys())
+            t_meta["_element_keys"] = list(element.keys())
         else:
-            meta["_element_keys"] = ["element"]
+            t_meta["_element_keys"] = ["element"]
     # MD5 hash of the metadata
-    md5_hash = _meta_hash(meta)
+    md5_hash = _meta_hash(t_meta)
-    return md5_hash, meta
+    return md5_hash, t_meta
 
 
 def element_to_prefix(element: Union[Tuple, Dict, str, int]) -> str:

junifer/testing/datagrabbers.py

@@ -39,8 +39,7 @@ class OasisVBMTestingDatagrabber(BaseDataGrabber):
         """
         out = super().__getitem__(element)
         i_sub = int(element.split("-")[1]) - 1
-        out["VBM_GM"] = {
+        out["VBM_GM"] = {"path": self._dataset.gray_matter_maps[i_sub]}
-            "path": self._dataset.gray_matter_maps[i_sub]}
         # Set the element accordingly
         out["meta"]["element"] = {"subject": element}
         return out