[BUG]: relative path in with block in YAML is interpreted relative from YAML, but copy operation to junifer jobs directory interprets relative to working directory. #224

Merged
synchon merged 3 commits from fix/with-section-file-copying into main 2023-11-17 11:44:50 +00:00
3 changed files with 13 additions and 1 deletions

View file

@ -0,0 +1 @@
Make copying of assets in ``with`` block of YAML, relative to YAML and not to current working directory by `Fede Raimondo`_ and `Synchon Mandal`_

View file

@ -50,6 +50,8 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
# Convert load modules to list
if not isinstance(to_load, list):
to_load = [to_load]
# Initialize list to have absolute paths for custom modules
final_to_load = []
for t_module in to_load:
if t_module.endswith(".py"):
logger.debug(f"Importing file: {t_module}")
@ -65,9 +67,18 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
module = importlib.util.module_from_spec(spec) # type: ignore
sys.modules[t_module] = module
spec.loader.exec_module(module) # type: ignore
# Add absolute path to final list
final_to_load.append(str(file_path.resolve()))
else:
logger.info(f"Importing module: {t_module}")
importlib.import_module(t_module)
# Add module to final list
final_to_load.append(t_module)
# Replace modules to be loaded so that custom modules will take the
# absolute path. This was not the case as found in #224. Similar thing
# is done with the storage URI below.
contents["with"] = final_to_load
# Compute path for the URI parameter in storage files that are relative
# This is a tricky thing that appeared in #127. The problem is that

View file

@ -53,7 +53,7 @@ def test_parse_yaml_success_with_module_autoload(tmp_path: Path) -> None:
assert "foo" in contents
assert contents["foo"] == "bar"
assert "with" in contents
assert contents["with"] == "numpy"
assert contents["with"] == ["numpy"]
assert "numpy" in sys.modules
assert "junifer.configs.wrong_config" not in sys.modules