From de8db8178750e84148b6d55c2652e9ed9144f187 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 11 Jun 2026 15:53:51 +0200 Subject: [PATCH 1/2] update: move parse_yaml from cli to api --- junifer/api/__init__.pyi | 19 +- junifer/api/functions.py | 134 ++++++++++++- junifer/api/tests/test_functions.py | 271 ++++++++++++++++++++++++++- junifer/cli/cli.py | 12 +- junifer/cli/parser.py | 133 +------------ junifer/cli/tests/test_parser.py | 280 ---------------------------- 6 files changed, 428 insertions(+), 421 deletions(-) delete mode 100644 junifer/cli/tests/test_parser.py diff --git a/junifer/api/__init__.pyi b/junifer/api/__init__.pyi index acdae47a3..943720d80 100644 --- a/junifer/api/__init__.pyi +++ b/junifer/api/__init__.pyi @@ -1,4 +1,19 @@ -__all__ = ["decorators", "collect", "queue", "run", "reset", "list_elements"] +__all__ = [ + "decorators", + "collect", + "queue", + "run", + "reset", + "list_elements", + "parse_yaml", +] from . import decorators -from .functions import collect, list_elements, reset, run, queue +from .functions import ( + collect, + list_elements, + parse_yaml, + reset, + run, + queue, +) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 35ab24f19..04392e64a 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -6,8 +6,11 @@ # License: AGPL import atexit +import importlib +import importlib.util import os import shutil +import sys from pathlib import Path import structlog @@ -32,7 +35,14 @@ from ..typing import ( from ..utils import raise_error, warn_with_log, yaml -__all__ = ["collect", "list_elements", "queue", "reset", "run"] +__all__ = [ + "collect", + "list_elements", + "parse_yaml", + "queue", + "reset", + "run", +] _log = structlog.get_logger("junifer") logger = _log.bind(pkg="api") @@ -468,3 +478,125 @@ def list_elements( elements_to_list.append(str_element) return "\n".join(elements_to_list) + + +def parse_yaml(filepath: str | Path) -> dict: # noqa: C901 + """Parse YAML. + + Parameters + ---------- + filepath : str or pathlib.Path + The filepath to read from. + + Returns + ------- + dict + The contents represented as dictionary. + + """ + # Convert str to Path + if not isinstance(filepath, Path): + filepath = Path(filepath) + + logger.info(f"Parsing yaml file: {filepath.absolute()!s}") + # Filepath existence check + if not filepath.exists(): + raise_error(f"File does not exist: {filepath.absolute()!s}") + # Filepath reading + contents = yaml.load(filepath) + if "elements" in contents: + if contents["elements"] is None: + raise_error( + "The elements key was defined but its content is empty. " + "Please define the elements to operate on or remove the key." + ) + # load modules + if "with" in contents: + to_load = contents["with"] + # Convert load modules to list + if not isinstance(to_load, list): + to_load = [to_load] + # Initialize list to have absolute paths for custom modules + final_to_load = [] + for t_module in to_load: + if t_module.endswith(".py"): + logger.debug(f"Importing file: {t_module}") + # This resolves both absolute and relative paths + file_path = filepath.parent / t_module + if not file_path.exists(): + raise_error( + f"File in 'with' section does not exist: {file_path}" + ) + # Add the parent directory to the sys.path so that the + # any imports from this module work correctly + t_path = str(file_path.parent) + if t_path not in sys.path: + sys.path.append(str(file_path.parent)) + + spec = importlib.util.spec_from_file_location( + t_module, file_path + ) + module = importlib.util.module_from_spec(spec) # type: ignore + sys.modules[t_module] = module + spec.loader.exec_module(module) # type: ignore + + # Add absolute path to final list + final_to_load.append(str(file_path.resolve())) + + # Check if the module has junifer_module_deps function + if hasattr(module, "junifer_module_deps"): + logger.debug( + f"Module {t_module} has junifer_module_deps function" + ) + # Get the dependencies + deps = module.junifer_module_deps() + # Add the dependencies to the final list + for dep in deps: + if dep not in final_to_load: + final_to_load.append( + str((file_path.parent / dep).resolve()) + ) + else: + logger.info(f"Importing module: {t_module}") + importlib.import_module(t_module) + # Add module to final list + final_to_load.append(t_module) + + # Replace modules to be loaded so that custom modules will take the + # absolute path. This was not the case as found in #224. Similar thing + # is done with the storage URI below. + contents["with"] = final_to_load + + # Compute path for the URI parameter in storage files that are relative + # This is a tricky thing that appeared in #127. The problem is that + # the path in the URI parameter is relative to YAML file, not to the + # current working directory. If we leave it as is in the contents + # dict, then it will be used later in the ``build`` function as is, + # which will be computed relative to the current working directory. + # The solution is to compute the absolute path and replace the + # relative path in the contents dict with the absolute path. + + # Check if the storage file is defined + if "storage" in contents: + if "uri" in contents["storage"]: + # Check if the storage file is relative + uri_path = Path(contents["storage"]["uri"]) + if not uri_path.is_absolute(): + # Compute the absolute path + contents["storage"]["uri"] = str( + (filepath.parent / uri_path).resolve() + ) + + # Allow relative path if queue env kind is venv; same motivation as above + if "queue" in contents: + if "env" in contents["queue"]: + if "venv" == contents["queue"]["env"]["kind"]: + # Check if the env name is relative + venv_path = Path(contents["queue"]["env"]["name"]) + if not venv_path.is_absolute(): + # Compute the absolute path + contents["queue"]["env"]["name"] = str( + (filepath.parent / venv_path).resolve() + ) + + return contents diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 7d7d780af..d6cc8c67c 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -6,6 +6,7 @@ # License: AGPL import logging +import sys from contextlib import AbstractContextManager, nullcontext from pathlib import Path from typing import Any @@ -15,7 +16,7 @@ from nibabel.filebasedimages import ImageFileError from ruamel.yaml import YAML import junifer.testing.registry # noqa: F401 -from junifer.api import collect, list_elements, queue, reset, run +from junifer.api import collect, list_elements, parse_yaml, queue, reset, run from junifer.datagrabber.base import BaseDataGrabber from junifer.pipeline import PipelineComponentRegistry from junifer.typing import Elements @@ -756,3 +757,271 @@ def test_list_elements( """ listed_elements = list_elements(datagrabber, elements) assert "sub-01" in listed_elements + + +def test_parse_yaml_failure() -> None: + """Test YAML parsing failure.""" + with pytest.raises(ValueError, match="does not exist"): + parse_yaml("foo.yaml") + + +def test_parse_yaml_empty_elements_failure(tmp_path: Path) -> None: + """Test YAML parsing with empty elements failure. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + # Write test file + fname = tmp_path / "test_parse_yaml_empty_elements_failure.yaml" + fname.write_text("elements:") + # Check test file + with pytest.raises(ValueError, match="elements key was defined"): + parse_yaml(fname) + + +def test_parse_yaml_success(tmp_path: Path) -> None: + """Test YAML parsing success. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + # Write test file + fname = tmp_path / "test_parse_yaml_success.yaml" + fname.write_text("foo: bar") + # Check test file + contents = parse_yaml(fname) + assert "foo" in contents + assert contents["foo"] == "bar" + + +def test_parse_yaml_success_with_module_autoload(tmp_path: Path) -> None: + """Test YAML parsing with single module autoload success. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + # Write test file + fname = tmp_path / "test_parse_yaml_with_single_module_autoload.yaml" + fname.write_text("foo: bar\nwith: numpy") + # Check test file + contents = parse_yaml(fname) + assert "foo" in contents + assert contents["foo"] == "bar" + assert "with" in contents + assert contents["with"] == ["numpy"] + assert "numpy" in sys.modules + assert "junifer.configs.wrong_config" not in sys.modules + + +def test_parse_yaml_failure_with_multi_module_autoload(tmp_path: Path) -> None: + """Test YAML parsing with multi module autoload failure. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + # Write test file + fname = tmp_path / "test_parse_yaml_with_multi_module_autoload.yaml" + fname.write_text( + "foo: bar\nwith:\n - numpy\n - junifer.testing.wrong_config" + ) + # Check test file + with pytest.raises(ImportError, match="wrong_config"): + parse_yaml(fname) + + +def test_parse_yaml_with_wrong_path(tmp_path: Path) -> None: + """Test YAML parsing with wrong paths in with. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + t_tmp_path = tmp_path / "test_relative_with" + # Write yaml that includes a relative path + yaml_path = t_tmp_path / "yamls" + yaml_path.mkdir(exist_ok=True, parents=True) + yaml_fname = yaml_path / "test_parse_yaml_wrong_path.yaml" + + yaml_fname.write_text("foo: bar\nwith:\n - missingt.py\n - scipy\n") + + # Check test file + with pytest.raises(ValueError, match="does not exist"): + parse_yaml(yaml_fname) + + +def test_parse_yaml_relative_path(tmp_path: Path) -> None: + """Test YAML parsing with relative paths in with. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + t_tmp_path = tmp_path / "test_relative_with" + + # Write .py to include + py_path = t_tmp_path / "external" + py_path.mkdir(exist_ok=True, parents=True) + py_fname = py_path / "first.py" + py_fname.write_text("import numpy as np\n") + + # Write yaml that includes a relative path + yaml_path = t_tmp_path / "yamls" + yaml_path.mkdir(exist_ok=True, parents=True) + yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml" + + yaml_fname.write_text( + "foo: bar\nwith:\n - ../external/first.py\n - scipy\n" + ) + + # Check test file + parse_yaml(yaml_fname) + + +def test_parse_yaml_absolute_path(tmp_path: Path) -> None: + """Test YAML parsing with absolute paths in with. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + t_tmp_path = tmp_path / "test_relative_with" + + # Write .py to include + py_path = t_tmp_path / "external" + py_path.mkdir(exist_ok=True, parents=True) + py_fname = py_path / "first.py" + py_fname.write_text("import numpy as np\n") + + # Write yaml that includes a relative path + yaml_path = t_tmp_path / "yamls" + yaml_path.mkdir(exist_ok=True, parents=True) + yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml" + + yaml_fname.write_text( + f"foo: bar\nwith:\n - {py_fname.absolute()}\n - scipy\n" + ) + + # Check test file + parse_yaml(yaml_fname) + + +def test_parse_yaml_multi_module_deps(tmp_path: Path) -> None: + """Test YAML parsing with multi-module import with deps. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + t_tmp_path = tmp_path / "test_with_multi_module" + + # Write .py to include + py_path = t_tmp_path / "external" + py_path.mkdir(exist_ok=True, parents=True) + py_fname_1 = py_path / "first.py" + py_fname_1.write_text( + "import numpy as np\nfrom second import hej\n" + "def junifer_module_deps(): return ['second.py']\n" + ) + py_fname_2 = py_path / "second.py" + py_fname_2.write_text("def hej(): print('hej')\n") + + # Write yaml + yaml_path = t_tmp_path / "yamls" + yaml_path.mkdir(exist_ok=True, parents=True) + yaml_fname = yaml_path / "test_parse_yaml_multi_module.yaml" + + yaml_fname.write_text( + "foo: bar\nwith:\n - ../external/first.py\n - scipy\n" + ) + + # Check test file + parse_yaml(yaml_fname) + + +def test_parse_storage_uri_relative(tmp_path: Path) -> None: + """Test YAML parsing with storage and relative URI. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" + fname.write_text("foo: bar\nwith: numpy\nstorage:\n uri: test.db\n") + + contents = parse_yaml(fname) + assert "foo" in contents + assert contents["foo"] == "bar" + assert "storage" in contents + assert "uri" in contents["storage"] + assert contents["storage"]["uri"] == str(tmp_path / "test.db") + + fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" + fname.write_text( + "foo: bar\nwith: numpy\nstorage:\n uri: ../another/test.db\n" + ) + + contents = parse_yaml(fname) + assert "foo" in contents + assert contents["foo"] == "bar" + assert "storage" in contents + assert "uri" in contents["storage"] + assert contents["storage"]["uri"] == str( + (tmp_path / "../another/test.db").resolve() + ) + + fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" + fname.write_text( + "foo: bar\nwith: numpy\nstorage:\n uri: /absolute/test.db\n" + ) + + contents = parse_yaml(fname) + assert "foo" in contents + assert contents["foo"] == "bar" + assert "storage" in contents + assert "uri" in contents["storage"] + assert contents["storage"]["uri"] == "/absolute/test.db" + + # Just to trick coverage + fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" + fname.write_text("foo: bar\nwith: numpy\nstorage:\n kind: SomeStorage\n") + + contents = parse_yaml(fname) + assert "foo" in contents + assert contents["foo"] == "bar" + assert "storage" in contents + + +def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None: + """Test YAML parsing with relative venv queue. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + fname = tmp_path / "test_parse_yaml_queue_venv_relative.yaml" + fname.write_text("queue:\n env:\n kind: venv\n name: .venv\n") + _ = parse_yaml(fname) diff --git a/junifer/cli/cli.py b/junifer/cli/cli.py index c54351c2d..094a98215 100644 --- a/junifer/cli/cli.py +++ b/junifer/cli/cli.py @@ -17,7 +17,7 @@ from ..utils import ( raise_error, yaml, ) -from .parser import parse_elements, parse_yaml +from .parser import parse_elements from .utils import ( _get_dependency_information, _get_environment_information, @@ -162,7 +162,7 @@ def run( configure_logging(level=verbose, level_datalad=verbose_datalad) # TODO(synchon): add validation # Parse YAML - config = parse_yaml(filepath) + config = cli_func.parse_yaml(filepath) # Retrieve working directory workdir = config["workdir"] # Fetch datagrabber @@ -232,7 +232,7 @@ def collect( configure_logging(level=verbose, level_datalad=verbose_datalad) # TODO: add validation # Parse YAML - config = parse_yaml(filepath) + config = cli_func.parse_yaml(filepath) # Fetch storage storage = config["storage"] # Perform operation @@ -299,7 +299,7 @@ def queue( configure_logging(level=verbose, level_datalad=verbose_datalad) # TODO: add validation # Parse YAML - config = parse_yaml(filepath) # type: ignore + config = cli_func.parse_yaml(filepath) # type: ignore # Check queue section if "queue" not in config: raise_error(f"No queue configuration found in {filepath}.") @@ -450,7 +450,7 @@ def reset( # Setup logging configure_logging(level=verbose, level_datalad=verbose_datalad) # Parse YAML - config = parse_yaml(filepath) + config = cli_func.parse_yaml(filepath) # Perform operation cli_func.reset(config) @@ -510,7 +510,7 @@ def list_elements( # Setup logging configure_logging(level=verbose, level_datalad=verbose_datalad) # Parse YAML - config = parse_yaml(filepath) + config = cli_func.parse_yaml(filepath) # Fetch datagrabber datagrabber = config["datagrabber"] # Parse elements diff --git a/junifer/cli/parser.py b/junifer/cli/parser.py index 930b5bad6..8dace937e 100644 --- a/junifer/cli/parser.py +++ b/junifer/cli/parser.py @@ -4,144 +4,15 @@ # Synchon Mandal # License: AGPL -import importlib -import importlib.util -import sys from pathlib import Path import pandas as pd -import structlog from ..typing import Elements -from ..utils import raise_error, warn_with_log, yaml +from ..utils import logger, raise_error, warn_with_log -__all__ = ["parse_elements", "parse_yaml"] - -_log = structlog.get_logger("junifer") -logger = _log.bind(pkg="cli") - - -def parse_yaml(filepath: str | Path) -> dict: # noqa: C901 - """Parse YAML. - - Parameters - ---------- - filepath : str or pathlib.Path - The filepath to read from. - - Returns - ------- - dict - The contents represented as dictionary. - - """ - # Convert str to Path - if not isinstance(filepath, Path): - filepath = Path(filepath) - - logger.info(f"Parsing yaml file: {filepath.absolute()!s}") - # Filepath existence check - if not filepath.exists(): - raise_error(f"File does not exist: {filepath.absolute()!s}") - # Filepath reading - contents = yaml.load(filepath) - if "elements" in contents: - if contents["elements"] is None: - raise_error( - "The elements key was defined but its content is empty. " - "Please define the elements to operate on or remove the key." - ) - # load modules - if "with" in contents: - to_load = contents["with"] - # Convert load modules to list - if not isinstance(to_load, list): - to_load = [to_load] - # Initialize list to have absolute paths for custom modules - final_to_load = [] - for t_module in to_load: - if t_module.endswith(".py"): - logger.debug(f"Importing file: {t_module}") - # This resolves both absolute and relative paths - file_path = filepath.parent / t_module - if not file_path.exists(): - raise_error( - f"File in 'with' section does not exist: {file_path}" - ) - # Add the parent directory to the sys.path so that the - # any imports from this module work correctly - t_path = str(file_path.parent) - if t_path not in sys.path: - sys.path.append(str(file_path.parent)) - - spec = importlib.util.spec_from_file_location( - t_module, file_path - ) - module = importlib.util.module_from_spec(spec) # type: ignore - sys.modules[t_module] = module - spec.loader.exec_module(module) # type: ignore - - # Add absolute path to final list - final_to_load.append(str(file_path.resolve())) - - # Check if the module has junifer_module_deps function - if hasattr(module, "junifer_module_deps"): - logger.debug( - f"Module {t_module} has junifer_module_deps function" - ) - # Get the dependencies - deps = module.junifer_module_deps() - # Add the dependencies to the final list - for dep in deps: - if dep not in final_to_load: - final_to_load.append( - str((file_path.parent / dep).resolve()) - ) - else: - logger.info(f"Importing module: {t_module}") - importlib.import_module(t_module) - # Add module to final list - final_to_load.append(t_module) - - # Replace modules to be loaded so that custom modules will take the - # absolute path. This was not the case as found in #224. Similar thing - # is done with the storage URI below. - contents["with"] = final_to_load - - # Compute path for the URI parameter in storage files that are relative - # This is a tricky thing that appeared in #127. The problem is that - # the path in the URI parameter is relative to YAML file, not to the - # current working directory. If we leave it as is in the contents - # dict, then it will be used later in the ``build`` function as is, - # which will be computed relative to the current working directory. - # The solution is to compute the absolute path and replace the - # relative path in the contents dict with the absolute path. - - # Check if the storage file is defined - if "storage" in contents: - if "uri" in contents["storage"]: - # Check if the storage file is relative - uri_path = Path(contents["storage"]["uri"]) - if not uri_path.is_absolute(): - # Compute the absolute path - contents["storage"]["uri"] = str( - (filepath.parent / uri_path).resolve() - ) - - # Allow relative path if queue env kind is venv; same motivation as above - if "queue" in contents: - if "env" in contents["queue"]: - if "venv" == contents["queue"]["env"]["kind"]: - # Check if the env name is relative - venv_path = Path(contents["queue"]["env"]["name"]) - if not venv_path.is_absolute(): - # Compute the absolute path - contents["queue"]["env"]["name"] = str( - (filepath.parent / venv_path).resolve() - ) - - return contents +__all__ = ["parse_elements"] def parse_elements(element: tuple[str, ...], config: dict) -> Elements | None: diff --git a/junifer/cli/tests/test_parser.py b/junifer/cli/tests/test_parser.py deleted file mode 100644 index e4d77258e..000000000 --- a/junifer/cli/tests/test_parser.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Provide tests for CLI parser.""" - -# Authors: Federico Raimondo -# Synchon Mandal -# License: AGPL - -import sys -from pathlib import Path - -import pytest - -from junifer.cli.parser import parse_yaml - - -def test_parse_yaml_failure() -> None: - """Test YAML parsing failure.""" - with pytest.raises(ValueError, match="does not exist"): - parse_yaml("foo.yaml") - - -def test_parse_yaml_empty_elements_failure(tmp_path: Path) -> None: - """Test YAML parsing with empty elements failure. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - # Write test file - fname = tmp_path / "test_parse_yaml_empty_elements_failure.yaml" - fname.write_text("elements:") - # Check test file - with pytest.raises(ValueError, match="elements key was defined"): - parse_yaml(fname) - - -def test_parse_yaml_success(tmp_path: Path) -> None: - """Test YAML parsing success. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - # Write test file - fname = tmp_path / "test_parse_yaml_success.yaml" - fname.write_text("foo: bar") - # Check test file - contents = parse_yaml(fname) - assert "foo" in contents - assert contents["foo"] == "bar" - - -def test_parse_yaml_success_with_module_autoload(tmp_path: Path) -> None: - """Test YAML parsing with single module autoload success. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - # Write test file - fname = tmp_path / "test_parse_yaml_with_single_module_autoload.yaml" - fname.write_text("foo: bar\nwith: numpy") - # Check test file - contents = parse_yaml(fname) - assert "foo" in contents - assert contents["foo"] == "bar" - assert "with" in contents - assert contents["with"] == ["numpy"] - assert "numpy" in sys.modules - assert "junifer.configs.wrong_config" not in sys.modules - - -def test_parse_yaml_failure_with_multi_module_autoload(tmp_path: Path) -> None: - """Test YAML parsing with multi module autoload failure. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - # Write test file - fname = tmp_path / "test_parse_yaml_with_multi_module_autoload.yaml" - fname.write_text( - "foo: bar\nwith:\n - numpy\n - junifer.testing.wrong_config" - ) - # Check test file - with pytest.raises(ImportError, match="wrong_config"): - parse_yaml(fname) - - -def test_parse_yaml_with_wrong_path(tmp_path: Path) -> None: - """Test YAML parsing with wrong paths in with. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - t_tmp_path = tmp_path / "test_relative_with" - # Write yaml that includes a relative path - yaml_path = t_tmp_path / "yamls" - yaml_path.mkdir(exist_ok=True, parents=True) - yaml_fname = yaml_path / "test_parse_yaml_wrong_path.yaml" - - yaml_fname.write_text("foo: bar\nwith:\n - missingt.py\n - scipy\n") - - # Check test file - with pytest.raises(ValueError, match="does not exist"): - parse_yaml(yaml_fname) - - -def test_parse_yaml_relative_path(tmp_path: Path) -> None: - """Test YAML parsing with relative paths in with. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - t_tmp_path = tmp_path / "test_relative_with" - - # Write .py to include - py_path = t_tmp_path / "external" - py_path.mkdir(exist_ok=True, parents=True) - py_fname = py_path / "first.py" - py_fname.write_text("import numpy as np\n") - - # Write yaml that includes a relative path - yaml_path = t_tmp_path / "yamls" - yaml_path.mkdir(exist_ok=True, parents=True) - yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml" - - yaml_fname.write_text( - "foo: bar\nwith:\n - ../external/first.py\n - scipy\n" - ) - - # Check test file - parse_yaml(yaml_fname) - - -def test_parse_yaml_absolute_path(tmp_path: Path) -> None: - """Test YAML parsing with absolute paths in with. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - t_tmp_path = tmp_path / "test_relative_with" - - # Write .py to include - py_path = t_tmp_path / "external" - py_path.mkdir(exist_ok=True, parents=True) - py_fname = py_path / "first.py" - py_fname.write_text("import numpy as np\n") - - # Write yaml that includes a relative path - yaml_path = t_tmp_path / "yamls" - yaml_path.mkdir(exist_ok=True, parents=True) - yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml" - - yaml_fname.write_text( - f"foo: bar\nwith:\n - {py_fname.absolute()}\n - scipy\n" - ) - - # Check test file - parse_yaml(yaml_fname) - - -def test_parse_yaml_multi_module_deps(tmp_path: Path) -> None: - """Test YAML parsing with multi-module import with deps. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - t_tmp_path = tmp_path / "test_with_multi_module" - - # Write .py to include - py_path = t_tmp_path / "external" - py_path.mkdir(exist_ok=True, parents=True) - py_fname_1 = py_path / "first.py" - py_fname_1.write_text( - "import numpy as np\nfrom second import hej\n" - "def junifer_module_deps(): return ['second.py']\n" - ) - py_fname_2 = py_path / "second.py" - py_fname_2.write_text("def hej(): print('hej')\n") - - # Write yaml - yaml_path = t_tmp_path / "yamls" - yaml_path.mkdir(exist_ok=True, parents=True) - yaml_fname = yaml_path / "test_parse_yaml_multi_module.yaml" - - yaml_fname.write_text( - "foo: bar\nwith:\n - ../external/first.py\n - scipy\n" - ) - - # Check test file - parse_yaml(yaml_fname) - - -def test_parse_storage_uri_relative(tmp_path: Path) -> None: - """Test YAML parsing with storage and relative URI. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" - fname.write_text("foo: bar\nwith: numpy\nstorage:\n uri: test.db\n") - - contents = parse_yaml(fname) - assert "foo" in contents - assert contents["foo"] == "bar" - assert "storage" in contents - assert "uri" in contents["storage"] - assert contents["storage"]["uri"] == str(tmp_path / "test.db") - - fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" - fname.write_text( - "foo: bar\nwith: numpy\nstorage:\n uri: ../another/test.db\n" - ) - - contents = parse_yaml(fname) - assert "foo" in contents - assert contents["foo"] == "bar" - assert "storage" in contents - assert "uri" in contents["storage"] - assert contents["storage"]["uri"] == str( - (tmp_path / "../another/test.db").resolve() - ) - - fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" - fname.write_text( - "foo: bar\nwith: numpy\nstorage:\n uri: /absolute/test.db\n" - ) - - contents = parse_yaml(fname) - assert "foo" in contents - assert contents["foo"] == "bar" - assert "storage" in contents - assert "uri" in contents["storage"] - assert contents["storage"]["uri"] == "/absolute/test.db" - - # Just to trick coverage - fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml" - fname.write_text("foo: bar\nwith: numpy\nstorage:\n kind: SomeStorage\n") - - contents = parse_yaml(fname) - assert "foo" in contents - assert contents["foo"] == "bar" - assert "storage" in contents - - -def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None: - """Test YAML parsing with relative venv queue. - - Parameters - ---------- - tmp_path : pathlib.Path - The path to the test directory. - - """ - fname = tmp_path / "test_parse_yaml_queue_venv_relative.yaml" - fname.write_text("queue:\n env:\n kind: venv\n name: .venv\n") - _ = parse_yaml(fname) -- 2.52.0 From d3dac3ccc85b18ded0091cb95281d35422af128e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 11 Jun 2026 15:58:15 +0200 Subject: [PATCH 2/2] chore: add changelog 499.enh --- docs/changes/newsfragments/499.enh | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/499.enh diff --git a/docs/changes/newsfragments/499.enh b/docs/changes/newsfragments/499.enh new file mode 100644 index 000000000..d7d9111be --- /dev/null +++ b/docs/changes/newsfragments/499.enh @@ -0,0 +1 @@ +Move ``parse_yaml`` from ``junifer.cli`` to ``junifer.api`` by `Synchon Mandal`_ -- 2.52.0