From 41e8a66d5ebb531e0155b20ef7a81e0760e22c06 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 26 Feb 2024 17:32:35 +0100 Subject: [PATCH 1/6] feat: add reset command and add it to cli --- junifer/api/cli.py | 38 ++++++++++++++++++++++++++++++++++++++ junifer/api/functions.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/junifer/api/cli.py b/junifer/api/cli.py index 71759aa76..e6a10a2c5 100644 --- a/junifer/api/cli.py +++ b/junifer/api/cli.py @@ -21,6 +21,7 @@ from ..utils.logging import ( ) from .functions import collect as api_collect from .functions import queue as api_queue +from .functions import reset as api_reset from .functions import run as api_run from .parser import parse_yaml from .utils import ( @@ -414,6 +415,43 @@ def selftest(subpkg: str) -> None: click.secho("Failure.", fg="red") +@cli.command() +@click.argument( + "filepath", + type=click.Path( + exists=True, readable=True, dir_okay=False, path_type=pathlib.Path + ), +) +@click.option( + "-v", + "--verbose", + type=click.UNPROCESSED, + callback=_validate_verbose, + default="info", +) +def reset( + filepath: click.Path, + verbose: Union[str, int], +) -> None: + """Reset command for CLI. + + \f + + Parameters + ---------- + filepath : click.Path + The filepath to the configuration file. + verbose : click.Choice + The verbosity level: warning, info or debug (default "info"). + + """ + configure_logging(level=verbose) + # Parse YAML + config = parse_yaml(filepath) + # Perform operation + api_reset(config) + + @cli.group() def setup() -> None: # pragma: no cover """Configure commands for Junifer.""" diff --git a/junifer/api/functions.py b/junifer/api/functions.py index efcd197b3..20b0e2451 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -709,3 +709,40 @@ def _queue_slurm( # logger.info( # f"SLURM job files created, to submit the job, run `{cmd}`" # ) + + +def reset(config: Dict) -> None: + """Reset the storage and jobs directory. + + Parameters + ---------- + config : dict + The configuration to be used for resetting. + + """ + # Fetch storage + storage = config["storage"] + storage_uri = Path(storage["uri"]) + logger.info(f"Deleting {storage_uri.resolve()!s}") + # Delete storage; will be str + if storage_uri.exists(): + # Delete files in the directory + for file in storage_uri.iterdir(): + file.unlink(missing_ok=True) + # Remove directory + storage_uri.parent.rmdir() + + # Fetch job name (if present) + if config.get("queue") is not None: + queue = config["queue"] + job_dir = ( + Path.cwd() + / "junifer_jobs" + / (queue.get("jobname") or "junifer_job") + ) + logger.info(f"Deleting job directory at {job_dir.resolve()!s}") + if job_dir.exists(): + # Remove files and directories + shutil.rmtree(job_dir) + # Remove directory + job_dir.parent.rmdir() -- 2.52.0 From 08042a6cdec7c9631a316dc8756a182b76c70d0c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 26 Feb 2024 17:33:59 +0100 Subject: [PATCH 2/6] update: add tests for queue in test_cli.py --- junifer/api/tests/test_cli.py | 45 ++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/junifer/api/tests/test_cli.py b/junifer/api/tests/test_cli.py index 7d94ca35f..da1b6077d 100644 --- a/junifer/api/tests/test_cli.py +++ b/junifer/api/tests/test_cli.py @@ -11,7 +11,14 @@ import pytest from click.testing import CliRunner from ruamel.yaml import YAML -from junifer.api.cli import _parse_elements_file, collect, run, selftest, wtf +from junifer.api.cli import ( + _parse_elements_file, + collect, + queue, + run, + selftest, + wtf, +) # Configure YAML class @@ -181,6 +188,42 @@ def test_multi_element_access( assert read_elements == expected_list +def test_queue(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Test queue command. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The pytest.MonkeyPatch object. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + # Get test config + infile = Path(__file__).parent / "data" / "gmd_mean_htcondor.yaml" + # Read test config + contents = yaml.load(infile) + # Working directory + contents["workdir"] = str(tmp_path.resolve()) + # Storage + contents["storage"]["uri"] = str((tmp_path / "out.sqlite").resolve()) + # Write new test config + outfile = tmp_path / "in.yaml" + yaml.dump(contents, stream=outfile) + # Queue command arguments + queue_args = [ + str(outfile.resolve()), + "--verbose", + "debug", + ] + # Invoke queue command + queue_result = runner.invoke(queue, queue_args) + # Check + assert queue_result.exit_code == 0 + + def test_wtf_short() -> None: """Test short version of wtf command.""" # Invoke wtf command -- 2.52.0 From 1d2b3e82146d38ac5ac49345c11b7c1bfd00a950 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 26 Feb 2024 17:34:29 +0100 Subject: [PATCH 3/6] update: add tests for reset command --- junifer/api/tests/test_cli.py | 66 +++++++++++++++++++++- junifer/api/tests/test_functions.py | 85 ++++++++++++++++++++++++++++- 2 files changed, 149 insertions(+), 2 deletions(-) diff --git a/junifer/api/tests/test_cli.py b/junifer/api/tests/test_cli.py index da1b6077d..c4167db4b 100644 --- a/junifer/api/tests/test_cli.py +++ b/junifer/api/tests/test_cli.py @@ -5,7 +5,7 @@ # License: AGPL from pathlib import Path -from typing import List, Tuple +from typing import Callable, List, Tuple import pytest from click.testing import CliRunner @@ -15,6 +15,7 @@ from junifer.api.cli import ( _parse_elements_file, collect, queue, + reset, run, selftest, wtf, @@ -224,6 +225,69 @@ def test_queue(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: assert queue_result.exit_code == 0 +@pytest.mark.parametrize( + "action, action_file", + [ + (run, "gmd_mean.yaml"), + (queue, "gmd_mean_htcondor.yaml"), + ], +) +def test_reset( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + action: Callable, + action_file: str, +) -> None: + """Test reset command. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The pytest.MonkeyPatch object. + action : callable + The parametrized action to perform. + action_file : str + The parametrized file for the action. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + # Get test config + infile = Path(__file__).parent / "data" / action_file + # Read test config + contents = yaml.load(infile) + # Working directory + contents["workdir"] = str(tmp_path.resolve()) + # Storage + contents["storage"]["uri"] = str((tmp_path / "out.sqlite").resolve()) + # Write new test config + outfile = tmp_path / "in.yaml" + yaml.dump(contents, stream=outfile) + # Command arguments + action_args = [ + str(outfile.resolve()), + "--verbose", + "debug", + ] + # Invoke command + result = runner.invoke(action, action_args) + # Check + assert result.exit_code == 0 + + # Reset arguments + reset_args = [ + str(outfile.resolve()), + "--verbose", + "debug", + ] + # Run reset + reset_result = runner.invoke(reset, reset_args) + # Check + assert reset_result.exit_code == 0 + + def test_wtf_short() -> None: """Test short version of wtf command.""" # Invoke wtf command diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 61aedecc6..19344e846 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -14,7 +14,7 @@ import pytest from ruamel.yaml import YAML import junifer.testing.registry # noqa: F401 -from junifer.api.functions import collect, queue, run +from junifer.api.functions import collect, queue, reset, run from junifer.datagrabber.base import BaseDataGrabber from junifer.pipeline.registry import build @@ -867,3 +867,86 @@ def test_queue_condor_submission_fail( def test_queue_slurm() -> None: """Test job queueing in SLURM.""" pass + + +def test_reset_run(tmp_path: Path) -> None: + """Test reset function for run. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + + """ + # Create storage + storage["uri"] = tmp_path / "test_reset_run.sqlite" # type: ignore + # Run operation to generate files + run( + workdir=tmp_path, + datagrabber=datagrabber, + markers=markers, + storage=storage, + elements=["sub-01"], + ) + # Reset operation + reset(config={"storage": storage}) + + assert not Path(storage["uri"]).exists() + + +@pytest.mark.parametrize( + "job_name", + ( + "job", + None, + ), +) +def test_reset_queue( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, job_name: str +) -> None: + """Test reset function for queue. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The pytest.MonkeyPatch object. + job_name : str + The parametrized job name. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + # Create storage + storage["uri"] = "test_reset_queue.sqlite" + # Set job name + if job_name is None: + job_name = "junifer_job" + # Queue operation to generate files + queue( + config={ + "with": "junifer.testing.registry", + "workdir": str(tmp_path.resolve()), + "datagrabber": datagrabber, + "markers": markers, + "storage": storage, + "env": { + "kind": "conda", + "name": "junifer", + }, + "mem": "8G", + }, + kind="HTCondor", + jobname=job_name, + ) + # Reset operation + reset( + config={ + "storage": storage, + "queue": {"jobname": job_name}, + } + ) + + assert not Path(storage["uri"]).exists() + assert not (tmp_path / "junifer_jobs" / job_name).exists() -- 2.52.0 From f01ba1499a5f823709c71b1ad8076a3221c086d4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 26 Feb 2024 17:34:42 +0100 Subject: [PATCH 4/6] chore: remove unnecessary code --- junifer/datagrabber/tests/test_dmcc13_benchmark.py | 1 - 1 file changed, 1 deletion(-) diff --git a/junifer/datagrabber/tests/test_dmcc13_benchmark.py b/junifer/datagrabber/tests/test_dmcc13_benchmark.py index 9d7436b95..cb7bb3d46 100644 --- a/junifer/datagrabber/tests/test_dmcc13_benchmark.py +++ b/junifer/datagrabber/tests/test_dmcc13_benchmark.py @@ -81,7 +81,6 @@ def test_DMCC13Benchmark( dg.uri = URI with dg: - # breakpoint() # Get all elements all_elements = dg.get_elements() # Get test element -- 2.52.0 From 6f3034202285ec60263c1db94768e4ee82b33002 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 26 Feb 2024 17:35:57 +0100 Subject: [PATCH 5/6] chore: improve docstrings for registry.py --- junifer/pipeline/registry.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/junifer/pipeline/registry.py b/junifer/pipeline/registry.py index 2d0452fda..0579d2b36 100644 --- a/junifer/pipeline/registry.py +++ b/junifer/pipeline/registry.py @@ -41,6 +41,11 @@ def register(step: str, name: str, klass: type) -> None: klass : class Class to be registered. + Raises + ------ + ValueError + If the ``step`` is invalid. + """ # Verify step if step not in _VALID_STEPS: @@ -63,6 +68,11 @@ def get_step_names(step: str) -> List[str]: list List of registered function names. + Raises + ------ + ValueError + If the ``step`` is invalid. + """ # Verify step if step not in _VALID_STEPS: @@ -86,6 +96,11 @@ def get_class(step: str, name: str) -> type: class Registered function class. + Raises + ------ + ValueError + If the ``step`` or ``name`` is invalid. + """ # Verify step if step not in _VALID_STEPS: @@ -123,6 +138,8 @@ def build( Raises ------ + RuntimeError + If there is a problem creating the instance. ValueError If the created object with the given name is not an instance of the base class. -- 2.52.0 From d58741bcc76bdc0fe36af05bf7740fca0dc20a1e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 26 Feb 2024 17:39:27 +0100 Subject: [PATCH 6/6] chore: add changelog 240.feature --- docs/changes/newsfragments/240.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/240.feature diff --git a/docs/changes/newsfragments/240.feature b/docs/changes/newsfragments/240.feature new file mode 100644 index 000000000..b7fc67ea0 --- /dev/null +++ b/docs/changes/newsfragments/240.feature @@ -0,0 +1 @@ +Add ``junifer reset`` to reset storage and jobs directory by `Synchon Mandal`_ -- 2.52.0