From 9fcb90a3a84ca8c01427e189b5107f4dc827a9d5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 15 Nov 2022 18:55:52 +0100 Subject: [PATCH 01/18] update: add test for invalid job queue kind in queue() --- junifer/api/tests/test_functions.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index c69c939b0..39f295cbd 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -10,7 +10,7 @@ from pathlib import Path import pytest import junifer.testing.registry # noqa: F401 -from junifer.api.functions import collect, run +from junifer.api.functions import collect, queue, run from junifer.datagrabber.base import BaseDataGrabber from junifer.pipeline.registry import build @@ -144,6 +144,29 @@ def test_run_and_collect(tmp_path: Path) -> None: assert uri.exists() +def test_queue_invalid_job_queue( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test queue function for invalid job queue. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + + """ + with pytest.raises(ValueError, match="Unknown queue kind"): + with monkeypatch.context() as m: + m.chdir(tmp_path) + queue( + config={"elements": ["sub-001"]}, + kind="ABC", + ) + + @pytest.mark.skip(reason="HTCondor not installed on system.") def test_queue_condor() -> None: """Test job queueing in HTCondor.""" -- 2.52.0 From 94aa3a25e9fa7e03d2f8b5a4a7e388faefde6f61 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 07:33:41 +0100 Subject: [PATCH 02/18] update: add test for preventing queue assets overwrite check for queue() --- junifer/api/tests/test_functions.py | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 39f295cbd..9f9abbd31 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -167,6 +167,37 @@ def test_queue_invalid_job_queue( ) +def test_queue_assets_disallow_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test overwrite prevention of queue files. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + + """ + with pytest.raises(ValueError, match="Either delete the directory"): + with monkeypatch.context() as m: + m.chdir(tmp_path) + # First generate assets + queue( + config={"elements": ["sub-001"]}, + kind="HTCondor", + jobname="prevent_overwrite", + ) + # Re-run to trigger error + queue( + config={"elements": ["sub-001"]}, + kind="HTCondor", + jobname="prevent_overwrite", + ) + + @pytest.mark.skip(reason="HTCondor not installed on system.") def test_queue_condor() -> None: """Test job queueing in HTCondor.""" -- 2.52.0 From e8a2b45fa176a2019cb693c70e14686412d06d03 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 07:34:29 +0100 Subject: [PATCH 03/18] update: add test for allowing queue assets overwrite check for queue() --- junifer/api/tests/test_functions.py | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 9f9abbd31..de94a45f8 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -5,6 +5,7 @@ # Synchon Mandal # License: AGPL +import logging from pathlib import Path import pytest @@ -14,6 +15,7 @@ from junifer.api.functions import collect, queue, run from junifer.datagrabber.base import BaseDataGrabber from junifer.pipeline.registry import build + # Define datagrabber datagrabber = { "kind": "OasisVBMTestingDatagrabber", @@ -198,6 +200,42 @@ def test_queue_assets_disallow_overwrite( ) +def test_queue_assets_allow_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test overwriting of queue files. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + # First generate assets + queue( + config={"elements": ["sub-001"]}, + kind="HTCondor", + jobname="allow_overwrite", + ) + with caplog.at_level(logging.INFO): + # Re-run to overwrite + queue( + config={"elements": ["sub-001"]}, + kind="HTCondor", + jobname="allow_overwrite", + overwrite=True, + ) + assert "Deleting existing job directory" in caplog.text + + @pytest.mark.skip(reason="HTCondor not installed on system.") def test_queue_condor() -> None: """Test job queueing in HTCondor.""" -- 2.52.0 From 4c7fb8f8524d6295228bf8cbc5ffd0b68667c719 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 08:08:05 +0100 Subject: [PATCH 04/18] update: add test for adding with imports for queue() --- junifer/api/tests/test_functions.py | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index de94a45f8..e4952a4e1 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -7,6 +7,7 @@ import logging from pathlib import Path +from typing import List, Union import pytest @@ -236,6 +237,48 @@ def test_queue_assets_allow_overwrite( assert "Deleting existing job directory" in caplog.text +@pytest.mark.parametrize( + "with_", + [ + "a.py", + ["a.py"], + ["a.py", "b"], + ], +) +def test_queue_with_imports( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + with_: Union[str, List[str]], +) -> None: + """Test queue with `with` imports. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + with_ : str or list of str + The parametrized imports. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + # Create test file, keeping it simple without conditionals + (tmp_path / "a.py").touch() + with caplog.at_level(logging.DEBUG): + queue( + config={"with": with_}, + kind="HTCondor", + elements="sub-001", + ) + assert "Copying" in caplog.text + assert "Queue done" in caplog.text + + @pytest.mark.skip(reason="HTCondor not installed on system.") def test_queue_condor() -> None: """Test job queueing in HTCondor.""" -- 2.52.0 From 718ff343c292b9aba261902392e9cba1bd20d3c5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 08:23:07 +0100 Subject: [PATCH 05/18] update: add test for specifying elements for queue() --- junifer/api/tests/test_functions.py | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index e4952a4e1..e43a74d2e 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -279,6 +279,45 @@ def test_queue_with_imports( assert "Queue done" in caplog.text +@pytest.mark.parametrize( + "elements", + [ + "sub-001", + ["sub-001"], + ["sub-001", "sub-002"], + ], +) +def test_queue_with_elements( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + elements: Union[str, List[str]], +) -> None: + """Test queue with elements. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + elements : str of list of str + The parametrized elements for the queue. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + with caplog.at_level(logging.INFO): + queue( + config={}, + kind="HTCondor", + elements=elements, + ) + assert "Queue done" in caplog.text + + @pytest.mark.skip(reason="HTCondor not installed on system.") def test_queue_condor() -> None: """Test job queueing in HTCondor.""" -- 2.52.0 From 620f9eaab4f25f63080410564e0c29d6e36009d9 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 08:23:38 +0100 Subject: [PATCH 06/18] update: add test for specifying no elements for queue() --- junifer/api/tests/test_functions.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index e43a74d2e..b467d8f8e 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -318,6 +318,33 @@ def test_queue_with_elements( assert "Queue done" in caplog.text +def test_queue_without_elements( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test queue without elements. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + with caplog.at_level(logging.INFO): + queue( + config={"datagrabber": datagrabber}, + kind="SLURM", + ) + assert "Queue done" in caplog.text + + @pytest.mark.skip(reason="HTCondor not installed on system.") def test_queue_condor() -> None: """Test job queueing in HTCondor.""" -- 2.52.0 From 6e20dbd73d0e4f933b6ba1eefe3dde88703fe199 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 08:24:31 +0100 Subject: [PATCH 07/18] chore: replace List with list for isinstance() checks in api/functions.py --- junifer/api/functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 90ea8f839..958482af4 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -84,7 +84,7 @@ def run( # Convert str to Path if isinstance(workdir, str): workdir = Path(workdir) - if not isinstance(elements, List) and elements is not None: + if not isinstance(elements, list) and elements is not None: elements = [elements] # Get datagrabber to use datagrabber_object = _get_datagrabber(datagrabber) @@ -234,7 +234,7 @@ def queue( elements = dg.get_elements() # TODO: Fix typing of elements - if not isinstance(elements, List): + if not isinstance(elements, list): elements = [elements] # type: ignore typing.cast(List[Union[str, Tuple]], elements) -- 2.52.0 From 04b91434b27d1f98d30ef42b86e533c943c74620 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 08:24:59 +0100 Subject: [PATCH 08/18] chore: simplify conditional check for overwriting for queue() --- junifer/api/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 958482af4..c8929195c 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -186,7 +186,7 @@ def queue( jobdir = cwd / "junifer_jobs" / jobname logger.info(f"Creating job in {str(jobdir.absolute())}") if jobdir.exists(): - if overwrite is not True: + if not overwrite: raise_error( f"Job folder for {jobname} already exists. " "This error is raised to prevent overwriting job files " -- 2.52.0 From 07df862a46716d1a8c9171948687026b6bcaf84d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 09:54:53 +0100 Subject: [PATCH 09/18] update: add test for checking invalid Python env for HTCondor queue() --- junifer/api/tests/test_functions.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index b467d8f8e..39eacd868 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -345,10 +345,30 @@ def test_queue_without_elements( assert "Queue done" in caplog.text -@pytest.mark.skip(reason="HTCondor not installed on system.") -def test_queue_condor() -> None: - """Test job queueing in HTCondor.""" - pass +def test_queue_condor_invalid_python_env( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test invalid Python environment check for HTCondor. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + + """ + with pytest.raises(ValueError, match="Unknown env kind"): + with monkeypatch.context() as m: + m.chdir(tmp_path) + queue( + config={"elements": "sub-001"}, + kind="HTCondor", + env={"kind": "galaxy"}, + ) + + @pytest.mark.skip(reason="SLURM not installed on system.") -- 2.52.0 From 0314e43c3849265e9bccdca8ac8bd9598abc150b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 09:55:46 +0100 Subject: [PATCH 10/18] update: add test for checking conda Python env for HTCondor queue() --- junifer/api/tests/test_functions.py | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 39eacd868..cb5923b7b 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -369,6 +369,46 @@ def test_queue_condor_invalid_python_env( ) +def test_queue_condor_conda_python( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test conda Python environment check for HTCondor. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + with caplog.at_level(logging.INFO): + queue( + config={"elements": "sub-001"}, + kind="HTCondor", + jobname="conda_env_check", + env={"kind": "conda", "name": "conda-env"}, + ) + assert "Copying" in caplog.text + assert ( + Path( + tmp_path + / "junifer_jobs" + / "conda_env_check" + / "run_conda.sh" + ) + .stat() + .st_mode + == 33261 + ) + + @pytest.mark.skip(reason="SLURM not installed on system.") -- 2.52.0 From ff4457fedfe41089f543a5687a735ce4325d9df6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 09:56:23 +0100 Subject: [PATCH 11/18] update: add test for checking venv Python env for HTCondor queue() --- junifer/api/tests/test_functions.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index cb5923b7b..605809bfd 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -409,6 +409,32 @@ def test_queue_condor_conda_python( ) +def test_queue_condor_venv_python( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test venv Python environment check for HTCondor. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + with caplog.at_level(logging.INFO): + queue( + config={"elements": "sub-001"}, + kind="HTCondor", + env={"kind": "venv", "name": "venv-env"}, + ) + # TODO: needs implementation for testing @pytest.mark.skip(reason="SLURM not installed on system.") -- 2.52.0 From 33174b4dd0dbb278c369490de01f48a5e54e224d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 09:56:47 +0100 Subject: [PATCH 12/18] chore: replace raise() with raise_error() in queue() --- junifer/api/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index c8929195c..21d2aa78a 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -258,7 +258,7 @@ def queue( **kwargs, ) else: - raise ValueError(f"Unknown queue kind: {kind}") + raise_error(f"Unknown queue kind: {kind}") logger.info("Queue done") -- 2.52.0 From 4d324bf93272fb8c335c68601885e3da978f3d30 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 19:36:50 +0100 Subject: [PATCH 13/18] update: add with import file copy check for queue() --- junifer/api/tests/test_functions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 605809bfd..78325460a 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -273,11 +273,17 @@ def test_queue_with_imports( queue( config={"with": with_}, kind="HTCondor", + jobname="with_import_check", elements="sub-001", ) assert "Copying" in caplog.text assert "Queue done" in caplog.text + # Check that file is copied + assert Path( + tmp_path / "junifer_jobs" / "with_import_check" / "a.py" + ).is_file() + @pytest.mark.parametrize( "elements", -- 2.52.0 From c7d3d21725a04b00c8c795a996c60ed2ac67dece Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 16 Nov 2022 19:43:58 +0100 Subject: [PATCH 14/18] update: add tuple and list of tuple for elements check of queue() --- junifer/api/tests/test_functions.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 78325460a..0261d1b15 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -7,7 +7,7 @@ import logging from pathlib import Path -from typing import List, Union +from typing import List, Tuple, Union import pytest @@ -291,13 +291,16 @@ def test_queue_with_imports( "sub-001", ["sub-001"], ["sub-001", "sub-002"], + ("sub-001", "ses-001"), + [("sub-001", "ses-001")], + [("sub-001", "ses-001"), ("sub-001", "ses-002")], ], ) def test_queue_with_elements( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, - elements: Union[str, List[str]], + elements: Union[str, List[Union[str, Tuple[str]]], Tuple[str]], ) -> None: """Test queue with elements. -- 2.52.0 From 62ac562a309c9763aa7f3dceda06fcb7ccde55e8 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 17 Nov 2022 09:08:12 +0100 Subject: [PATCH 15/18] update: add test for checking proper YAML config generation for queue() --- junifer/api/tests/test_functions.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 0261d1b15..abef9aed3 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import List, Tuple, Union import pytest +import yaml import junifer.testing.registry # noqa: F401 from junifer.api.functions import collect, queue, run @@ -147,6 +148,60 @@ def test_run_and_collect(tmp_path: Path) -> None: assert uri.exists() +def test_queue_correct_yaml_config( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test proper YAML config generation for queueing. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + + """ + with monkeypatch.context() as m: + m.chdir(tmp_path) + queue( + config={ + "with": "junifer.testing.registry", + "workdir": str(Path(tmp_path).resolve()), + "datagrabber": datagrabber, + "markers": markers, + "storage": storage, + "env": { + "kind": "conda", + "name": "junifer", + }, + "mem": "8G", + }, + kind="HTCondor", + jobname="yaml_config_gen_check", + ) + + generated_config_yaml_path = Path( + tmp_path / "junifer_jobs" / "yaml_config_gen_check" / "config.yaml" + ) + with open(generated_config_yaml_path, "r") as f: + yaml_config = yaml.unsafe_load(f) + # Check for correct YAML config generation + assert all( + key in yaml_config.keys() + for key in [ + "with", + "workdir", + "datagrabber", + "markers", + "storage", + "env", + "mem", + ] + ) + assert "queue" not in yaml_config.keys() + + def test_queue_invalid_job_queue( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, -- 2.52.0 From 4f8d0fcb1d1b00f5552650cbeea213f5c2f2666c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 17 Nov 2022 16:10:06 +0100 Subject: [PATCH 16/18] update: add test for HTCondor job submission failure in queue() --- junifer/api/tests/test_functions.py | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index abef9aed3..4df08c68b 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -501,6 +501,40 @@ def test_queue_condor_venv_python( # TODO: needs implementation for testing +def test_queue_condor_submission_fail( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test HTCondor job submission failure. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + + """ + with pytest.raises( + FileNotFoundError, + match="No such file or directory: 'condor_submit_dag'", + ): + with monkeypatch.context() as m: + m.chdir(tmp_path) + with caplog.at_level(logging.INFO): + queue( + config={"elements": ["sub-001"]}, + kind="HTCondor", + jobname="condor_job_submission_fail", + submit=True, + ) + # Check submit log + assert "Submitting HTCondor job" in caplog.text + + @pytest.mark.skip(reason="SLURM not installed on system.") def test_queue_slurm() -> None: """Test job queueing in SLURM.""" -- 2.52.0 From db8a6f2749f62c72e57ceecfd297d508c0abb520 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 17 Nov 2022 16:11:38 +0100 Subject: [PATCH 17/18] update: add test for checking job submission resource allocation for HTCondor in queue() --- junifer/api/tests/test_functions.py | 127 ++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 4df08c68b..3c843e9f6 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -501,6 +501,133 @@ def test_queue_condor_venv_python( # TODO: needs implementation for testing +@pytest.mark.parametrize( + "elements, env, mem, cpus, disk, collect", + [ + ( + ["sub-001"], + {"kind": "conda", "name": "conda-env"}, + "4G", + 4, + "4G", + True, + ), + ( + ["sub-001"], + {"kind": "venv", "name": "venv-env"}, + "8G", + 8, + "8G", + False, + ), + (["sub-001"], {"kind": "local"}, "12G", 12, "12G", True), + ], +) +def test_queue_condor_assets_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + elements: str, + env: str, + mem: str, + cpus: int, + disk: str, + collect: bool, +) -> None: + """Test HTCondor generated assets. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + caplog : pytest.LogCaptureFixture + The logcapturefixture object. + elements : str + The parametrized element names. + env : dict + The parametrized env names. + mem : str + The parametrized memory size. + cpus : int + The parametrized CPUs. + disk : str + The parametrized disk size. + collect : bool + The parametrized collect option. + + """ + jobname = "condor_assets_gen_check" + with monkeypatch.context() as m: + m.chdir(tmp_path) + with caplog.at_level(logging.INFO): + queue( + config={"elements": elements}, + kind="HTCondor", + jobname=jobname, + env=env, + mem=mem, + cpus=cpus, + disk=disk, + collect=collect, + ) + + # Check log directory creation + assert Path(tmp_path / "junifer_jobs" / jobname / "logs").is_dir() + + run_submit_file_path = Path( + tmp_path / "junifer_jobs" / jobname / f"run_{jobname}.submit" + ) + # Check junifer run submit file + assert run_submit_file_path.is_file() + # Read run submit file to check if resources are correct + with open(run_submit_file_path, "r") as f: + for line in f.read().splitlines(): + if "request_cpus" in line: + assert int(line.split("=")[1].strip()) == cpus + if "request_memory" in line: + assert line.split("=")[1].strip() == mem + if "request_disk" in line: + assert line.split("=")[1].strip() == disk + + # Check junifer collect submit file + assert Path( + tmp_path + / "junifer_jobs" + / jobname + / f"collect_{jobname}.submit" + ).is_file() + + dag_file_path = Path( + tmp_path / "junifer_jobs" / jobname / f"{jobname}.dag" + ) + # Check junifer dag file + assert dag_file_path.is_file() + # Read dag file to check if collect job is found + element_count = 0 + has_collect_job = False + with open(dag_file_path, "r") as f: + for line in f.read().splitlines(): + if "JOB" in line: + element_count += 1 + if "collect" in line: + has_collect_job = True + + if collect: + assert len(elements) == element_count - 1 + assert has_collect_job is True + else: + assert len(elements) == element_count + assert has_collect_job is False + + # Check submit log + assert ( + "HTCondor job files created, to submit the job, run" + in caplog.text + ) + + def test_queue_condor_submission_fail( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, -- 2.52.0 From 5bdc89f49fcd2845ecba72ca86d758eef1e6d8ae Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 17 Nov 2022 16:20:37 +0100 Subject: [PATCH 18/18] update: add test for checking extra preamble in HTCondor generated submit files --- junifer/api/tests/test_functions.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 3c843e9f6..c1ffd3d5d 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -628,6 +628,50 @@ def test_queue_condor_assets_generation( ) +def test_queue_condor_extra_preamble( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test HTCondor extra preamble addition. + + Parameters + ---------- + tmp_path : pathlib.Path + The path to the test directory. + monkeypatch : pytest.MonkeyPatch + The monkeypatch object. + + """ + jobname = "condor_extra_preamble_check" + extra_preamble = "FOO = BAR" + with monkeypatch.context() as m: + m.chdir(tmp_path) + queue( + config={"elements": ["sub-001"]}, + kind="HTCondor", + jobname=jobname, + extra_preamble=extra_preamble, + ) + + # Check extra preamble in run submit file + run_submit_file_path = Path( + tmp_path / "junifer_jobs" / jobname / f"run_{jobname}.submit" + ) + with open(run_submit_file_path, "r") as f: + for line in f.read().splitlines(): + if "FOO" in line: + assert line.strip() == extra_preamble + + # Check extra preamble in collect submit file + collect_submit_file_path = Path( + tmp_path / "junifer_jobs" / jobname / f"collect_{jobname}.submit" + ) + with open(collect_submit_file_path, "r") as f: + for line in f.read().splitlines(): + if "FOO" in line: + assert line.strip() == extra_preamble + + def test_queue_condor_submission_fail( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, -- 2.52.0