From 5eaa078c07d1f93ca156dd8def3afff6eea07d39 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Mar 2024 14:47:54 +0100 Subject: [PATCH 01/10] update: add support for selecting shell in HTCondorAdapter --- junifer/api/queue_context/htcondor_adapter.py | 21 +++++++++++++---- .../api/res/{run_conda.sh => run_conda.bash} | 0 junifer/api/res/run_conda.zsh | 23 +++++++++++++++++++ .../api/res/{run_venv.sh => run_venv.bash} | 0 junifer/api/res/run_venv.zsh | 22 ++++++++++++++++++ 5 files changed, 61 insertions(+), 5 deletions(-) rename junifer/api/res/{run_conda.sh => run_conda.bash} (100%) create mode 100644 junifer/api/res/run_conda.zsh rename junifer/api/res/{run_venv.sh => run_venv.bash} (100%) create mode 100644 junifer/api/res/run_venv.zsh diff --git a/junifer/api/queue_context/htcondor_adapter.py b/junifer/api/queue_context/htcondor_adapter.py index 9435d0026..0b350fc7c 100644 --- a/junifer/api/queue_context/htcondor_adapter.py +++ b/junifer/api/queue_context/htcondor_adapter.py @@ -126,7 +126,8 @@ class HTCondorAdapter(QueueContextAdapter): Raises ------ ValueError - If ``env.kind`` is invalid. + If ``env.kind`` is invalid or + if ``env.shell`` is invalid. """ # Set env related variables @@ -140,13 +141,22 @@ class HTCondorAdapter(QueueContextAdapter): f"must be one of {valid_env_kinds}" ) else: + # Check shell + shell = env.get("shell", "bash") + valid_shells = ["bash", "zsh"] + if shell not in valid_shells: + raise_error( + f"Invalid value for `env.shell`: {shell}, " + f"must be one of {valid_shells}" + ) + self._shell = shell # Set variables if env["kind"] == "local": # No virtual environment self._executable = "junifer" self._arguments = "" else: - self._executable = f"run_{env['kind']}.sh" + self._executable = f"run_{env['kind']}.{self._shell}" self._arguments = f"{env['name']} junifer" self._exec_path = self._job_dir / self._executable @@ -181,7 +191,7 @@ class HTCondorAdapter(QueueContextAdapter): def pre_run(self) -> str: """Return pre-run commands.""" fixed = ( - "#!/bin/bash\n\n" + f"#!/bin/{self._shell}\n\n" "# This script is auto-generated by junifer.\n\n" "# Force datalad to run in non-interactive mode\n" "DATALAD_UI_INTERACTIVE=false\n" @@ -225,12 +235,13 @@ class HTCondorAdapter(QueueContextAdapter): def pre_collect(self) -> str: """Return pre-collect commands.""" fixed = ( - "#!/bin/bash\n\n" "# This script is auto-generated by junifer.\n" + f"#!/bin/{self._shell}\n\n" + "# This script is auto-generated by junifer.\n" ) var = self._pre_collect or "" # Add commands if collect="yes" if self._collect == "yes": - var += 'if [ "${1}" == "4" ]; then\n' " exit 1\n" "fi\n" + var += 'if [ "${1}" == "4" ]; then\n exit 1\nfi\n' return fixed + "\n" + var def collect(self) -> str: diff --git a/junifer/api/res/run_conda.sh b/junifer/api/res/run_conda.bash similarity index 100% rename from junifer/api/res/run_conda.sh rename to junifer/api/res/run_conda.bash diff --git a/junifer/api/res/run_conda.zsh b/junifer/api/res/run_conda.zsh new file mode 100644 index 000000000..b54d36a8d --- /dev/null +++ b/junifer/api/res/run_conda.zsh @@ -0,0 +1,23 @@ +#!/bin/zsh + +if [ $# -lt 2 ]; then + echo "This script is meant to run a command within a conda environment." + echo "It needs at least 2 parameters." + echo "The first one must be the environment name." + echo "The rest will be the command." + exit 255 +fi + +eval "$(conda shell.zsh hook)" +env_name=$1 +echo "Activating ${env_name}" +conda activate "${env_name}" +shift 1 + +if [ -f "pre_run.sh" ]; then + echo "Sourcing pre_run.sh" + . ./pre_run.sh +fi + +echo "Running ${*} in conda environment" +"$@" diff --git a/junifer/api/res/run_venv.sh b/junifer/api/res/run_venv.bash similarity index 100% rename from junifer/api/res/run_venv.sh rename to junifer/api/res/run_venv.bash diff --git a/junifer/api/res/run_venv.zsh b/junifer/api/res/run_venv.zsh new file mode 100644 index 000000000..d1df3b605 --- /dev/null +++ b/junifer/api/res/run_venv.zsh @@ -0,0 +1,22 @@ +#!/bin/zsh + +if [ $# -lt 2 ]; then + echo "This script is meant to run a command within a Python virtual environment." + echo "It needs at least 2 parameters." + echo "The first one must be the virtualenv path." + echo "The rest will be the command." + exit 255 +fi + +env_path=$1 +echo "Activating ${env_path}" +source "${env_path}"/bin/activate +shift 1 + +if [ -f "pre_run.sh" ]; then + echo "Sourcing pre_run.sh" + . ./pre_run.sh +fi + +echo "Running ${*} in Python virtual environment" +"$@" -- 2.52.0 From 2d1d1ccd431d122285426291726206cca7ced4ea Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Mar 2024 14:48:24 +0100 Subject: [PATCH 02/10] update: improve tests for HTCondorAdapter --- .../tests/test_htcondor_adapter.py | 63 ++++++++++++++----- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/junifer/api/queue_context/tests/test_htcondor_adapter.py b/junifer/api/queue_context/tests/test_htcondor_adapter.py index cf41e6d9e..ee32a85ae 100644 --- a/junifer/api/queue_context/tests/test_htcondor_adapter.py +++ b/junifer/api/queue_context/tests/test_htcondor_adapter.py @@ -12,11 +12,11 @@ import pytest from junifer.api.queue_context import HTCondorAdapter -def test_HTCondorAdapter_env_error() -> None: +def test_HTCondorAdapter_env_kind_error() -> None: """Test error for invalid env kind.""" with pytest.raises(ValueError, match="Invalid value for `env.kind`"): HTCondorAdapter( - job_name="check_env", + job_name="check_env_kind", job_dir=Path("."), yaml_config_path=Path("."), elements=["sub01"], @@ -24,6 +24,18 @@ def test_HTCondorAdapter_env_error() -> None: ) +def test_HTCondorAdapter_env_shell_error() -> None: + """Test error for invalid env shell.""" + with pytest.raises(ValueError, match="Invalid value for `env.shell`"): + HTCondorAdapter( + job_name="check_env_shell", + job_dir=Path("."), + yaml_config_path=Path("."), + elements=["sub01"], + env={"kind": "conda", "shell": "fish"}, + ) + + def test_HTCondorAdapter_collect_error() -> None: """Test error for invalid collect option.""" with pytest.raises(ValueError, match="Invalid value for `collect`"): @@ -37,14 +49,18 @@ def test_HTCondorAdapter_collect_error() -> None: @pytest.mark.parametrize( - "pre_run, expected_text", + "pre_run, expected_text, shell", [ - (None, "# Force datalad"), - ("# Check this out\n", "# Check this out"), + (None, "# Force datalad", "bash"), + (None, "# Force datalad", "zsh"), + ("# Check this out\n", "# Check this out", "bash"), + ("# Check this out\n", "# Check this out", "zsh"), ], ) def test_HTCondorAdapter_pre_run( - pre_run: Optional[str], expected_text: str + pre_run: Optional[str], + expected_text: str, + shell: str, ) -> None: """Test HTCondorAdapter pre_run(). @@ -54,6 +70,8 @@ def test_HTCondorAdapter_pre_run( The parametrized pre run text. expected_text : str The parametrized expected text. + shell : str + The parametrized expected shell. """ adapter = HTCondorAdapter( @@ -61,22 +79,31 @@ def test_HTCondorAdapter_pre_run( job_dir=Path("."), yaml_config_path=Path("."), elements=["sub01"], + env={"kind": "conda", "name": "junifer", "shell": shell}, pre_run=pre_run, ) + assert shell in adapter.pre_run() assert expected_text in adapter.pre_run() @pytest.mark.parametrize( - "pre_collect, expected_text, collect", + "pre_collect, expected_text, collect, shell", [ - (None, "exit 1", "yes"), - (None, "# This script", "on_success_only"), - ("# Check this out\n", "# Check this out", "yes"), - ("# Check this out\n", "# Check this out", "on_success_only"), + (None, "exit 1", "yes", "bash"), + (None, "exit 1", "yes", "zsh"), + (None, "# This script", "on_success_only", "bash"), + (None, "# This script", "on_success_only", "zsh"), + ("# Check this out\n", "# Check this out", "yes", "bash"), + ("# Check this out\n", "# Check this out", "yes", "zsh"), + ("# Check this out\n", "# Check this out", "on_success_only", "bash"), + ("# Check this out\n", "# Check this out", "on_success_only", "zsh"), ], ) def test_HTCondorAdapter_pre_collect( - pre_collect: Optional[str], expected_text: str, collect: str + pre_collect: Optional[str], + expected_text: str, + collect: str, + shell: str, ) -> None: """Test HTCondorAdapter pre_collect(). @@ -88,6 +115,8 @@ def test_HTCondorAdapter_pre_collect( The parametrized expected text. collect : str The parametrized collect parameter. + shell : str + The parametrized expected shell. """ adapter = HTCondorAdapter( @@ -95,9 +124,11 @@ def test_HTCondorAdapter_pre_collect( job_dir=Path("."), yaml_config_path=Path("."), elements=["sub01"], + env={"kind": "venv", "name": "junifer", "shell": shell}, pre_collect=pre_collect, collect=collect, ) + assert shell in adapter.pre_collect() assert expected_text in adapter.pre_collect() @@ -177,8 +208,10 @@ def test_HTCondor_dag( @pytest.mark.parametrize( "env", [ - {"kind": "conda", "name": "junifer"}, - {"kind": "venv", "name": "./junifer"}, + {"kind": "conda", "name": "junifer", "shell": "bash"}, + {"kind": "conda", "name": "junifer", "shell": "zsh"}, + {"kind": "venv", "name": "./junifer", "shell": "bash"}, + {"kind": "venv", "name": "./junifer", "shell": "zsh"}, ], ) def test_HTCondorAdapter_prepare( @@ -215,7 +248,7 @@ def test_HTCondorAdapter_prepare( assert "Creating HTCondor job" in caplog.text assert "Creating logs directory" in caplog.text - assert f"Copying run_{env['kind']}" in caplog.text + assert f"Copying run_{env['kind']}.{env['shell']}" in caplog.text assert "Writing pre_run.sh" in caplog.text assert "Writing run_test_prepare.submit" in caplog.text assert "Writing pre_collect.sh" in caplog.text -- 2.52.0 From 2d5a1cdca040137c71a7edfc622037a8f3e74c20 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Mar 2024 15:06:33 +0100 Subject: [PATCH 03/10] chore: add changelog 273.feature --- docs/changes/newsfragments/273.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/273.feature diff --git a/docs/changes/newsfragments/273.feature b/docs/changes/newsfragments/273.feature new file mode 100644 index 000000000..6975846dd --- /dev/null +++ b/docs/changes/newsfragments/273.feature @@ -0,0 +1 @@ +Add support for choosing between ``bash`` and ``zsh`` when queueing a HTCondor job by `Synchon Mandal`_ -- 2.52.0 From cf2f028073aad4d33426969cead818f4fd71f58e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 21 Mar 2024 09:11:09 +0100 Subject: [PATCH 04/10] update: change instances of /bin/ prefixes to /usr/bin/env prefixes in HTCondorAdapter --- junifer/api/queue_context/htcondor_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junifer/api/queue_context/htcondor_adapter.py b/junifer/api/queue_context/htcondor_adapter.py index 0b350fc7c..4b62e3f72 100644 --- a/junifer/api/queue_context/htcondor_adapter.py +++ b/junifer/api/queue_context/htcondor_adapter.py @@ -191,7 +191,7 @@ class HTCondorAdapter(QueueContextAdapter): def pre_run(self) -> str: """Return pre-run commands.""" fixed = ( - f"#!/bin/{self._shell}\n\n" + f"#!/usr/bin/env {self._shell}\n\n" "# This script is auto-generated by junifer.\n\n" "# Force datalad to run in non-interactive mode\n" "DATALAD_UI_INTERACTIVE=false\n" @@ -235,7 +235,7 @@ class HTCondorAdapter(QueueContextAdapter): def pre_collect(self) -> str: """Return pre-collect commands.""" fixed = ( - f"#!/bin/{self._shell}\n\n" + f"#!/usr/bin/env {self._shell}\n\n" "# This script is auto-generated by junifer.\n" ) var = self._pre_collect or "" -- 2.52.0 From bedb8f7693b4dc70bd258712a3fe833ed9984ceb Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 21 Mar 2024 09:12:05 +0100 Subject: [PATCH 05/10] chore: replace /bin/ prefixes to /usr/bin/env prefixes for api/res/* assets --- junifer/api/res/run_conda.bash | 2 +- junifer/api/res/run_conda.zsh | 2 +- junifer/api/res/run_venv.bash | 2 +- junifer/api/res/run_venv.zsh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/junifer/api/res/run_conda.bash b/junifer/api/res/run_conda.bash index 3efcbbba5..e5cba358a 100755 --- a/junifer/api/res/run_conda.bash +++ b/junifer/api/res/run_conda.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if [ $# -lt 2 ]; then echo "This script is meant to run a command within a conda environment." diff --git a/junifer/api/res/run_conda.zsh b/junifer/api/res/run_conda.zsh index b54d36a8d..629315cec 100644 --- a/junifer/api/res/run_conda.zsh +++ b/junifer/api/res/run_conda.zsh @@ -1,4 +1,4 @@ -#!/bin/zsh +#!/usr/bin/env zsh if [ $# -lt 2 ]; then echo "This script is meant to run a command within a conda environment." diff --git a/junifer/api/res/run_venv.bash b/junifer/api/res/run_venv.bash index b6648031e..d5a5e9050 100755 --- a/junifer/api/res/run_venv.bash +++ b/junifer/api/res/run_venv.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if [ $# -lt 2 ]; then echo "This script is meant to run a command within a Python virtual environment." diff --git a/junifer/api/res/run_venv.zsh b/junifer/api/res/run_venv.zsh index d1df3b605..e9679775a 100644 --- a/junifer/api/res/run_venv.zsh +++ b/junifer/api/res/run_venv.zsh @@ -1,4 +1,4 @@ -#!/bin/zsh +#!/usr/bin/env zsh if [ $# -lt 2 ]; then echo "This script is meant to run a command within a Python virtual environment." -- 2.52.0 From 0e6a1ec9d1a7c4aaee427291ef26df04deff2371 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Apr 2024 13:33:37 +0200 Subject: [PATCH 06/10] chore: lint --- junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py | 4 +--- junifer/configs/juseless/datagrabbers/ixi_vbm.py | 4 +--- junifer/datagrabber/aomic/tests/test_id1000.py | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py index d1f045594..d62064e72 100644 --- a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py +++ b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py @@ -32,9 +32,7 @@ class JuselessDataladAOMICID1000VBM(PatternDataladDataGrabber): replacements = ["subject"] patterns = { "VBM_GM": { - "pattern": ( - "{subject}/mri/mwp1{subject}_run-2_T1w.nii.gz" - ), + "pattern": ("{subject}/mri/mwp1{subject}_run-2_T1w.nii.gz"), "space": "IXI549Space", }, } diff --git a/junifer/configs/juseless/datagrabbers/ixi_vbm.py b/junifer/configs/juseless/datagrabbers/ixi_vbm.py index 5b149d251..1c4bcf7f0 100644 --- a/junifer/configs/juseless/datagrabbers/ixi_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ixi_vbm.py @@ -44,9 +44,7 @@ class JuselessDataladIXIVBM(PatternDataladDataGrabber): replacements = ["site", "subject"] patterns = { "VBM_GM": { - "pattern": ( - "{site}/{subject}/mri/m0wp1{subject}.nii.gz" - ), + "pattern": ("{site}/{subject}/mri/m0wp1{subject}.nii.gz"), "space": "IXI549Space", }, } diff --git a/junifer/datagrabber/aomic/tests/test_id1000.py b/junifer/datagrabber/aomic/tests/test_id1000.py index b797cfcc4..6ffd3ce8d 100644 --- a/junifer/datagrabber/aomic/tests/test_id1000.py +++ b/junifer/datagrabber/aomic/tests/test_id1000.py @@ -33,8 +33,7 @@ def test_DataladAOMICID1000() -> None: assert "BOLD" in out assert ( - out["BOLD"]["path"].name - == f"{test_element}_task-moviewatching_" + out["BOLD"]["path"].name == f"{test_element}_task-moviewatching_" "space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" ) -- 2.52.0 From 864024580fb822005f91c70138ab001ad5459bcd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Apr 2024 13:34:36 +0200 Subject: [PATCH 07/10] update: add support for selecting shell in GnuParallelLocalAdapter --- .../gnu_parallel_local_adapter.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/junifer/api/queue_context/gnu_parallel_local_adapter.py b/junifer/api/queue_context/gnu_parallel_local_adapter.py index 766a21d1c..cdd38030f 100644 --- a/junifer/api/queue_context/gnu_parallel_local_adapter.py +++ b/junifer/api/queue_context/gnu_parallel_local_adapter.py @@ -43,7 +43,8 @@ class GnuParallelLocalAdapter(QueueContextAdapter): Raises ------ ValueError - If``env`` is invalid. + If ``env.kind`` is invalid or + if ``env.shell`` is invalid. See Also -------- @@ -110,13 +111,22 @@ class GnuParallelLocalAdapter(QueueContextAdapter): f"must be one of {valid_env_kinds}" ) else: + # Check shell + shell = env.get("shell", "bash") + valid_shells = ["bash", "zsh"] + if shell not in valid_shells: + raise_error( + f"Invalid value for `env.shell`: {shell}, " + f"must be one of {valid_shells}" + ) + self._shell = shell # Set variables if env["kind"] == "local": # No virtual environment self._executable = "junifer" self._arguments = "" else: - self._executable = f"run_{env['kind']}.sh" + self._executable = f"run_{env['kind']}.{self._shell}" self._arguments = f"{env['name']} junifer" self._exec_path = self._job_dir / self._executable @@ -135,7 +145,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter): def pre_run(self) -> str: """Return pre-run commands.""" fixed = ( - "#!/usr/bin/env bash\n\n" + f"#!/usr/bin/env {self._shell}\n\n" "# This script is auto-generated by junifer.\n\n" "# Force datalad to run in non-interactive mode\n" "DATALAD_UI_INTERACTIVE=false\n" @@ -146,7 +156,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter): def run(self) -> str: """Return run commands.""" return ( - "#!/usr/bin/env bash\n\n" + f"#!/usr/bin/env {self._shell}\n\n" "# This script is auto-generated by junifer.\n\n" "# Run pre_run.sh\n" f"sh {self._pre_run_path.resolve()!s}\n\n" @@ -166,7 +176,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter): def pre_collect(self) -> str: """Return pre-collect commands.""" fixed = ( - "#!/usr/bin/env bash\n\n" + f"#!/usr/bin/env {self._shell}\n\n" "# This script is auto-generated by junifer.\n" ) var = self._pre_collect or "" @@ -175,7 +185,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter): def collect(self) -> str: """Return collect commands.""" return ( - "#!/usr/bin/env bash\n\n" + f"#!/usr/bin/env {self._shell}\n\n" "# This script is auto-generated by junifer.\n\n" "# Run pre_collect.sh\n" f"sh {self._pre_collect_path.resolve()!s}\n\n" -- 2.52.0 From 488a611e20200692b9e913590931889d42b333ff Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Apr 2024 13:34:59 +0200 Subject: [PATCH 08/10] update: improve tests for GnuParallelLocalAdapter --- .../tests/test_gnu_parallel_local_adapter.py | 53 ++++++++++++++----- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py b/junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py index 7bae705fc..19d1f61c6 100644 --- a/junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py +++ b/junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py @@ -12,11 +12,11 @@ import pytest from junifer.api.queue_context import GnuParallelLocalAdapter -def test_GnuParallelLocalAdapter_env_error() -> None: +def test_GnuParallelLocalAdapter_env_kind_error() -> None: """Test error for invalid env kind.""" with pytest.raises(ValueError, match="Invalid value for `env.kind`"): GnuParallelLocalAdapter( - job_name="check_env", + job_name="check_env_kind", job_dir=Path("."), yaml_config_path=Path("."), elements=["sub01"], @@ -24,6 +24,18 @@ def test_GnuParallelLocalAdapter_env_error() -> None: ) +def test_GnuParallelLocalAdapter_env_shell_error() -> None: + """Test error for invalid env shell.""" + with pytest.raises(ValueError, match="Invalid value for `env.shell`"): + GnuParallelLocalAdapter( + job_name="check_env_shell", + job_dir=Path("."), + yaml_config_path=Path("."), + elements=["sub01"], + env={"kind": "conda", "shell": "fish"}, + ) + + @pytest.mark.parametrize( "elements, expected_text", [ @@ -55,14 +67,18 @@ def test_GnuParallelLocalAdapter_elements( @pytest.mark.parametrize( - "pre_run, expected_text", + "pre_run, expected_text, shell", [ - (None, "# Force datalad"), - ("# Check this out\n", "# Check this out"), + (None, "# Force datalad", "bash"), + (None, "# Force datalad", "zsh"), + ("# Check this out\n", "# Check this out", "bash"), + ("# Check this out\n", "# Check this out", "zsh"), ], ) def test_GnuParallelLocalAdapter_pre_run( - pre_run: Optional[str], expected_text: str + pre_run: Optional[str], + expected_text: str, + shell: str, ) -> None: """Test GnuParallelLocalAdapter pre_run(). @@ -72,6 +88,8 @@ def test_GnuParallelLocalAdapter_pre_run( The parametrized pre run text. expected_text : str The parametrized expected text. + shell : str + The parametrized expected shell. """ adapter = GnuParallelLocalAdapter( @@ -79,21 +97,26 @@ def test_GnuParallelLocalAdapter_pre_run( job_dir=Path("."), yaml_config_path=Path("."), elements=["sub01"], + env={"kind": "conda", "name": "junifer", "shell": shell}, pre_run=pre_run, ) + assert shell in adapter.pre_run() assert expected_text in adapter.pre_run() @pytest.mark.parametrize( - "pre_collect, expected_text", + "pre_collect, expected_text, shell", [ - (None, "# This script"), - ("# Check this out\n", "# Check this out"), + (None, "# This script", "bash"), + (None, "# This script", "zsh"), + ("# Check this out\n", "# Check this out", "bash"), + ("# Check this out\n", "# Check this out", "zsh"), ], ) def test_GnuParallelLocalAdapter_pre_collect( pre_collect: Optional[str], expected_text: str, + shell: str, ) -> None: """Test GnuParallelLocalAdapter pre_collect(). @@ -103,6 +126,8 @@ def test_GnuParallelLocalAdapter_pre_collect( The parametrized pre collect text. expected_text : str The parametrized expected text. + shell : str + The parametrized expected shell. """ adapter = GnuParallelLocalAdapter( @@ -110,8 +135,10 @@ def test_GnuParallelLocalAdapter_pre_collect( job_dir=Path("."), yaml_config_path=Path("."), elements=["sub01"], + env={"kind": "venv", "name": "junifer", "shell": shell}, pre_collect=pre_collect, ) + assert shell in adapter.pre_collect() assert expected_text in adapter.pre_collect() @@ -140,8 +167,10 @@ def test_GnuParallelLocalAdapter_collect() -> None: @pytest.mark.parametrize( "env", [ - {"kind": "conda", "name": "junifer"}, - {"kind": "venv", "name": "./junifer"}, + {"kind": "conda", "name": "junifer", "shell": "bash"}, + {"kind": "conda", "name": "junifer", "shell": "zsh"}, + {"kind": "venv", "name": "./junifer", "shell": "bash"}, + {"kind": "venv", "name": "./junifer", "shell": "zsh"}, ], ) def test_GnuParallelLocalAdapter_prepare( @@ -177,7 +206,7 @@ def test_GnuParallelLocalAdapter_prepare( adapter.prepare() assert "GNU parallel" in caplog.text - assert f"Copying run_{env['kind']}" in caplog.text + assert f"Copying run_{env['kind']}.{env['shell']}" in caplog.text assert "Writing pre_run.sh" in caplog.text assert "Writing run_test_prepare.sh" in caplog.text assert "Writing pre_collect.sh" in caplog.text -- 2.52.0 From a20e75c1ba7c81727b5211db1d961113b7869036 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Apr 2024 13:36:19 +0200 Subject: [PATCH 09/10] chore: update changelog 273.feature --- docs/changes/newsfragments/273.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes/newsfragments/273.feature b/docs/changes/newsfragments/273.feature index 6975846dd..c876502d2 100644 --- a/docs/changes/newsfragments/273.feature +++ b/docs/changes/newsfragments/273.feature @@ -1 +1 @@ -Add support for choosing between ``bash`` and ``zsh`` when queueing a HTCondor job by `Synchon Mandal`_ +Add support for choosing between ``bash`` and ``zsh`` shells when queueing by `Synchon Mandal`_ -- 2.52.0 From de69448e1741680d24cba2b2e5b63c21541cef3d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Apr 2024 13:41:12 +0200 Subject: [PATCH 10/10] chore: update whats_new.rst --- docs/changes/newsfragments/320.fix | 1 - docs/changes/newsfragments/321.fix | 1 - docs/whats_new.rst | 4 ++++ 3 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 docs/changes/newsfragments/320.fix delete mode 100644 docs/changes/newsfragments/321.fix diff --git a/docs/changes/newsfragments/320.fix b/docs/changes/newsfragments/320.fix deleted file mode 100644 index 3b4291a39..000000000 --- a/docs/changes/newsfragments/320.fix +++ /dev/null @@ -1 +0,0 @@ -Add a validation step on the :func:`.run` function to validate the marker collection by `Fede Raimondo`_ \ No newline at end of file diff --git a/docs/changes/newsfragments/321.fix b/docs/changes/newsfragments/321.fix deleted file mode 100644 index ac6d55da0..000000000 --- a/docs/changes/newsfragments/321.fix +++ /dev/null @@ -1 +0,0 @@ -Add the executable flag to the ants docker scripts, fsl docker scripts and other running scripts by `Fede Raimondo_` \ No newline at end of file diff --git a/docs/whats_new.rst b/docs/whats_new.rst index ba5968ecf..276dbb294 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -46,6 +46,10 @@ Bugfixes (:gh:`312`) - Fix element access for :class:`.DMCC13Benchmark` DataGrabber by `Synchon Mandal`_ (:gh:`314`) +- Add a validation step on the :func:`.run` function to validate the marker + collection by `Fede Raimondo`_ (:gh:`320`) +- Add the executable flag to the ants docker scripts, fsl docker scripts and + other running scripts by `Fede Raimondo`_ (:gh:`321`) - Force ``str`` dtype when parsing elements from file by `Synchon Mandal`_ (:gh:`322`) -- 2.52.0