From 08361d0f1738fd939f307ccad45fc98e1524e2cc Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 27 Feb 2024 16:33:44 +0100 Subject: [PATCH 1/6] feature: add support for using venv for queue --- junifer/api/functions.py | 5 ++++- junifer/api/res/run_venv.sh | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 junifer/api/res/run_venv.sh diff --git a/junifer/api/functions.py b/junifer/api/functions.py index ec00a93ee..efcd197b3 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -418,7 +418,10 @@ def _queue_condor( env_name = env["name"] executable = "run_venv.sh" arguments = f"{env_name} junifer" - # TODO: Copy run_venv.sh to jobdir + exec_path = jobdir / executable + logger.info(f"Copying {executable} to {exec_path.absolute()!s}") + shutil.copy(Path(__file__).parent / "res" / executable, exec_path) + make_executable(exec_path) elif env["kind"] == "local": executable = "junifer" arguments = "" diff --git a/junifer/api/res/run_venv.sh b/junifer/api/res/run_venv.sh new file mode 100644 index 000000000..5e6780c2f --- /dev/null +++ b/junifer/api/res/run_venv.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ $# -lt 2 ]; then + echo "This script is meant to run a command within a python environment" + echo "It needs at least 2 parameters." + echo "The first one must be the environment name." + echo "The rest will be the command" + exit 255 +fi + +env_name=$1 +echo "Activating ${env_name}" +source "${env_name}"/bin/activate +shift 1 + +if [ -f "pre_run.sh" ]; then + echo "Sourcing pre_run.sh" + . ./pre_run.sh +fi + +echo "Running ${*} in virtual environment" +"$@" -- 2.52.0 From d0b24b7997ac5291b6c9c97c4f8416710bc93e09 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 27 Feb 2024 17:08:38 +0100 Subject: [PATCH 2/6] docs: update queueing.rst --- docs/using/queueing.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/using/queueing.rst b/docs/using/queueing.rst index 8861cd51b..24fef7716 100644 --- a/docs/using/queueing.rst +++ b/docs/using/queueing.rst @@ -52,11 +52,12 @@ The following parameters are available for HTCondor: * ``kind``: This is the kind of virtual environment to use: * ``conda`` - * ``virtualenv`` (not yet supported) + * ``venv`` * ``local`` (no virtual environment) * ``name``: This is the name of the environment to use in case a virtual - environment is used. + environment is used. It should be the name when ``conda`` is used and + the absolute path to the virtualenv when ``venv`` is used. * ``mem``: Memory to be used by the job. It must be provided as a string with the units (e.g. ``2GB``). -- 2.52.0 From 280b6a9a737b0c97913ffd2ab055b2ab4f04b862 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 27 Feb 2024 17:08:50 +0100 Subject: [PATCH 3/6] chore: add changelog 249.feature --- docs/changes/newsfragments/249.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/249.feature diff --git a/docs/changes/newsfragments/249.feature b/docs/changes/newsfragments/249.feature new file mode 100644 index 000000000..1d36037e1 --- /dev/null +++ b/docs/changes/newsfragments/249.feature @@ -0,0 +1 @@ +Support ``venv`` as environment kind for queueing jobs by `Synchon Mandal`_ -- 2.52.0 From 22ddaeb852b8d855c38a35f08420bdc38d6dfe0f Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 28 Feb 2024 11:27:17 +0100 Subject: [PATCH 4/6] chore: improve echo in run_conda.sh and run_venv.sh --- junifer/api/res/run_conda.sh | 8 ++++---- junifer/api/res/run_venv.sh | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/junifer/api/res/run_conda.sh b/junifer/api/res/run_conda.sh index 114247feb..3efcbbba5 100755 --- a/junifer/api/res/run_conda.sh +++ b/junifer/api/res/run_conda.sh @@ -1,17 +1,17 @@ #!/bin/bash if [ $# -lt 2 ]; then - echo "This script is meant to run a command within a python environment" + echo "This script is meant to run a command within a conda environment." echo "It needs at least 2 parameters." echo "The first one must be the environment name." - echo "The rest will be the command" + echo "The rest will be the command." exit 255 fi eval "$(conda shell.bash hook)" env_name=$1 echo "Activating ${env_name}" -conda activate "$1" +conda activate "${env_name}" shift 1 if [ -f "pre_run.sh" ]; then @@ -19,5 +19,5 @@ if [ -f "pre_run.sh" ]; then . ./pre_run.sh fi -echo "Running ${*} in virtual environment" +echo "Running ${*} in conda environment" "$@" diff --git a/junifer/api/res/run_venv.sh b/junifer/api/res/run_venv.sh index 5e6780c2f..b6648031e 100644 --- a/junifer/api/res/run_venv.sh +++ b/junifer/api/res/run_venv.sh @@ -1,16 +1,16 @@ #!/bin/bash if [ $# -lt 2 ]; then - echo "This script is meant to run a command within a python environment" + echo "This script is meant to run a command within a Python virtual environment." echo "It needs at least 2 parameters." - echo "The first one must be the environment name." - echo "The rest will be the command" + echo "The first one must be the virtualenv path." + echo "The rest will be the command." exit 255 fi -env_name=$1 -echo "Activating ${env_name}" -source "${env_name}"/bin/activate +env_path=$1 +echo "Activating ${env_path}" +source "${env_path}"/bin/activate shift 1 if [ -f "pre_run.sh" ]; then @@ -18,5 +18,5 @@ if [ -f "pre_run.sh" ]; then . ./pre_run.sh fi -echo "Running ${*} in virtual environment" +echo "Running ${*} in Python virtual environment" "$@" -- 2.52.0 From 6dfc2154a3bd88e143066daf3af9790085206104 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 28 Feb 2024 11:53:05 +0100 Subject: [PATCH 5/6] update: allow relative path for venv name --- junifer/api/parser.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/junifer/api/parser.py b/junifer/api/parser.py index 97ecf3fc2..535520995 100644 --- a/junifer/api/parser.py +++ b/junifer/api/parser.py @@ -99,4 +99,17 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict: contents["storage"]["uri"] = str( (filepath.parent / uri_path).resolve() ) + + # Allow relative path if queue env kind is venv; same motivation as above + if "queue" in contents: + if "env" in contents["queue"]: + if "venv" == contents["queue"]["env"]["kind"]: + # Check if the env name is relative + venv_path = Path(contents["queue"]["env"]["name"]) + if not venv_path.is_absolute(): + # Compute the absolute path + contents["queue"]["env"]["name"] = str( + (filepath.parent / venv_path).resolve() + ) + return contents -- 2.52.0 From e0b66de0a16b55a12ecf1676ec6b74ed63a50423 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 28 Feb 2024 11:54:59 +0100 Subject: [PATCH 6/6] docs: update queueing.rst --- docs/using/queueing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/using/queueing.rst b/docs/using/queueing.rst index 24fef7716..7d09577fd 100644 --- a/docs/using/queueing.rst +++ b/docs/using/queueing.rst @@ -57,7 +57,8 @@ The following parameters are available for HTCondor: * ``name``: This is the name of the environment to use in case a virtual environment is used. It should be the name when ``conda`` is used and - the absolute path to the virtualenv when ``venv`` is used. + the absolute or relative path to the virtualenv when ``venv`` is used. + If relative path is used then it should be relative to the YAML. * ``mem``: Memory to be used by the job. It must be provided as a string with the units (e.g. ``2GB``). -- 2.52.0