diff --git a/docs/changes/newsfragments/249.feature b/docs/changes/newsfragments/249.feature new file mode 100644 index 000000000..1d36037e1 --- /dev/null +++ b/docs/changes/newsfragments/249.feature @@ -0,0 +1 @@ +Support ``venv`` as environment kind for queueing jobs by `Synchon Mandal`_ diff --git a/docs/using/queueing.rst b/docs/using/queueing.rst index 8861cd51b..7d09577fd 100644 --- a/docs/using/queueing.rst +++ b/docs/using/queueing.rst @@ -52,11 +52,13 @@ The following parameters are available for HTCondor: * ``kind``: This is the kind of virtual environment to use: * ``conda`` - * ``virtualenv`` (not yet supported) + * ``venv`` * ``local`` (no virtual environment) * ``name``: This is the name of the environment to use in case a virtual - environment is used. + environment is used. It should be the name when ``conda`` is used and + the absolute or relative path to the virtualenv when ``venv`` is used. + If relative path is used then it should be relative to the YAML. * ``mem``: Memory to be used by the job. It must be provided as a string with the units (e.g. ``2GB``). diff --git a/junifer/api/functions.py b/junifer/api/functions.py index ec00a93ee..efcd197b3 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -418,7 +418,10 @@ def _queue_condor( env_name = env["name"] executable = "run_venv.sh" arguments = f"{env_name} junifer" - # TODO: Copy run_venv.sh to jobdir + exec_path = jobdir / executable + logger.info(f"Copying {executable} to {exec_path.absolute()!s}") + shutil.copy(Path(__file__).parent / "res" / executable, exec_path) + make_executable(exec_path) elif env["kind"] == "local": executable = "junifer" arguments = "" diff --git a/junifer/api/parser.py b/junifer/api/parser.py index 97ecf3fc2..535520995 100644 --- a/junifer/api/parser.py +++ b/junifer/api/parser.py @@ -99,4 +99,17 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict: contents["storage"]["uri"] = str( (filepath.parent / uri_path).resolve() ) + + # Allow relative path if queue env kind is venv; same motivation as above + if "queue" in contents: + if "env" in contents["queue"]: + if "venv" == contents["queue"]["env"]["kind"]: + # Check if the env name is relative + venv_path = Path(contents["queue"]["env"]["name"]) + if not venv_path.is_absolute(): + # Compute the absolute path + contents["queue"]["env"]["name"] = str( + (filepath.parent / venv_path).resolve() + ) + return contents diff --git a/junifer/api/res/run_conda.sh b/junifer/api/res/run_conda.sh index 114247feb..3efcbbba5 100755 --- a/junifer/api/res/run_conda.sh +++ b/junifer/api/res/run_conda.sh @@ -1,17 +1,17 @@ #!/bin/bash if [ $# -lt 2 ]; then - echo "This script is meant to run a command within a python environment" + echo "This script is meant to run a command within a conda environment." echo "It needs at least 2 parameters." echo "The first one must be the environment name." - echo "The rest will be the command" + echo "The rest will be the command." exit 255 fi eval "$(conda shell.bash hook)" env_name=$1 echo "Activating ${env_name}" -conda activate "$1" +conda activate "${env_name}" shift 1 if [ -f "pre_run.sh" ]; then @@ -19,5 +19,5 @@ if [ -f "pre_run.sh" ]; then . ./pre_run.sh fi -echo "Running ${*} in virtual environment" +echo "Running ${*} in conda environment" "$@" diff --git a/junifer/api/res/run_venv.sh b/junifer/api/res/run_venv.sh new file mode 100644 index 000000000..b6648031e --- /dev/null +++ b/junifer/api/res/run_venv.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ $# -lt 2 ]; then + echo "This script is meant to run a command within a Python virtual environment." + echo "It needs at least 2 parameters." + echo "The first one must be the virtualenv path." + echo "The rest will be the command." + exit 255 +fi + +env_path=$1 +echo "Activating ${env_path}" +source "${env_path}"/bin/activate +shift 1 + +if [ -f "pre_run.sh" ]; then + echo "Sourcing pre_run.sh" + . ./pre_run.sh +fi + +echo "Running ${*} in Python virtual environment" +"$@"