docs/changes/newsfragments/249.feature

@@ -0,0 +1 @@
+Support ``venv`` as environment kind for queueing jobs by `Synchon Mandal`_

docs/using/queueing.rst

@@ -52,11 +52,13 @@ The following parameters are available for HTCondor:
   * ``kind``: This is the kind of virtual environment to use:
 
     * ``conda``
-    * ``virtualenv`` (not yet supported)
+    * ``venv``
     * ``local`` (no virtual environment)
 
   * ``name``: This is the name of the environment to use in case a virtual
-    environment is used.
+    environment is used. It should be the name when ``conda`` is used and
+    the absolute or relative path to the virtualenv when ``venv`` is used.
+    If relative path is used then it should be relative to the YAML.
 
 * ``mem``: Memory to be used by the job. It must be provided as a string with
   the units (e.g. ``2GB``).

junifer/api/functions.py

@@ -418,7 +418,10 @@ def _queue_condor(
         env_name = env["name"]
         executable = "run_venv.sh"
         arguments = f"{env_name} junifer"
-        # TODO: Copy run_venv.sh to jobdir
+        exec_path = jobdir / executable
+        logger.info(f"Copying {executable} to {exec_path.absolute()!s}")
+        shutil.copy(Path(__file__).parent / "res" / executable, exec_path)
+        make_executable(exec_path)
     elif env["kind"] == "local":
         executable = "junifer"
         arguments = ""

junifer/api/parser.py

@@ -99,4 +99,17 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
                 contents["storage"]["uri"] = str(
                     (filepath.parent / uri_path).resolve()
                 )
+
+    # Allow relative path if queue env kind is venv; same motivation as above
+    if "queue" in contents:
+        if "env" in contents["queue"]:
+            if "venv" == contents["queue"]["env"]["kind"]:
+                # Check if the env name is relative
+                venv_path = Path(contents["queue"]["env"]["name"])
+                if not venv_path.is_absolute():
+                    # Compute the absolute path
+                    contents["queue"]["env"]["name"] = str(
+                        (filepath.parent / venv_path).resolve()
+                    )
+
     return contents

junifer/api/res/run_conda.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 if [ $# -lt 2 ]; then
-    echo "This script is meant to run a command within a python environment"
+    echo "This script is meant to run a command within a conda environment."
     echo "It needs at least 2 parameters."
     echo "The first one must be the environment name."
-    echo "The rest will be the command"
+    echo "The rest will be the command."
     exit 255
 fi
 
 eval "$(conda shell.bash hook)"
 env_name=$1
 echo "Activating ${env_name}"
-conda activate "$1"
+conda activate "${env_name}"
 shift 1
 
 if [ -f "pre_run.sh" ]; then
@@ -19,5 +19,5 @@ if [ -f "pre_run.sh" ]; then
     . ./pre_run.sh
 fi
 
-echo "Running ${*} in virtual environment"
+echo "Running ${*} in conda environment"
 "$@"

junifer/api/res/run_venv.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ $# -lt 2 ]; then
+    echo "This script is meant to run a command within a Python virtual environment."
+    echo "It needs at least 2 parameters."
+    echo "The first one must be the virtualenv path."
+    echo "The rest will be the command."
+    exit 255
+fi
+
+env_path=$1
+echo "Activating ${env_path}"
+source "${env_path}"/bin/activate
+shift 1
+
+if [ -f "pre_run.sh" ]; then
+    echo "Sourcing pre_run.sh"
+    . ./pre_run.sh
+fi
+
+echo "Running ${*} in Python virtual environment"
+"$@"