562 lines
20 KiB
Shell
562 lines
20 KiB
Shell
set -e -u
|
|
|
|
# project name space
|
|
PROJECT="mriqc"
|
|
# define SAMPLE to be processed
|
|
SAMPLE="$1"
|
|
# define the output store to push all results to
|
|
output_store="https://cerebra.fz-juelich.de/f.hoffstaedter"
|
|
# define and input ria-store only to clone from
|
|
inputstore="/data/project/QC_workflow/RIA/inputstore"
|
|
# add spec to name for debugging runs
|
|
spec=""
|
|
|
|
# define clonable location for BIDS inputs
|
|
raw_store="https://cerebra.fz-juelich.de/openneuro/${SAMPLE}"
|
|
raw_ds=""
|
|
# define temporal working directory for job execution
|
|
temporary_store="/tmp"
|
|
|
|
# how much to parallelize your single subject functional jobs
|
|
parallelize=2
|
|
RAM=6000
|
|
### parallelization of subjects on compute nodes for node wise job submission
|
|
nodesubs=128
|
|
|
|
# mriqc container from the Repronim container datalad dataset
|
|
container_store="https://github.com/ReproNim/containers.git"
|
|
container_ds=""
|
|
container='bids-mriqc'
|
|
version='24.0.2'
|
|
|
|
##### don't change anything below if you are not sure what you want to do #####
|
|
|
|
# define ID for git commits (take from local user configuration)
|
|
git_name="$(git config user.name)"
|
|
git_email="$(git config user.email)"
|
|
|
|
# define and create the input ria-store only to clone from
|
|
input_store="ria+file://${inputstore}"
|
|
mkdir -p ${inputstore}
|
|
|
|
# all results a tracked in a single output dataset
|
|
# create a fresh one
|
|
# job submission will take place from a checkout of this dataset, but no
|
|
# results will be pushed into it
|
|
# Create a source dataset with all analysis components as an analysis access
|
|
# point.
|
|
datalad create -c yoda ${SAMPLE}-${PROJECT}${version}${spec}
|
|
cd ${SAMPLE}-${PROJECT}${version}${spec}
|
|
# add html in unlocked mode and keep tsvs and datalset_description in git
|
|
git annex config --set annex.addunlocked 'include=*.html'
|
|
echo ".bidsignore annex.largefiles=nothing
|
|
dataset_description.json annex.largefiles=nothing
|
|
group*.tsv annex.largefiles=nothing
|
|
mclf_data-unseen_pred.csv annex.largefiles=nothing" >> .gitattributes
|
|
datalad save -m "keep .bidsignore, dataset_description.json & group results in git"
|
|
|
|
# the actual compute job specification
|
|
cat > README.md << EOT
|
|
|
|
# ${SAMPLE} - MRIQC derivatives
|
|
|
|
This dataset is a BIDS Derivatives dataset resulting from running MRIQC ${version} on [${SAMPLE}](https://openneuro.org/datasets/${SAMPLE}).
|
|
|
|
## Methods
|
|
|
|
MRIQC documentation can be found on https://mriqc.readthedocs.io/.
|
|
|
|
This dataset was created using [BOOTSTRAP - a high-throughput container workflow for MRIQC](https://cerebra.fz-juelich.de/f.hoffstaedter/bootstrap_MRIQC) implementing the [FAIRly Big Workflow](https://doi.org/10.1038/s41597-022-01163-2) using [Datalad](https://www.datalad.org/) and [Singularity containers](https://apptainer.org/) for full reproducibility and provenance capture including resource consumption via [con-duct](https://github.com/con/duct).
|
|
|
|
## Acknowledgements
|
|
The authors from the Institute of Neuroscience and Medicine (INM-7) at the Forschungszentrum Jülich, Germany gratefully acknowledge computing time on the supercomputer JURECA[1] at Forschungszentrum Jülich under grant no. 65913.
|
|
|
|
[1] Jülich Supercomputing Centre. (2021). JURECA: Data Centric and Booster Modules implementing the Modular Supercomputing Architecture at Jülich Supercomputing Centre Journal of large-scale research facilities, 7, A182. http://dx.doi.org/10.17815/jlsrf-7-182
|
|
EOT
|
|
datalad save -m "add README with Acknowledgements"
|
|
|
|
# clone the container-dataset as a subdataset.
|
|
datalad clone -d . "${container_store}" code/containers
|
|
# configure a custom container call to satisfy the needs of this analysis
|
|
datalad run -m "Freeze mriqc container version" \
|
|
code/containers/scripts/freeze_versions --save-dataset=. ${container}=${version}
|
|
datalad run -m "Setup elderly mriqc version with classification support" \
|
|
code/containers/scripts/freeze_versions --save-dataset=. ${container}:${container}-clf=0.15.1
|
|
datalad run -m "Remove datalad-get option and do singularity exec instead of run" \
|
|
"sed -i -e 's, --no-datalad-get,,g' .datalad/config
|
|
sed -i -e 's, run, exec,g' .datalad/config"
|
|
|
|
# create dedicated input and output locations. Results will be pushed into the
|
|
# output sibling, and the analysis will start with a clone from the input
|
|
# sibling.
|
|
datalad create-sibling-ria -s ${PROJECT}_in "${input_store}" \
|
|
--alias ${SAMPLE}-${PROJECT}${version}${spec} --new-store-ok --storage-sibling off
|
|
git remote add ${PROJECT}_out ${output_store}
|
|
git push ${PROJECT}_out
|
|
|
|
# register the input dataset, a superdataset comprising all participants
|
|
datalad clone -d . "${raw_store}${raw_ds}" sourcedata/raw
|
|
# amend the previous commit with a nicer commit message
|
|
git commit --amend -m "Register ${SAMPLE} BIDS dataset as sourcedata"
|
|
|
|
# the actual compute job specification
|
|
cat > code/participant_job << EOT
|
|
#!/bin/bash
|
|
|
|
# the job assumes that it is a good idea to run everything in PWD
|
|
# the job manager should make sure that is true
|
|
|
|
# fail whenever something is fishy, use -x to get verbose logfiles
|
|
set -e -u -x
|
|
|
|
dssource="\$1"
|
|
pushgitremote="\$2"
|
|
subid="\$3"
|
|
|
|
export DUCT_OUTPUT_PREFIX="logs/\${subid}_{datetime_filesafe}-{pid}."
|
|
|
|
# get the analysis dataset, which includes the inputs as well
|
|
# importantly, we do not clone from the lcoation that we want to push the
|
|
# results too, in order to avoid too many jobs blocking access to
|
|
# the same location and creating a throughput bottleneck
|
|
datalad clone "\${dssource}" ds
|
|
|
|
# all following actions are performed in the context of the superdataset
|
|
cd ds
|
|
|
|
# in order to avoid accumulation temporary git-annex availability information
|
|
# and to avoid a syncronization bottleneck by having to consolidate the
|
|
# git-annex branch across jobs, we will only push the main tracking branch
|
|
# back to the output store (plus the actual file content). Final availability
|
|
# information can be establish via an eventual "git-annex fsck -f ${PROJECT}_out-storage".
|
|
# this remote is never fetched, it accumulates a larger number of branches
|
|
# and we want to avoid progressive slowdown. Instead we only ever push
|
|
# a unique branch per each job (subject AND process specific name)
|
|
git remote add outputstore "\$pushgitremote"
|
|
|
|
# all results of this job will be put into a dedicated branch
|
|
git checkout -b "job_\${JOBID}"
|
|
|
|
# we pull down the input subject manually in order to discover relevant
|
|
# files. We do this outside the recorded call, because on a potential
|
|
# re-run we want to be able to do fine-grained recomputing of individual
|
|
# outputs. The recorded calls will have specific paths that will enable
|
|
# recomputation outside the scope of the original Condor setup
|
|
datalad get -n "sourcedata/raw/"
|
|
|
|
# the meat of the matter
|
|
# look for T1w files in the input data for the given participant
|
|
# it is critical for reproducibility that the command given to
|
|
# "containers-run" does not rely on any property of the immediate
|
|
# computational environment (env vars, services, etc)
|
|
|
|
# do T1w first
|
|
datalad containers-run \\
|
|
-m "Compute MRIQC for T1w of \${subid}" \\
|
|
-n ${container} \\
|
|
-i sourcedata/raw/\${subid} \\
|
|
-i sourcedata/raw/dataset_description.json \\
|
|
mriqc sourcedata/raw mriqc participant \\
|
|
--participant-label \$subid \\
|
|
--modalities T1w T2w \\
|
|
--no-datalad-get \\
|
|
--no-sub \\
|
|
--verbose \\
|
|
--nprocs ${nthreads} \\
|
|
--omp-nthreads ${nthreads} \\
|
|
--mem $mb_RAM \\
|
|
--work-dir tmp \\
|
|
--float32 \\
|
|
--verbose-reports || true
|
|
|
|
# do bold next
|
|
datalad containers-run \\
|
|
-m "Compute MRIQC for BOLD of \${subid}" \\
|
|
-n ${container} \\
|
|
-i sourcedata/raw/\${subid} \\
|
|
-i sourcedata/raw/dataset_description.json \\
|
|
mriqc sourcedata/raw mriqc participant \\
|
|
--participant-label \$subid \\
|
|
--modalities bold \\
|
|
--no-datalad-get \\
|
|
--no-sub \\
|
|
--verbose \\
|
|
--nprocs ${nthreads} \\
|
|
--omp-nthreads ${nthreads} \\
|
|
--mem $mb_RAM \\
|
|
--work-dir tmp \\
|
|
--float32 \\
|
|
--min-bold-length 10 \\
|
|
--verbose-reports || true
|
|
|
|
# then run dwi seperately and continue after error
|
|
datalad containers-run \\
|
|
-m "Compute MRIQC for DWI of \${subid}" \\
|
|
-n ${container} \\
|
|
-i sourcedata/raw/\${subid} \\
|
|
-i sourcedata/raw/dataset_description.json \\
|
|
mriqc sourcedata/raw mriqc participant \\
|
|
--participant-label \$subid \\
|
|
--modalities dwi \\
|
|
--no-datalad-get \\
|
|
--no-sub \\
|
|
--verbose \\
|
|
--nprocs ${nthreads} \\
|
|
--omp-nthreads ${nthreads} \\
|
|
--mem $mb_RAM \\
|
|
--work-dir tmp \\
|
|
--float32 \\
|
|
--min-dwi-length 6 \\
|
|
--verbose-reports || true
|
|
|
|
# file content first -- does not need a lock, no interaction with Git
|
|
git annex copy -t outputstore
|
|
# and the output branch
|
|
flock --verbose \$DSLOCKFILE git push outputstore
|
|
|
|
echo "SUBJECT JOB COMPLETED"
|
|
# job handler should clean up workspace
|
|
EOT
|
|
|
|
chmod +x code/participant_job
|
|
datalad save -m "Participant compute job implementation" code/participant_job
|
|
|
|
|
|
cat > code/process.sub << EOT
|
|
#!/bin/bash
|
|
|
|
subid="\$1"
|
|
|
|
executable=\$(pwd)/code/participant_job
|
|
|
|
# the job expects these environment variables for labeling and synchronization
|
|
# - JOBID: subject AND process specific ID to make a branch name from
|
|
# (must be unique across all (even multiple) submissions)
|
|
# including the cluster ID will enable sorting multiple computing attempts
|
|
# - DSLOCKFILE: lock (must be accessible from all compute jobs) to synchronize
|
|
# write access to the output dataset
|
|
# - DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__...:
|
|
# (additional) locations for datalad to locate relevant subdatasets, in case
|
|
# a configured URL is outdated
|
|
# - GIT_AUTHOR_...: Identity information used to save dataset changes in compute
|
|
# jobs
|
|
export JOBID=\${subid} \\
|
|
DSLOCKFILE=\$(pwd)/.condor_datalad_lock \\
|
|
GIT_AUTHOR_NAME='${git_name}' \\
|
|
GIT_AUTHOR_EMAIL='${git_email}' \\
|
|
REPRONIM_USE_DUCT=1
|
|
|
|
# essential args for "participant_job"
|
|
# 1: where to clone the analysis dataset
|
|
# 2: location to push the result git branch to. The "ria+" prefix is stripped.
|
|
# 3: ID of the subject to process
|
|
arguments="${input_store}#$(datalad -f '{infos[dataset][id]}' wtf -S dataset) \\
|
|
$(git remote get-url --push ${PROJECT}_out) \\
|
|
\${subid} \\
|
|
"
|
|
|
|
mkdir -p ${temporary_store}/tmp_\${subid:4}
|
|
cd ${temporary_store}/tmp_\${subid:4}
|
|
|
|
\${executable} \${arguments} \\
|
|
> $(pwd)/logs/\${subid}.out \\
|
|
2> $(pwd)/logs/\${subid}.err
|
|
|
|
chmod +w -R ${temporary_store}/tmp_\${subid:4} && \
|
|
rm -rf ${temporary_store}/tmp_\${subid:4}
|
|
EOT
|
|
|
|
chmod +x code/process.sub
|
|
datalad save -m "individual job submission" code/process.sub
|
|
|
|
|
|
cat > code/results.merger << EOT
|
|
#!/bin/bash
|
|
|
|
# fail whenever something is fishy, use -x to get verbose logfiles
|
|
set -e -x
|
|
|
|
# finalize FAIRly Big Workflow dataset
|
|
|
|
dssource="\$1"
|
|
|
|
if [[ ! -z \${dssource} ]]; then
|
|
datalad clone "\${dssource}" ds
|
|
cd ds
|
|
else
|
|
datalad update -s ${PROJECT}_out
|
|
fi
|
|
|
|
# octopus merge job branches and move data to root
|
|
git merge -m "Merge results" \$(git branch -al | grep 'job_' | tr -d ' ')
|
|
git mv mriqc/.bidsignore mriqc/* .
|
|
datalad save -m "move data to dataset root"
|
|
|
|
# clean git annex branch
|
|
if [[ ! -z \${dssource} ]]; then
|
|
git annex fsck --fast
|
|
else
|
|
git annex fsck -f ${PROJECT}_out --fast
|
|
fi
|
|
|
|
# declare local data clone as dead
|
|
git annex dead here
|
|
|
|
# datalad push merged results
|
|
if [[ ! -z \${dssource} ]]; then
|
|
datalad push --data nothing
|
|
else
|
|
datalad push --data nothing --to ${PROJECT}_out
|
|
fi
|
|
|
|
export DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__102container=ria+file:///data/project/cat_preprocessed/dataladstore#{id}" \
|
|
REPRONIM_USE_DUCT=1 \
|
|
DUCT_OUTPUT_PREFIX="logs/mriqc-group-stats_{datetime_filesafe}-{pid}_"
|
|
|
|
# get mri input for MRIQC group stats
|
|
datalad get -n sourcedata/raw
|
|
datalad get sourcedata/raw/dataset_description.json \\
|
|
\$(find sourcedata/raw -maxdepth 3 -name anat) \\
|
|
\$(find sourcedata/raw -maxdepth 3 -name func) \\
|
|
\$(find sourcedata/raw -maxdepth 3 -name dwi)
|
|
|
|
# run mriqc group stats
|
|
datalad containers-run \\
|
|
-m "Compute MRIQC group stats" \\
|
|
-n bids-mriqc \\
|
|
--explicit \\
|
|
-o 'group*' \\
|
|
-o .bidsignore \\
|
|
mriqc sourcedata/raw . group \\
|
|
--no-datalad-get \\
|
|
--notrack \\
|
|
--verbose \\
|
|
--work-dir tmp
|
|
|
|
datalad drop --what datasets --reckless kill -r -d sourcedata/raw
|
|
|
|
# run mriqc quality classifier
|
|
datalad containers-run \\
|
|
-m "run MRIQC classifier" \\
|
|
-n bids-mriqc-clf \\
|
|
-i group_T1w.tsv \\
|
|
"mriqc_clf --load-classifier -X group_T1w.tsv; \\
|
|
mv mclf_run*data-unseen_pred.csv mclf_data-unseen_pred.csv"
|
|
|
|
if [[ ! -z \${dssource} ]]; then
|
|
datalad push --data nothing
|
|
else
|
|
datalad push --data nothing --to ${PROJECT}_out
|
|
fi
|
|
|
|
echo SUCCESS
|
|
EOT
|
|
chmod +x code/results.merger
|
|
datalad save -m "finalize dataset by merging results branches into master" code/results.merger
|
|
|
|
# create clean up script
|
|
cat > code/killall.sh << EOT
|
|
#!/bin/bash
|
|
#
|
|
# if setup went wrong, delete everything
|
|
TOKEN=\$1
|
|
|
|
[[ "\$(read -e -p 'Are you sure you want to delete everything? [yes_sure!/NO]> '; echo \$REPLY)" == yes_sure! ]] && echo KILLALLNOW || exit
|
|
|
|
# delete wrong input & output RIA stores + aliases
|
|
rm -rf $(git config --get remote.${PROJECT}_in.url)
|
|
rm -f ${inputstore}/alias/${SAMPLE}-${PROJECT}${version}
|
|
|
|
curl -X 'DELETE' \
|
|
"https://cerebra.fz-juelich.de/api/v1/repos/f.hoffstaedter/${SAMPLE}-${PROJECT}${version}${spec}" \
|
|
-H \'accept: application/json\' \
|
|
-H \"Authorization: token \${TOKEN}\"
|
|
|
|
# remove faulty dataset
|
|
cd ..
|
|
datalad drop --what datasets --reckless kill -r -d ${SAMPLE}-${PROJECT}${spec}${version}
|
|
EOT
|
|
|
|
chmod +x code/killall.sh
|
|
datalad save -m "add clean up script" code/killall.sh
|
|
|
|
|
|
###############################################################################
|
|
# HTCONDOR SETUP START - FIXME remove or adjust this according to your needs.
|
|
###############################################################################
|
|
|
|
# HTCondor compute setup
|
|
# the workspace is to be ignored by git
|
|
echo dag_tmp >> .gitignore
|
|
echo .condor_datalad_lock >> .gitignore
|
|
|
|
## define ID for git commits (take from local user configuration)
|
|
git_name="$(git config user.name)"
|
|
git_email="$(git config user.email)"
|
|
|
|
# compute environment for a single job
|
|
#-------------------------------------------------------------------------------
|
|
# FIXME: Adjust job requirements to your needs
|
|
|
|
cat > code/process.condor_submit << EOT
|
|
universe = vanilla
|
|
# resource requirements for each job
|
|
request_cpus = 1
|
|
request_memory = ${mb_RAM}M
|
|
request_disk = 10G
|
|
|
|
# be nice and only use free resources
|
|
# nice_user = true
|
|
|
|
# tell condor that a job is self contained and the executable
|
|
# is enough to bootstrap the computation on the execute node
|
|
should_transfer_files = yes
|
|
# explicitly do not transfer anything back
|
|
# we are using datalad for everything that matters
|
|
transfer_output_files = ""
|
|
|
|
# the actual job script, nothing condor-specific in it
|
|
executable = \$ENV(PWD)/code/participant_job
|
|
|
|
# the job expects these environment variables for labeling and synchronization
|
|
# - JOBID: subject AND process specific ID to make a branch name from
|
|
# (must be unique across all (even multiple) submissions)
|
|
# including the cluster ID will enable sorting multiple computing attempts
|
|
# - DSLOCKFILE: lock (must be accessible from all compute jobs) to synchronize
|
|
# write access to the output dataset
|
|
# - DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__...:
|
|
# (additional) locations for datalad to locate relevant subdatasets, in case
|
|
# a configured URL is outdated
|
|
# - GIT_AUTHOR_...: Identity information used to save dataset changes in compute
|
|
# jobs
|
|
# - REPRONIM_USE_DUCT: use duct to log compute resource usage in Repronim container
|
|
environment = "\\
|
|
JOBID=\$(subject)_\$(Cluster) \\
|
|
DSLOCKFILE=\$ENV(PWD)/.condor_datalad_lock \\
|
|
GIT_AUTHOR_NAME='${git_name}' \\
|
|
GIT_AUTHOR_EMAIL='${git_email}' \\
|
|
DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__101mriqc=ria+file:///data/project/cat_preprocessed/dataladstore#{id} \\
|
|
REPRONIM_USE_DUCT=1 \\
|
|
"
|
|
|
|
# provide local environment in condor job
|
|
getenv = true
|
|
|
|
# place the job logs into PWD/logs, using the same name as for the result branches
|
|
# (JOBID)
|
|
log = \$ENV(PWD)/logs/\$(subject)_\$(Cluster).log
|
|
output = \$ENV(PWD)/logs/\$(subject)_\$(Cluster).out
|
|
error = \$ENV(PWD)/logs/\$(subject)_\$(Cluster).err
|
|
|
|
# essential args for "participant_job"
|
|
# 1: where to clone the analysis dataset
|
|
# 2: location to push the result git branch to. The "ria+" prefix is stripped.
|
|
# 3: ID of the subject to process
|
|
arguments = "\\
|
|
${input_store}#$(datalad -f '{infos[dataset][id]}' wtf -S dataset) \\
|
|
$(git remote get-url --push ${PROJECT}_out) \\
|
|
\$(subject) \\
|
|
"
|
|
queue
|
|
EOT
|
|
|
|
cat > code/process.condor_submit-merge << EOT
|
|
universe = vanilla
|
|
# resource requirements for each job
|
|
request_cpus = 1
|
|
request_memory = ${mb_RAM}M
|
|
request_disk = 100G
|
|
|
|
# be nice and only use free resources
|
|
# nice_user = true
|
|
|
|
# tell condor that a job is self contained and the executable
|
|
# is enough to bootstrap the computation on the execute node
|
|
should_transfer_files = yes
|
|
# explicitly do not transfer anything back
|
|
# we are using datalad for everything that matters
|
|
transfer_output_files = ""
|
|
|
|
# the actual job script, nothing condor-specific in it
|
|
executable = \$ENV(PWD)/code/results.merger
|
|
|
|
# the job expects these environment variables for labeling and synchronization
|
|
# - JOBID: subject AND process specific ID to make a branch name from
|
|
# (must be unique across all (even multiple) submissions)
|
|
# including the cluster ID will enable sorting multiple computing attempts
|
|
# - DSLOCKFILE: lock (must be accessible from all compute jobs) to synchronize
|
|
# write access to the output dataset
|
|
# - DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__...:
|
|
# (additional) locations for datalad to locate relevant subdatasets, in case
|
|
# a configured URL is outdated
|
|
# - GIT_AUTHOR_...: Identity information used to save dataset changes in compute
|
|
# jobs
|
|
environment = "\\
|
|
JOBID=\$(subject)_\$(Cluster) \\
|
|
DSLOCKFILE=\$ENV(PWD)/.condor_datalad_lock \\
|
|
GIT_AUTHOR_NAME='${git_name}' \\
|
|
GIT_AUTHOR_EMAIL='${git_email}' \\
|
|
REPRONIM_USE_DUCT=1 \\
|
|
PATH=~/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \\
|
|
"
|
|
|
|
# place the job logs into PWD/logs, using the same name as for the result branches
|
|
# (JOBID)
|
|
log = \$ENV(PWD)/logs/postpro_\$(Cluster).log
|
|
output = \$ENV(PWD)/logs/postpro_\$(Cluster).out
|
|
error = \$ENV(PWD)/logs/postpro_\$(Cluster).err
|
|
# essential args for "results.merger"
|
|
# 1: where to clone the analysis dataset
|
|
arguments = "\\
|
|
${output_store}#$(datalad -f '{infos[dataset][id]}' wtf -S dataset) \\
|
|
"
|
|
queue
|
|
EOT
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# FIXME: Adjust the find command below to return the unit over which your
|
|
# analysis should parallelize. Here, subject directories on the first hierarchy
|
|
# level in the input data are returned by searching for the "sub-*" prefix.
|
|
# The setup below creates an HTCondor DAG.
|
|
# ------------------------------------------------------------------------------
|
|
# processing graph specification for computing all jobs
|
|
cat > code/process.condor_dag << "EOT"
|
|
# Processing DAG
|
|
EOT
|
|
|
|
for s in $(find sourcedata/raw -maxdepth 1 -name 'sub-*' -printf '%f\n'); do
|
|
printf "JOB ${s%.*} code/process.condor_submit\nVARS ${s%.*} subject=\"$s\"\n" >> code/process.condor_dag
|
|
done
|
|
|
|
printf "JOB MERGE code/process.condor_submit-merge\n" >> code/process.condor_dag
|
|
printf "PARENT $(cd sourcedata/raw && echo sub*) CHILD MERGE" >> code/process.condor_dag
|
|
|
|
|
|
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
|
|
datalad save -m "HTCondor submission setup" code/ .gitignore
|
|
|
|
################################################################################
|
|
# HTCONDOR SETUP END
|
|
################################################################################
|
|
|
|
|
|
|
|
# cleanup - we have generated the job definitions, we do not need to keep a
|
|
# massive input dataset around. Having it around wastes resources and makes many
|
|
# git operations needlessly slow
|
|
datalad drop --what datasets --reckless kill -r -d sourcedata/raw
|
|
datalad drop --what datasets --reckless kill -r -d code/containers
|
|
|
|
# make sure the fully configured input & output datasets are available
|
|
# from the designated ria stores
|
|
datalad push --to ${PROJECT}_in
|
|
datalad push --to ${PROJECT}_out
|
|
|
|
# submit condor dag to run the sample
|
|
condor_submit_dag \
|
|
-include_env "USER","HOME","PATH","PYTHONPATH","LANG","PWD" \
|
|
code/process.condor_dag
|
|
|
|
# if we get here, we are happy
|
|
echo SUCCESS ${SAMPLE} is running
|