f.hoffstaedter/bootstrap_MRIQC
Watch
1
0
Fork
You've already forked bootstrap_MRIQC
 0
Code Pull requests Packages Activity
bootstrap_MRIQC/bootstrap_MRIQC_TRR-TEMPLATE_JSC.sh

519 lines
17 KiB
Shell
Raw Permalink Blame History

set -e -u
# project name space
PROJECT="mriqc"
# define SAMPLE to be processed
SAMPLE="q01-*"
# define the output store to push all results to
output_store="https://hub.trr379.de/f.hoffstaedter"
# define and input ria-store only to clone from
inputstore="/p/project1/qaprep/ria-trr379_inputstore"
# add spec to name for debugging runs
spec=""
# define clonable location for BIDS inputs
raw_store="https://hub.trr379.de/${SAMPLE}/q01-bids-dataset"
raw_ds=""
# define temporal working directory for job execution
temporary_store="/dev/shm"
# define location for annex cache to store container
annex_cache="/p/scratch/icei-hbp-00000000004/STAGE/annex-cache"
# how much to parallelize your single subject functional jobs
parallelize=2
RAM=6000
### parallelization of subjects on compute nodes for node wise job submission
nodesubs=128
# mriqc container from the Repronim container datalad dataset
container_store="https://github.com/ReproNim/containers.git"
container_ds=""
container='bids-mriqc'
version='24.0.2'
##### don't change anything below if you are not sure what you want to do #####
# define ID for git commits (take from local user configuration)
git_name="$(git config user.name)"
git_email="$(git config user.email)"
# define and create the input ria-store only to clone from
input_store="ria+file://${inputstore}"
mkdir -p ${inputstore}
# all results a tracked in a single output dataset
# create a fresh one
# job submission will take place from a checkout of this dataset, but no
# results will be pushed into it
# Create a source dataset with all analysis components as an analysis access
# point.
datalad create -c yoda ${SAMPLE}-${PROJECT}${version}${spec}
cd ${SAMPLE}-${PROJECT}${version}${spec}
# add html in unlocked mode and keep tsvs and datalset_description in git
git annex config --set annex.addunlocked 'include=*.html'
echo ".bidsignore annex.largefiles=nothing
dataset_description.json annex.largefiles=nothing
group*.tsv annex.largefiles=nothing
mclf_data-unseen_pred.csv annex.largefiles=nothing" >> .gitattributes
datalad save -m "keep .bidsignore, dataset_description.json & group results in git"
# the actual compute job specification
cat > README.md << EOT
# ${SAMPLE} - MRIQC derivatives
This dataset is a BIDS Derivatives dataset resulting from running MRIQC ${version} on [${SAMPLE}](https://openneuro.org/datasets/${SAMPLE}).
## Methods
MRIQC documentation can be found on https://mriqc.readthedocs.io/.
This dataset was created using [BOOTSTRAP - a high-throughput container workflow for MRIQC](https://cerebra.fz-juelich.de/f.hoffstaedter/bootstrap_MRIQC) implementing the [FAIRly Big Workflow](https://doi.org/10.1038/s41597-022-01163-2) using [Datalad](https://www.datalad.org/) and [Singularity containers](https://apptainer.org/) for full reproducibility and provenance capture including resource consumption via [con-duct](https://github.com/con/duct).
## Acknowledgements
The authors from the Institute of Neuroscience and Medicine (INM-7) at the Forschungszentrum Jülich, Germany gratefully acknowledge computing time on the supercomputer JURECA[1] at Forschungszentrum Jülich under grant no. 65913.
[1] Jülich Supercomputing Centre. (2021). JURECA: Data Centric and Booster Modules implementing the Modular Supercomputing Architecture at Jülich Supercomputing Centre Journal of large-scale research facilities, 7, A182. http://dx.doi.org/10.17815/jlsrf-7-182
EOT
datalad save -m "add README with Acknowledgements"
# clone the container-dataset as a subdataset.
datalad clone -d . "${container_store}" code/containers
# configure a custom container call to satisfy the needs of this analysis
datalad run -m "Freeze mriqc container version" \
code/containers/scripts/freeze_versions --save-dataset=. ${container}=${version}
datalad run -m "Setup elderly mriqc version with classification support" \
code/containers/scripts/freeze_versions --save-dataset=. ${container}:${container}-clf=0.15.1
datalad run -m "Remove datalad-get option and do singularity exec instead of run" \
"sed -i -e 's, --no-datalad-get,,g' .datalad/config
sed -i -e 's, run, exec,g' .datalad/config"
# create dedicated input and output locations. Results will be pushed into the
# output sibling, and the analysis will start with a clone from the input
# sibling.
datalad create-sibling-ria -s ${PROJECT}_in "${input_store}" \
--alias ${SAMPLE}-${PROJECT}${version}${spec} --new-store-ok --storage-sibling off
git remote add ${PROJECT}_out ${output_store}
git push ${PROJECT}_out
# register the input dataset, a superdataset comprising all participants
datalad clone -d . "${raw_store}${raw_ds}" sourcedata/raw
# amend the previous commit with a nicer commit message
git commit --amend -m "Register ${SAMPLE} BIDS dataset as sourcedata"
# the actual compute job specification
cat > code/participant_job << EOT
#!/bin/bash
# the job assumes that it is a good idea to run everything in PWD
# the job manager should make sure that is true
# fail whenever something is fishy, use -x to get verbose logfiles
set -e -u -x
dssource="\$1"
pushgitremote="\$2"
subid="\$3"
export DUCT_OUTPUT_PREFIX="duct-logs/\${subid}_{datetime_filesafe}-{pid}."
# get the analysis dataset, which includes the inputs as well
# importantly, we do not clone from the lcoation that we want to push the
# results too, in order to avoid too many jobs blocking access to
# the same location and creating a throughput bottleneck
datalad clone "\${dssource}" ds
# all following actions are performed in the context of the superdataset
cd ds
# in order to avoid accumulation temporary git-annex availability information
# and to avoid a syncronization bottleneck by having to consolidate the
# git-annex branch across jobs, we will only push the main tracking branch
# back to the output store (plus the actual file content). Final availability
# information can be establish via an eventual "git-annex fsck -f ${PROJECT}_out-storage".
# this remote is never fetched, it accumulates a larger number of branches
# and we want to avoid progressive slowdown. Instead we only ever push
# a unique branch per each job (subject AND process specific name)
git remote add outputstore "\$pushgitremote"
# all results of this job will be put into a dedicated branch
git checkout -b "job_\${JOBID}"
# we pull down the input subject manually in order to discover relevant
# files. We do this outside the recorded call, because on a potential
# re-run we want to be able to do fine-grained recomputing of individual
# outputs. The recorded calls will have specific paths that will enable
# recomputation outside the scope of the original Condor setup
datalad get -n "sourcedata/raw/"
# setup annex cache for container dataset
datalad get -n "code/containers"
git -C code/containers remote add cache ${annex_cache}
git -C code/containers config remote.cache.annex-speculate-present true
git -C code/containers config remote.cache.annex-cost 10
git -C code/containers config remote.cache.annex-pull false
git -C code/containers config remote.cache.annex-push false
git -C code/containers config remote.cache.fetch do-not-fetch-from-this-remote:
# the meat of the matter
# look for T1w files in the input data for the given participant
# it is critical for reproducibility that the command given to
# "containers-run" does not rely on any property of the immediate
# computational environment (env vars, services, etc)
# do T1w first
datalad containers-run \\
-m "Compute MRIQC for T1w of \${subid}" \\
-n ${container} \\
-i sourcedata/raw/\${subid} \\
-i sourcedata/raw/dataset_description.json \\
mriqc sourcedata/raw mriqc participant \\
--participant-label \$subid \\
--modalities T1w \\
--no-datalad-get \\
--no-sub \\
--verbose \\
--nprocs ${nthreads} \\
--omp-nthreads ${nthreads} \\
--mem $mb_RAM \\
--work-dir tmp \\
--float32 \\
--verbose-reports || true
# do bold next
datalad containers-run \\
-m "Compute MRIQC for BOLD of \${subid}" \\
-n ${container} \\
-i sourcedata/raw/\${subid} \\
-i sourcedata/raw/dataset_description.json \\
mriqc sourcedata/raw mriqc participant \\
--participant-label \$subid \\
--modalities bold \\
--no-datalad-get \\
--no-sub \\
--verbose \\
--nprocs ${nthreads} \\
--omp-nthreads ${nthreads} \\
--mem $mb_RAM \\
--work-dir tmp \\
--float32 \\
--min-bold-length 10 \\
--verbose-reports || true
# then run dwi seperately and continue after error
datalad containers-run \\
-m "Compute MRIQC for DWI of \${subid}" \\
-n ${container} \\
-i sourcedata/raw/\${subid} \\
-i sourcedata/raw/dataset_description.json \\
mriqc sourcedata/raw mriqc participant \\
--participant-label \$subid \\
--modalities dwi \\
--no-datalad-get \\
--no-sub \\
--verbose \\
--nprocs ${nthreads} \\
--omp-nthreads ${nthreads} \\
--mem $mb_RAM \\
--work-dir tmp \\
--float32 \\
--min-dwi-length 6 \\
--verbose-reports || true
# file content first -- does not need a lock, no interaction with Git
git annex copy -t outputstore
# and the output branch
flock --verbose \$DSLOCKFILE git push outputstore
echo "SUBJECT JOB COMPLETED"
# job handler should clean up workspace
EOT
chmod +x code/participant_job
datalad save -m "Participant compute job implementation" code/participant_job
cat > code/process.sub << EOT
#!/bin/bash
subid="\$1"
executable=\$(pwd)/code/participant_job
# the job expects these environment variables for labeling and synchronization
# - JOBID: subject AND process specific ID to make a branch name from
# (must be unique across all (even multiple) submissions)
# including the cluster ID will enable sorting multiple computing attempts
# - DSLOCKFILE: lock (must be accessible from all compute jobs) to synchronize
# write access to the output dataset
# - DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__...:
# (additional) locations for datalad to locate relevant subdatasets, in case
# a configured URL is outdated
# - GIT_AUTHOR_...: Identity information used to save dataset changes in compute
# jobs
export JOBID=\${subid} \\
DSLOCKFILE=\$(pwd)/.condor_datalad_lock \\
GIT_AUTHOR_NAME='${git_name}' \\
GIT_AUTHOR_EMAIL='${git_email}' \\
REPRONIM_USE_DUCT=1
# essential args for "participant_job"
# 1: where to clone the analysis dataset
# 2: location to push the result git branch to. The "ria+" prefix is stripped.
# 3: ID of the subject to process
arguments="${input_store}#$(datalad -f '{infos[dataset][id]}' wtf -S dataset) \\
$(git remote get-url --push ${PROJECT}_out) \\
\${subid} \\
"
mkdir -p ${temporary_store}/tmp_\${subid:4}
cd ${temporary_store}/tmp_\${subid:4}
\${executable} \${arguments} \\
> $(pwd)/logs/\${subid}.out \\
2> $(pwd)/logs/\${subid}.err
chmod +w -R ${temporary_store}/tmp_\${subid:4} && \
rm -rf ${temporary_store}/tmp_\${subid:4}
EOT
chmod +x code/process.sub
datalad save -m "individual job submission" code/process.sub
cat > code/results.merger << EOT
#!/bin/bash
# fail whenever something is fishy, use -x to get verbose logfiles
set -e -x
# finalize FAIRly Big Workflow dataset
dssource="\$1"
if [[ ! -z \${dssource} ]]; then
datalad clone "\${dssource}" ds
cd ds
else
datalad update -s ${PROJECT}_out
fi
# octopus merge job branches and move data to root
git merge -m "Merge results" \$(git branch -al | grep 'job_' | tr -d ' ')
git mv mriqc/.bidsignore mriqc/* .
datalad save -m "move data to dataset root"
# clean git annex branch
if [[ ! -z \${dssource} ]]; then
git annex fsck --fast
else
git annex fsck -f ${PROJECT}_out --fast
fi
# declare local data clone as dead
git annex dead here
# datalad push merged results
if [[ ! -z \${dssource} ]]; then
datalad push --data nothing
else
datalad push --data nothing --to ${PROJECT}_out
fi
export DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__102container="ria+file:///p/data1/inm7/containerstores#{id}" \
REPRONIM_USE_DUCT=1 \
DUCT_OUTPUT_PREFIX="logs/duct/mriqc-group-stats_{datetime_filesafe}-{pid}_"
# get mri input for MRIQC group stats
datalad get -n sourcedata/raw
datalad get sourcedata/raw/dataset_description.json \\
\$(find sourcedata/raw -maxdepth 3 -name anat) \\
\$(find sourcedata/raw -maxdepth 3 -name func) \\
\$(find sourcedata/raw -maxdepth 3 -name dwi)
# run mriqc group stats
datalad containers-run \\
-m "Compute MRIQC group stats" \\
-n bids-mriqc \\
--explicit \\
-o 'group*' \\
-o .bidsignore \\
mriqc sourcedata/raw . group \\
--no-datalad-get \\
--notrack \\
--verbose \\
--work-dir tmp
datalad drop --what datasets --reckless kill -r -d sourcedata/raw
# run mriqc quality classifier
datalad containers-run \\
-m "run MRIQC classifier" \\
-n bids-mriqc-clf \\
-i group_T1w.tsv \\
"mriqc_clf --load-classifier -X group_T1w.tsv; \\
mv mclf_run*data-unseen_pred.csv mclf_data-unseen_pred.csv"
if [[ ! -z \${dssource} ]]; then
datalad push --data nothing
else
datalad push --data nothing --to ${PROJECT}_out
fi
echo SUCCESS
EOT
chmod +x code/results.merger
datalad save -m "finalize dataset by merging results branches into master" code/results.merger
# create clean up script
cat > code/killall.sh << EOT
#!/bin/bash
#
# if setup went wrong, delete everything
TOKEN=\$1
[[ "\$(read -e -p 'Are you sure you want to delete everything? [yes_sure!/NO]> '; echo \$REPLY)" == yes_sure! ]] && echo KILLALLNOW || exit
# delete wrong input & output RIA stores + aliases
rm -rf $(git config --get remote.${PROJECT}_in.url)
rm -f ${inputstore}/alias/${SAMPLE}-${PROJECT}${version}
curl -X 'DELETE' \
"https://hub.trr379.de/api/v1/repos/f.hoffstaedter/${SAMPLE}-${PROJECT}${version}${spec}" \
-H \'accept: application/json\' \
-H \"Authorization: token \${TOKEN}\"
# remove faulty dataset
cd ..
datalad drop --what datasets --reckless kill -r -d ${SAMPLE}-${PROJECT}${spec}${version}
EOT
chmod +x code/killall.sh
datalad save -m "add clean up script" code/killall.sh
###############################################################################
# SLURM SETUP START - FIXME remove or adjust this according to your needs.
###############################################################################
# *!*!*!* GNUPARALLEL is necessary for node wise job allocation *!*!*!*
# makes sure that the jobs per node don't exceed RAM and wall clock time !!
# SLURM compute environment for the whole dataset
cat > code/submit.sbatch << EOT
#!/bin/bash -x
#SBATCH --account=qaprep
#SBATCH --mail-user=f.hoffstaedter@fz-juelich.de
#SBATCH --mail-type=END
#SBATCH --job-name=${PROJECT}_${spec}${SAMPLE}
#SBATCH --output=logs/${SAMPLE}_${PROJECT}-out.%j
#SBATCH --error=logs/${SAMPLE}_${PROJECT}-err.%j
#SBATCH --time=24:00:00
#SBATCH --partition=dc-cpu-bigmem
#SBATCH --nodes=1
parallel --delay 0.2 -a \$1 -j $nodesubs
wait
exit
EOT
# create job.call-file for all commands to call
# each subject is processed on RAMDISK in an own dataset
cat > code/slurm.submit << EOT
#!/bin/bash -x
#
# redundant input per subject
subid=\$1
# define DSLOCKFILE, DATALAD & GIT ENV for participant_job
export DSLOCKFILE=$(pwd)/.SLURM_datalad_lock \
DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__102container="ria+file:///p/data1/inm7/containerstores#{id}" \
GIT_AUTHOR_NAME=\$(git config user.name) \
GIT_AUTHOR_EMAIL=\$(git config user.email) \
JOBID=${SAMPLE}_\${subid:4}.\${SLURM_JOB_ID} \
REPRONIM_USE_DUCT=1
# use subject specific folder
mkdir ${temporary_store}/\${JOBID}
cd ${temporary_store}/\${JOBID}
# run things
$(pwd)/code/participant_job \
${input_store}#$(datalad -f '{infos[dataset][id]}' wtf -S dataset) \
$(git remote get-url --push ${PROJECT}_out) \
\${subid} \
>$(pwd)/logs/\${JOBID}.out \
2>$(pwd)/logs/\${JOBID}.err
cd ${temporary_store}/
chmod 777 -R ${temporary_store}/\${JOBID}
rm -fr ${temporary_store}/\${JOBID}
EOT
cat > code/slurm.submit_post << EOT
#!/bin/bash -x
export JOBID=${SAMPLE}_postpro.\${SLURM_JOB_ID} \
REPRONIM_USE_DUCT=1
# use subject specific folder
mkdir ${temporary_store}/\${JOBID}
cd ${temporary_store}/\${JOBID}
# run things
$(pwd)/code/results.merger \
$(git remote get-url --push ${PROJECT}_out) \
>$(pwd)/logs/\${JOBID}.out \
2>$(pwd)/logs/\${JOBID}.err
cd ${temporary_store}/
chmod 777 -R ${temporary_store}/\${JOBID}
rm -fr ${temporary_store}/\${JOBID}
EOT
chmod +x code/slurm.submit_post
for s in $(find sourcedata/raw -maxdepth 1 -name 'sub-*' -printf '%f\n'); do
printf "code/slurm.submit $s\n" >> code/${SAMPLE}_mriqc.jobs
done
mkdir logs
echo logs >> .gitignore
echo tmp >> .gitignore
echo .bids_db >> .gitignore
echo .SLURM_datalad_lock >> .gitignore
# create run scripts for slurm batches
cat > code/run-mriqc_JURECA.sh << EOT
sbatch code/submit.sbatch code/${SAMPLE}_mriqc.jobs
EOT
chmod +x code/run-mriqc_JURECA.sh
datalad save -m "SLURM submission setup" code/ .gitignore
################################################################################
# SLURM SETUP END
################################################################################
# cleanup - we have generated the job definitions, we do not need to keep a
# massive input dataset around. Having it around wastes resources and makes many
# git operations needlessly slow
datalad drop --what datasets --reckless kill -r -d sourcedata/raw
datalad drop --what datasets --reckless kill -r -d code/containers
# make sure the fully configured input & output datasets are available
# from the designated ria stores
datalad push --to ${PROJECT}_in
datalad push --to ${PROJECT}_out
# submit condor dag to run the sample
sbatch code/submit.sbatch code/${SAMPLE}_mriqc.jobs
# if we get here, we are happy
echo SUCCESS ${SAMPLE} is submitted to SLURM
View git blame Copy permalink