diff --git a/install.sh b/install.sh index 03e51f4..8837c43 100755 --- a/install.sh +++ b/install.sh @@ -30,10 +30,12 @@ fi echo "# install exec and config" rsync -hav --info=name2,stats0,flist0 $FLAGS \ ./snakemakeslurm \ + ./slurm-status.py \ ./cluster.slurm.cheaha.json \ $1 echo "# install modulefiles" rsync -hav --info=name2,stats0,flist0 $FLAGS \ + --exclude="*~" \ ./modulefiles/snakemakeslurm \ $1/../../modulefiles diff --git a/modulefiles/snakemakeslurm/5.2.4-3 b/modulefiles/snakemakeslurm/5.2.4-3 new file mode 100644 index 0000000..82d8133 --- /dev/null +++ b/modulefiles/snakemakeslurm/5.2.4-3 @@ -0,0 +1,43 @@ +#%Module1.0##################################################################### +## +## snakemakeslurm +## +## https://github.com/rusalkaguy/snakemake-slurm-module +## +################################################################################ +set components [ file split [ module-info name ] ] +set prog [ lindex $components 0 ] +set snakemake-version [ lindex [ split [ lindex $components 1 ] - ] 0 ] +set script_version [ lindex [ split [ lindex $components 1 ] - ] 1 ] +set version ${snakemake-version}-${script_version} +set modroot /share/apps/ngs-ccts/${prog}/${prog}-${version} +set url https://github.com/rusalkaguy/snakemake-slurm-module + +proc ModulesHelp { } { + global version modroot + puts stderr "\tsnakemakeslurm - have snakemake submit jobs via slurm to cheaha" + puts stderr "\n\tComponents $components\n" + puts stderr "\n\tVersion $version\n" + puts stderr "\n\tmodroot $modroot\n" +} +module-whatis "Loads snakemakeslurm environment." + +# load required modules +# make sure to load dvctools first; if loaded second, it loads an older GCC. +module load dvctools +# UAB-HPC old module naming convention +# #module load rc/snakemake/${snakemake-version} +module load snakemake/${snakemake-version}-foss-2018b-Python-3.6.6 +# snakemake 4.8.0 doesn't work with Anaconda3/5.3.0 +# snakemake 5.2.4 DOES work with Anaconda3/5.3.1 - +# update Anaconda3 to 2020.02 to work with RNA-Seq pipeline +module load Anaconda3/2020.02 +# only one version at a time +#conflict multiqc + +# +# +# Make the directories available +prepend-path PATH $modroot/ +#prepend-path MANPATH $modroot/share/man +#prepend-path PYTHONPATH odroot/lib/python2.7/site-packages diff --git a/slurm-status.py b/slurm-status.py index a712d84..a8ae124 100755 --- a/slurm-status.py +++ b/slurm-status.py @@ -11,7 +11,9 @@ STATUS_ATTEMPTS = 20 -jobid = sys.argv[1] +#jobid = sys.argv[1] # original from https://github.com/Snakemake-Profiles/slurm +# Locally, our snakemake --cluster-status passes the args ["Submitted", "batch", "job", JOBID]! +jobid = sys.argv[4] for i in range(STATUS_ATTEMPTS): diff --git a/snakemakeslurm b/snakemakeslurm index 8d85b3e..dddfcc2 100755 --- a/snakemakeslurm +++ b/snakemakeslurm @@ -95,6 +95,23 @@ fi # list in order, later files override earlier ones # +# global config lives with this script + +# +# check if we have a script to get "cause of death" from SLURM +# w/o this, if SLURM kills a job for over-time or over-mem, snakemake wont know +# +# See: https://snakemake.readthedocs.io/en/stable/tutorial/additional_features.html?highlight=--cluster-status%20#using-cluster-status +# +SLURM_STATUS_FLAG= +SLURM_STATUS_SCRIPT=$(dirname $0)/slurm-status.py +if [ -e "$SLURM_STATUS_SCRIPT" ]; then + if [ "$DEBUG" == "1" ]; then echo "# found $SLURM_STATUS_SCRIPT"; fi; + SLURM_STATUS_FLAG=" --cluster-status $SLURM_STATUS_SCRIPT" +else + echo "# WARNING: missing $SLURM_STATUS_SCRIPT " +fi + # global config lives with this script CCONFIGS="--cluster-config $(dirname $0)/cluster.slurm.cheaha.json" @@ -130,5 +147,6 @@ $EXEC snakemake \ --latency-wait 45 \ --jobs 999 \ $CCONFIGS \ + $SLURM_STATUS_FLAG \ --cluster "sbatch $SM_ARGS" \ $*