diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..d7695f9 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,8 @@ +# Utility scripts + +This folder contains utility scripts that are used on the HPC to monitor the queues and start models on the HPC. + +*upload_download.sh* - This script will download and start 2 singulatiry containers (if not already started) and tail the output logs. +*model_launcher.sh* - This script will check if any models are needed to be run and start them. This will launch about 1 pipeline for every 10 waiting jobs. + +Both scripts require a files called secrets.sh to be in the same folder. If it does not exist, it will print a message if it does not exist. diff --git a/scripts/download_upload.sh b/scripts/download_upload.sh new file mode 100644 index 0000000..970a80b --- /dev/null +++ b/scripts/download_upload.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +if [ ! -e secrets.sh ]; then + cat < 5 ? 5 : ${NEEDED} )) + if [ $RUNNING -lt $NEEDED ]; then + echo -en "${SKIP}Starting another pipeline for $queue. " + SKIP="" + sbatch --job-name ${queue} "/projects/bbym/shared/CDR_processing/pipeline_processing_003/${queue}_launcher.bash" + fi + LOG="${LOG} [$queue : Running=$RUNNING jobs=$JOBS need=$NEEDED] " + done + echo -ne "${LOG} \r" + sleep 1 +done