mpis-schedule

#!/bin/bash

. "${MPIS_UI}"


REQUIRED=( MPIS_OUTPUT_ROOT MPIS_USER_ID MPIS_STDOUT_EXTENSION MPIS_STDERR_EXTENSION MPIS_CORES)

for value in "${REQUIRED[@]}"
do
    if [[ -z "${!value// }" ]]
    then
        log "ERROR" "'$value' has not been specified"; exit 1
    fi
done

if [[ "$*" == *--clean* ]]
then
    rm -rfv "$MPIS_OUTPUT_ROOT"

    find . -name "*.mpiP"  -delete
    find . -name "*job.sh" -delete
    find . -name "core.*"  -delete

    enqueued="$( qstat | grep "$MPIS_USER_ID" | grep -oh "^[0-9]*.argo" | tr '\n' ' ' )"

    if [[ ! -z "${enqueued// }" ]]
    then
        log "MESSAGE" "Dequeuing jobs $enqueued"

        for job in $enqueued
        do
            qdel "$job"
        done
    fi

    exit 0
fi

if [ "$#" -lt 3 ]
then
    log "ERROR" "usage: $( basename "$0" ) [EXECUTABLE] [PROCESSES] [THREADS_PER_PROCESS]"; exit 1
fi

exe="$1"

if [ ! -x "$exe" ]
then
    log "ERROR" "'$1' is not executable"; exit 1
fi

if ! [[ "$2" =~ ^[1-9][0-9]*$ ]]
then
    log "ERROR" "The number of processes must be an integer"; exit 1
fi

PROCESSES="$2"

if ! [[ "$3" =~ ^[1-9][0-9]*$ ]]
then
    log "ERROR" "The number of threads must be an integer"; exit 1
fi

THREADS_PER_PROCESS="$3"

PPN="$(( $MPIS_CORES / $THREADS_PER_PROCESS ))"

if [ "$PPN" -le 0 ]
then
    log "ERROR" "The number of threads is greater than the maximum number of processes per node"; exit 1
fi

args="--mca mca_component_show_load_errors 0"

if [ "$(( $PROCESSES % $PPN ))" -eq 0 ]
then
    NODES="$(( $PROCESSES / $PPN ))"
else
    NODES="$(( ( $PROCESSES / $PPN ) + 1 ))"
fi

if [[ "$(( $PROCESSES * $THREADS_PER_PROCESS ))" -lt "$PPN" ]]
then
    args="$args -np $PROCESSES"
fi

job="$( basename $exe )"
job="${job%.*}"
job="${job}_${PROCESSES}_${THREADS_PER_PROCESS}_${MPIS_USER_ID}_job"

if [[ ! -z "${SCHEDULE_DIRECTORY}" ]]
then
    DIR="${MPIS_OUTPUT_ROOT}/${SCHEDULE_DIRECTORY}"
else
    DIR="${MPIS_OUTPUT_ROOT}/${PROCESSES}/$( date +"%d_%m_%Y" )/$( date +"%H_%M_%S" )"
fi

JOB=\
"#!/bin/bash

# Max VM size #
#PBS -l pvmem=2G

# Max Wall time #
#PBS -l walltime=00:01:00

# How many nodes and tasks per node #
#PBS -l nodes=${NODES}:ppn=${PPN}

# Which Queue to use, DO NOT CHANGE #
#PBS -q workq

# Job Name #
#PBS -N $job

# Streams #
#PBS -o ${DIR}/${job}.${MPIS_STDOUT_EXTENSION}
#PBS -e ${DIR}/${job}.${MPIS_STDERR_EXTENSION}

#Change Working directory to SUBMIT directory
cd \$PBS_O_WORKDIR

#OpenMP Threads #
# OMP_NUM_THREADS * ppn should be max 8 (the total number of node cores= 8).
# To use OpenMPI remember to include -fopenmp in compiler flags in order to activate OpenMP directives.
export OMP_NUM_THREADS=$THREADS_PER_PROCESS

# Run executable #
mpirun $args $exe
"

mkdir -p "${DIR}"

echo "$JOB" > "${job}.sh"

export MPIP="-f ${DIR}"

id="$( qsub "${job}.sh" )"

echo

if [[ ! -z "${id// }" ]]
then
    log "MESSAGE" "name='${job}', id='${id}', ps=${PROCESSES}, ts=${THREADS_PER_PROCESS}, ns=${NODES}, ppn=${PPN}"

    if [[ -z "${SCHEDULE_DIRECTORY}" ]]
    then
        echo -e "\n$( qstat )\n"
    fi
else
    log "ERROR" "Scheduling job '$job' has failed"
fi