Skip to content

Commit

Permalink
Updates to modules and cmake for regtests
Browse files Browse the repository at this point in the history
  • Loading branch information
Saeideh Banihashemi committed Oct 2, 2024
1 parent aedb49e commit bad181f
Show file tree
Hide file tree
Showing 4 changed files with 170 additions and 101 deletions.
3 changes: 2 additions & 1 deletion regtests/bin/matrix_cmake_datarmor
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`"
echo " export NETCDF_CONFIG=/home/datawork-wave/NETCDF2019/${COMP}/bin/nc-config" >> matrix.head
echo " export NetCDF_ROOT=/home/datawork-wave/NETCDF2019/${COMP}" >> matrix.head
echo " export METIS_PATH=/home/datawork-wave/PARMETIS2019/${COMP}" >> matrix.head
echo " export SCOTCH_PATH=/home/datawork-wave/LIB/SCOTCH/v7.0.3/${COMP}" >> matrix.head
echo " export WW3_PARCOMPN=4" >> matrix.head
echo " export G2_LIB4=/home/datawork-wave/NCEPLIBS/${COMP}/g2-3.4.5/lib64/libg2_4.a" >> matrix.head
echo " export BACIO_LIB4=/home/datawork-wave/NCEPLIBS/${COMP}/bacio-2.4.1/lib/libbacio_4.a" >> matrix.head
Expand All @@ -128,7 +129,7 @@ main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`"
export mpi='$MPI_LAUNCH'

# Compile option
opt="-f -N -S -T"
opt="-f -N -S -T -o both"

# Base run_test command line
export rtst="./bin/run_cmake_test $opt"
Expand Down
139 changes: 94 additions & 45 deletions regtests/bin/matrix_cmake_ncep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,65 +22,109 @@ usage ()
{
cat 2>&1 << EOF
Usage: $myname model_dir
Usage: $myname model_dir compiler
Required:
model_dir : path to model dir of WW3 source
Optional:
compiler : intel (default) or gnu
EOF
}


# Get required arguments
if [ ! $# = 0 ]
then
main_dir="$1" ; shift
if [ ! $# = 0 ]
then
compiler="$1"; shift
else
compiler='intel'
fi
else
usage
exit 1
fi



# Convert main_dir to absolute path
main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`"

# Module Versions from HPC-Stack that are common for all platforms
modnetcdf='netcdf/4.7.4'
modjasper='jasper/2.0.25'
modzlib='zlib/1.2.11'
# Module Versions from spack-stack that are common for all platforms
modnetcdfc='netcdf-c/4.9.2'
modnetcdff='netcdf-fortran/4.6.1'
modjasper='jasper/2.0.32'
modzlib='zlib/1.2.13'
modpng='libpng/1.6.37'
modhdf5='hdf5/1.10.6'
modhdf5='hdf5/1.14.0'
modbacio='bacio/2.4.1'
modg2='g2/3.4.5'
modw3emc='w3emc/2.9.2'
modesmf='esmf/8.3.0b09'
modw3emc='w3emc/2.10.0'
modesmf='esmf/8.6.0'
modscotch='scotch/7.0.4'

# Set batchq queue, choose modules and other custom variables to fit system and
# to define headers etc (default to original version if empty)
ishera=`hostname | grep hfe`
isorion=`hostname | grep Orion`
isorion=`hostname | grep orion`
ishercules=`hostname | grep hercules`
if [ $ishera ]
then
# If no other h, assuming Hera
batchq='slurm'
basemodcomp='intel/2022.1.2'
basemodmpi='impi/2022.1.2'
hpcstackpath='/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack'
hpcstackversion='hpc/1.2.0'
modcomp='hpc-intel/2022.1.2'
modmpi='hpc-impi/2022.1.2'
scotchpath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hpc-stack/scotch-v7.0.3/install'
metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hpc-stack/parmetis-4.0.3/install'
modcmake='cmake/3.20.1'
if [ $compiler = "intel" ]
then
spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core'
modcomp='stack-intel/2021.5.0'
modmpi='stack-intel-oneapi-mpi/2021.5.1'
metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
modcmake='cmake/3.23.1'
elif [ $compiler = "gnu" ]
then
spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core'
modcomp='stack-gcc/9.2.0'
modmpi='stack-openmpi/4.1.5'
metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/gnu/spack-stack/1.6.0/parmetis-4.0.3/install'
modcmake='cmake/3.23.1'
else
echo "Compiler $compiler not supported on hera"
exit 1
fi
elif [ $isorion ]
then
if [ $compiler = "intel" ]
then
batchq='slurm'
spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core'
modcomp='stack-intel/2021.9.0'
modmpi='stack-intel-oneapi-mpi/2021.9.0'
metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/orion/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
modcmake='cmake/3.23.1'
else
echo "Compiler $compiler not supported on orion"
exit 1
fi
elif [ $ishercules ]
then
batchq='slurm'
basemodcomp='intel/2022.1.2'
basemodmpi='impi/2022.1.2'
hpcstackpath='/work/noaa/epic-ps/role-epic-ps/hpc-stack/libs/intel-2022.1.2/modulefiles/stack'
hpcstackversion='hpc/1.2.0'
modcomp='hpc-intel/2022.1.2'
modmpi='hpc-impi/2022.1.2'
scotchpath='/work2/noaa/marine/mmasarik/waves/opt/hpc-stack/scotch-v7.0.3/install'
metispath='/work2/noaa/marine/mmasarik/waves/opt/hpc-stack/parmetis-4.0.3/install'
modcmake='cmake/3.22.1'
if [ $compiler = "intel" ]
then
spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
modcomp='stack-intel/2021.9.0'
modmpi='stack-intel-oneapi-mpi/2021.9.0'
metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/intel/spack-stack/1.6.0/parmetis-4.0.3/install'
modcmake='cmake/3.23.1'
elif [ $compiler = "gnu" ]
then
spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core'
spackstackpath2='/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles'
modcomp='stack-gcc/12.2.0'
modmpi='stack-mvapich2/2.3.7'
metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/gnu/spack-stack/1.6.0/parmetis-4.0.3/install'
modcmake='cmake/3.23.1'
else
echo "Compiler $compiler not supported on hercules"
exit 1
fi
else
batchq=
fi
Expand All @@ -96,7 +140,7 @@ EOF

# 1.a Computer/ user dependent set up

echo '#!/bin/sh --login' > matrix.head
echo '#!/bin/sh' > matrix.head
echo ' ' >> matrix.head
if [ $batchq = "slurm" ] && [ $isorion ]
then
Expand All @@ -111,10 +155,19 @@ EOF
echo '#SBATCH --exclusive' >> matrix.head
echo ' ' >> matrix.head
echo 'ulimit -s unlimited' >> matrix.head
echo 'ulimit -c 0' >> matrix.head
echo 'export KMP_STACKSIZE=2G' >> matrix.head
echo 'export FI_OFI_RXM_BUFFER_SIZE=128000' >> matrix.head
echo 'export FI_OFI_RXM_RX_SIZE=64000' >> matrix.head
elif [ $batchq = "slurm" ] && [ $ishercules ]
then
echo "#SBATCH -n ${np}" >> matrix.head
echo "##SBATCH --cpus-per-task=${nth}" >> matrix.head
echo '#SBATCH -q batch' >> matrix.head
echo '#SBATCH -t 08:00:00' >> matrix.head
echo '#SBATCH -A marine-cpu' >> matrix.head
echo '#SBATCH -J ww3_regtest' >> matrix.head
echo '#SBATCH -o matrix.out' >> matrix.head
echo '#SBATCH -p hercules' >> matrix.head
echo '#SBATCH --exclusive' >> matrix.head
echo ' ' >> matrix.head
echo 'ulimit -s unlimited' >> matrix.head
elif [ $batchq = "slurm" ]
then
echo "#SBATCH -n ${np}" >> matrix.head
Expand All @@ -139,29 +192,25 @@ EOF

# Netcdf, Parmetis and SCOTCH modules & variables
echo " module purge" >> matrix.head
echo " module load $modcmake" >> matrix.head
if [ ! -z $basemodcomp ]; then
echo " module load $basemodcomp" >> matrix.head
fi
if [ ! -z $basemodmpi ]; then
echo " module load $basemodmpi" >> matrix.head
echo " module use $spackstackpath" >> matrix.head
if [ ! -z $spackstackpath2 ]; then
echo " module use $spackstackpath2" >> matrix.head
fi
echo " module use $hpcstackpath" >> matrix.head
echo " module load $hpcstackversion" >> matrix.head
echo " module load $modcomp" >> matrix.head
echo " module load $modmpi" >> matrix.head
echo " module load $modcmake" >> matrix.head
echo " module load $modpng" >> matrix.head
echo " module load $modzlib" >> matrix.head
echo " module load $modjasper" >> matrix.head
echo " module load $modhdf5" >> matrix.head
echo " module load $modnetcdf" >> matrix.head
echo " module load $modnetcdfc" >> matrix.head
echo " module load $modnetcdff" >> matrix.head
echo " module load $modbacio" >> matrix.head
echo " module load $modg2" >> matrix.head
echo " module load $modw3emc" >> matrix.head
echo " module load $modesmf" >> matrix.head

echo " module load $modscotch" >> matrix.head
echo " export METIS_PATH=${metispath}" >> matrix.head
echo " export SCOTCH_PATH=${scotchpath}" >> matrix.head
echo " export path_build_root=$(dirname $main_dir)/regtests/buildmatrix" >> matrix.head
echo ' [[ -d ${path_build_root} ]] && rm -rf ${path_build_root}' >> matrix.head
echo ' '
Expand Down
Loading

0 comments on commit bad181f

Please sign in to comment.