Update Derecho Port

Update inteloneapi, cray, intel, nvhpc modules/compiler. Changes answers for inteloneapi and cray. Switch default Derecho queue from main to develop to support quicker turn around and lower costs. Develop is a shared queue.
CICE-Consortium · Sep 26, 2024 · 1849b38 · 1849b38
1 parent 05ac0ec
commit 1849b38
Show file tree

Hide file tree

Showing 8 changed files with 26 additions and 25 deletions.
diff --git a/configuration/scripts/icepack.batch.csh b/configuration/scripts/icepack.batch.csh
@@ -55,7 +55,7 @@ cat >> ${jobfile} << EOFB
 #PBS -l job_priority=regular
 #PBS -N ${ICE_CASENAME}
 #PBS -A ${acct}
-#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
+#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=5GB
 #PBS -l walltime=${ICE_RUNLENGTH}
 #PBS -j oe
 #PBS -W umask=022

diff --git a/configuration/scripts/machines/Macros.derecho_inteloneapi b/configuration/scripts/machines/Macros.derecho_inteloneapi
@@ -12,11 +12,12 @@ FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -trace
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)
-  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+# -check uninit is needed on the ld step but it still throws errors in 2023.* and 2024.0.*, likely compiler bug
+  FFLAGS     += -O0 -g  -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
 #  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
 #  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
 else
-  FFLAGS     += -O2
+  FFLAGS     += -O1
 endif
 
 SCC   := icx

diff --git a/configuration/scripts/machines/env.derecho_cray b/configuration/scripts/machines/env.derecho_cray
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load cce/15.0.1
+module load cce/16.0.1
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME cray
-setenv ICE_MACHINE_ENVINFO "cce 15.0.1, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "Cray clang/fortran cce 16.0.1, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_gnu b/configuration/scripts/machines/env.derecho_gnu
@@ -45,7 +45,7 @@ setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_intel b/configuration/scripts/machines/env.derecho_intel
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load intel/2023.0.0
+module load intel/2023.2.1
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME intel
-setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "ifort 2021.10.0 20230609, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_intelclassic b/configuration/scripts/machines/env.derecho_intelclassic
@@ -45,7 +45,7 @@ setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_inteloneapi b/configuration/scripts/machines/env.derecho_inteloneapi
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load intel-oneapi/2023.0.0
+module load intel-oneapi/2023.2.1
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME inteloneapi
-setenv ICE_MACHINE_ENVINFO "ifx 2023.0.0 20221201, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "ifx 2023.2.0 20230721, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.derecho_nvhpc b/configuration/scripts/machines/env.derecho_nvhpc
@@ -10,14 +10,14 @@ if ("$inp" != "-nomodules") then
 source ${MODULESHOME}/init/csh
 
 module --force purge
-module load ncarenv/23.06
+module load ncarenv/23.09
 module load craype
-module load nvhpc/23.5
+module load nvhpc/23.7
 module load ncarcompilers
 #module load cray-mpich/8.1.25
 #module load hdf5/1.12.2
 module load netcdf/4.9.2
-module load cray-libsci/23.02.1.1
+module load cray-libsci/23.09.1.1
 
 # For perftools with mpiexec
 # module load perftools-base
@@ -38,14 +38,14 @@ setenv OMP_STACKSIZE 64M
 setenv ICE_MACHINE_MACHNAME derecho
 setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
 setenv ICE_MACHINE_ENVNAME nvhpc
-setenv ICE_MACHINE_ENVINFO "nvc 23.5-0, netcdf4.9.2"
+setenv ICE_MACHINE_ENVINFO "nvfortran 23.7-0, netcdf4.9.2"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
 setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
 setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub"
 setenv ICE_MACHINE_ACCT P00000000
-setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_QUEUE "develop"
 setenv ICE_MACHINE_TPNODE 128
 setenv ICE_MACHINE_BLDTHRDS 1
 setenv ICE_MACHINE_QSTAT "qstat "