Merge pull request #50 from NorESMhub/login3-update

Betzy OS update

machines/betzy/config_batch.xml

@@ -8,7 +8,6 @@
     <directive> --ntasks={{ total_tasks }}</directive>
     <directive> --export=ALL</directive>
     <directive> --switches=1</directive>
-    <directive> --exclude=b3355,b3356,b3357,b3359,b3379,b3382,b3383</directive>
   </directives>
   <directives queue = "normal">
     <directive> --partition=normal</directive>
@@ -19,11 +18,11 @@
   </directives>
   <directives queue = "preproc">
     <directive> --partition=preproc</directive>
-    <directive> --mem=16G</directive>
+    <directive> --mem-per-cpu=1900M</directive>
   </directives>
-  <queues>
-    <queue walltimemax="00:59:00" nodemin="4" nodemax="1024" default="true">normal</queue>
-    <queue walltimemax="00:59:00" nodemin="1" nodemax="1" >preproc</queue>
-    <queue walltimemax="00:59:00" nodemin="1" nodemax="4" >devel</queue>
-  </queues>
+    <queues>
+      <queue walltimedef="00:59:00" walltimemax="48:00:00" nodemin="4" nodemax="1024" default="true">normal</queue>
+      <queue walltimemax="00:59:00" jobmin="1" jobmax="1" >preproc</queue>
+      <queue walltimemax="00:59:00" nodemin="1" nodemax="4" >devel</queue>
+    </queues>
 </batch_system>

machines/betzy/config_machines.xml

@@ -1,14 +1,14 @@
 <machine MACH="betzy">
   <DESC> BullSequana XH2000 AMD® Epyc™ "Rome" 2.2GHz, 128-way nodes, os is Linux, batch system is SLURM</DESC>
   <OS>LINUX</OS>
-  <COMPILERS>intel-oneapi,intel</COMPILERS>
+  <COMPILERS>intel</COMPILERS>
   <MPILIBS>openmpi,impi</MPILIBS>
   <CIME_OUTPUT_ROOT>/cluster/work/users/$USER/noresm</CIME_OUTPUT_ROOT>
   <DIN_LOC_ROOT>/cluster/shared/noresm/inputdata</DIN_LOC_ROOT>
   <DIN_LOC_ROOT_CLMFORC>/cluster/shared/noresm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
   <DOUT_S_ROOT>/cluster/work/users/$USER/archive/$CASE</DOUT_S_ROOT>
   <BASELINE_ROOT> /cluster/shared/noresm/noresm_baselines</BASELINE_ROOT>
-  <CCSM_CPRNC>/cluster/shared/noresm/tools/cprnc/cprnc</CCSM_CPRNC>
+  <CCSM_CPRNC>/cluster/shared/noresm/tools/cprnc-iompi-2022a/bin/cprnc</CCSM_CPRNC>
   <GMAKE_J>8</GMAKE_J>
   <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
   <SUPPORTED_BY>noresmCommunity</SUPPORTED_BY>
@@ -19,7 +19,7 @@
     <executable></executable>
   </mpirun>
   <mpirun mpilib="default">
-    <executable>srun</executable>
+    <executable>srun --kill-on-bad-exit --label</executable>
   </mpirun>
   <module_system type="module" allow_error="true">
     <init_path lang="perl">$ENV{LMOD_PKG}/init/perl</init_path>
@@ -30,38 +30,17 @@
     <cmd_path lang="python">$ENV{LMOD_PKG}/libexec/lmod python</cmd_path>
     <cmd_path lang="sh">module</cmd_path>
     <cmd_path lang="csh">module</cmd_path>
-    <modules compiler="intel-oneapi" mpilib='openmpi'>
+    <modules compiler="intel" mpilib='openmpi'>
       <command name="--force purge"></command>
       <command name="load">StdEnv</command>
-      <command name="load">git/2.41.0-GCCcore-12.3.0-nodocs</command>
       <command name="use">/cluster/shared/noresm/eb_mods/modules/all</command>
       <command name="load">ESMF/8.4.2-iomkl-2022a-ParallelIO-2.5.10</command>
       <command name="load">Python/3.11.3-GCCcore-12.3.0</command>
       <command name="load">CMake/3.26.3-GCCcore-12.3.0</command>
       <command name="load">ParMETIS/4.0.3-iompi-2022a</command>
-      <command name="load">XML-LibXML/2.0209-GCCcore-12.3.0</command>
-    </modules>
-    <modules compiler="intel" mpilib="openmpi">
-      <command name="--force purge"></command>
-      <command name="load">StdEnv</command>
-      <command name="load">git/2.41.0-GCCcore-12.3.0-nodocs</command>
-      <command name="use">/cluster/shared/noresm/eb_mods/modules/all</command>
-      <command name="load">ESMF/8.4.1-iomkl-2021b-ParallelIO-2.5.10</command>
-      <command name="load">CMake/3.21.1-GCCcore-11.2.0</command>
-      <command name="load">Python/3.9.6-GCCcore-11.2.0</command>
-      <command name="load">ParMETIS/4.0.3-iompi-2021b</command>
-      <command name="load">XML-LibXML/2.0209-GCCcore-12.3.0</command>
-    </modules>
-    <modules compiler="intel" mpilib="impi">
-      <command name="--force purge"></command>
-      <command name="load">StdEnv</command>
       <command name="load">git/2.41.0-GCCcore-12.3.0-nodocs</command>
-      <command name="use">/cluster/shared/noresm/eb_mods/modules/all</command>
-      <command name="load">ESMF/8.4.1-intel-2021b-ParallelIO-2.5.10</command>
-      <command name="load">CMake/3.21.1-GCCcore-11.2.0</command>
-      <command name="load">Python/3.9.6-GCCcore-11.2.0</command>
-      <command name="load">ParMETIS/4.0.3-iimpi-2021b</command>
       <command name="load">XML-LibXML/2.0209-GCCcore-12.3.0</command>
+      <command name="load">hpcx/2.20</command>
     </modules>
   </module_system>
   <environment_variables>

machines/betzy/intel-oneapi_betzy.cmake

@@ -10,7 +10,7 @@ if(DEFINED ENV{PIO})
   set(PIO_INCDIR "$ENV{PIO}/include")
 endif()
 
-string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
+#string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
 string(APPEND CPPDEFS " -DHAVE_GETTID")
 
 if (COMP_WAV STREQUAL ww3)

machines/betzy/intel_betzy.cmake

@@ -1,14 +1,23 @@
 set(PIO_FILESYSTEM_HINTS "lustre")
-
+string(APPEND CFLAGS " -qopt-report -march=core-avx2")
+string(APPEND FFLAGS " -qopt-report -march=core-avx2")
 set(NETCDF_C_PATH	$ENV{EBROOTNETCDF})
 set(NETCDF_FORTRAN_PATH $ENV{EBROOTNETCDFMINFORTRAN})
 set(PNETCDF_PATH	$ENV{PNETCDF})
 
+if(DEFINED ENV{PIO})
+  set(PIO_LIBDIR "$ENV{PIO}/lib")
+  set(PIO_INCDIR "$ENV{PIO}/include")
+endif()
+
+#string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
+string(APPEND CPPDEFS " -DHAVE_GETTID")
+
 if (COMP_WAV STREQUAL ww3)
   if (MPILIB STREQUAL openmpi)
      string(APPEND SLIBS "  -l:libparmetis.a -lmetis ")
   endif()
-endif()
+endif() 
 
 if (MPILIB STREQUAL impi)
  set(MPICC "mpiicc")