From 892b693ba49b37c23f08cc8e18550ba72e108762 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:05:22 -0500 Subject: [PATCH] Upgrade libraries to spack-stack/1.5.0. (#866) Update libraries to match those used by UFS. Done for Hera, Jet and Orion. Fixes #859. --- ci/spack.yaml | 8 +-- cmake/mpiexec.jet | 2 +- modulefiles/build.hera.gnu.lua | 51 ++++++++++--------- modulefiles/build.hera.intel.lua | 33 ++++++------- modulefiles/build.jet.intel.lua | 70 +++++++++++++-------------- modulefiles/build.orion.intel.lua | 38 +++++++-------- reg_tests/chgres_cube/driver.orion.sh | 2 +- reg_tests/cpld_gridgen/rt.sh | 2 +- util/gdas_init/driver.jet.sh | 3 ++ 9 files changed, 104 insertions(+), 105 deletions(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index 0549a8345..8711ee2cc 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -4,14 +4,14 @@ spack: all: compiler: [intel, gcc@10:10, apple-clang@14] specs: - - netcdf-c@4.7.4 - - netcdf-fortran@4.5.3 + - netcdf-c@4.9.2 + - netcdf-fortran@4.6.0 - bacio@2.4.1 - g2@3.4.5 - - ip@3.3.3 + - ip@4.4.0 precision=d - nemsio@2.5.4 - sp@2.3.3 - - w3emc@2.9.2 + - w3emc@2.10.0 - sfcio@1.4.1 - sigio@2.3.2 - nccmp@1.9.0.1 diff --git a/cmake/mpiexec.jet b/cmake/mpiexec.jet index a59fbcbd9..b5813d4fe 100755 --- a/cmake/mpiexec.jet +++ b/cmake/mpiexec.jet @@ -7,7 +7,7 @@ # ACCOUNT=nesdis-rdo2 -QOS=debug +QOS=batch PARTITION=xjet NP=$1 diff --git a/modulefiles/build.hera.gnu.lua b/modulefiles/build.hera.gnu.lua index d3bd08e3d..1c5d0835a 100644 --- a/modulefiles/build.hera.gnu.lua +++ b/modulefiles/build.hera.gnu.lua @@ -2,31 +2,21 @@ help([[ Load environment to compile UFS_UTILS on Hera using Gnu ]]) -cmake_ver=os.getenv("cmake_ver") or "3.20.1" -load(pathJoin("cmake", cmake_ver)) - hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack") - -gnu_ver=os.getenv("gnu_ver") or "9.2" -load(pathJoin("gnu", gnu_ver)) +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") +-- For openmpi: +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -hpc_ver=os.getenv("hpc_ver") or "1.2.0" -load(pathJoin("hpc", hpc_ver)) +stack_gcc_ver=os.getenv("stack_gcc_ver") or "9.2" +load(pathJoin("stack-gcc", gnu_ver)) -hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2" -load(pathJoin("hpc-gnu", hpc_gnu_ver)) +stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" +load(pathJoin("stack-openmpi", stack_openmpi_ver)) -hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2" -load(pathJoin("hpc-mpich", hpc_mpich_ver)) - -netcdf_ver=os.getenv("netcdf_ver") or "4.9.1" -load(pathJoin("netcdf", netcdf_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.4.1b07" -load(pathJoin("esmf", esmf_ver)) +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -34,7 +24,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.0.0" +ip_ver=os.getenv("ip_ver") or "4.3.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -43,7 +33,7 @@ load(pathJoin("nemsio", nemsio_ver)) sp_ver=os.getenv("sp_ver") or "2.3.3" load(pathJoin("sp", sp_ver)) -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" @@ -52,13 +42,22 @@ load(pathJoin("sfcio", sfcio_ver)) sigio_ver=os.getenv("sigio_ver") or "2.3.2" load(pathJoin("sigio", sigio_ver)) -nccmp_ver=os.getenv("nccmp_ver") or "1.9.1.0" +hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" +load(pathJoin("hdf5", hdf5_ver)) + +netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +load(pathJoin("netcdf-c", netcdf_c_ver)) + +netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) + +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -zlib_ver=os.getenv("zlib_ver") or "1.2.12" -load(pathJoin("zlib", zlib_ver)) +esmf_ver=os.getenv("esmf_ver") or "8.4.2" +load(pathJoin("esmf", esmf_ver)) -png_ver=os.getenv("png_ver") or "1.6.37" -load(pathJoin("libpng", png_ver)) +nco_ver=os.getenv("nco_ver") or "5.0.6" +load(pathJoin("nco", nco_ver)) whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.hera.intel.lua b/modulefiles/build.hera.intel.lua index bf92b2634..016c35ff3 100644 --- a/modulefiles/build.hera.intel.lua +++ b/modulefiles/build.hera.intel.lua @@ -2,22 +2,19 @@ help([[ Load environment to compile UFS_UTILS on Hera using Intel ]]) -cmake_ver=os.getenv("cmake_ver") or "3.20.1" -load(pathJoin("cmake", cmake_ver)) - hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") -hpc_ver=os.getenv("hpc_ver") or "1.2.0" -load(pathJoin("hpc", hpc_ver)) +stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +load(pathJoin("stack-intel", stack_intel_ver)) -hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -load(pathJoin("hpc-intel", hpc_intel_ver)) +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -load(pathJoin("hpc-impi", hpc_impi_ver)) +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -25,7 +22,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.0.0" +ip_ver=os.getenv("ip_ver") or "4.3.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -34,7 +31,7 @@ load(pathJoin("nemsio", nemsio_ver)) sp_ver=os.getenv("sp_ver") or "2.3.3" load(pathJoin("sp", sp_ver)) -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" @@ -43,22 +40,22 @@ load(pathJoin("sfcio", sfcio_ver)) sigio_ver=os.getenv("sigio_ver") or "2.3.2" load(pathJoin("sigio", sigio_ver)) -zlib_ver=os.getenv("zlib_ver") or "1.2.12" +zlib_ver=os.getenv("zlib_ver") or "1.2.13" load(pathJoin("zlib", zlib_ver)) png_ver=os.getenv("png_ver") or "1.6.37" load(pathJoin("libpng", png_ver)) -hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" -load(pathJoin("hdf5", hdf5_ver)) +netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +load(pathJoin("netcdf-c", netcdf_c_ver)) -netcdf_ver=os.getenv("netcdf_ver") or "4.9.1" -load(pathJoin("netcdf", netcdf_ver)) +netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.4.1" +esmf_ver=os.getenv("esmf_ver") or "8.4.2" load(pathJoin("esmf", esmf_ver)) nco_ver=os.getenv("nco_ver") or "5.0.6" diff --git a/modulefiles/build.jet.intel.lua b/modulefiles/build.jet.intel.lua index e6ce6a54a..106d8c9b3 100644 --- a/modulefiles/build.jet.intel.lua +++ b/modulefiles/build.jet.intel.lua @@ -1,64 +1,64 @@ help([[ -Load environment to compile UFS_UTILS on Jet +Load environment to compile UFS_UTILS on Jet using Intel ]]) -cmake_ver=os.getenv("cmake_ver") or "3.16.1" -load(pathJoin("cmake", cmake_ver)) - hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") -hpc_ver=os.getenv("hpc_ver") or "1.2.0" -load(pathJoin("hpc", hpc_ver)) +stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +load(pathJoin("stack-intel", stack_intel_ver)) -hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -load(pathJoin("hpc-intel", hpc_intel_ver)) +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -impi_ver=os.getenv("impi_ver") or "2022.1.2" -load(pathJoin("hpc-impi", impi_ver)) - -hdf5_ver=os.getenv("hdf5_ver") or "1.10.6" -load(pathJoin("hdf5", hdf5_ver)) +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) -netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" -load(pathJoin("netcdf", netcdf_ver)) +bacio_ver=os.getenv("bacio_ver") or "2.4.1" +load(pathJoin("bacio", bacio_ver)) -nccmp_ver=os.getenv("nccmp_ver") or "1.8.9.0" -load(pathJoin("nccmp", nccmp_ver)) +g2_ver=os.getenv("g2_ver") or "3.4.5" +load(pathJoin("g2", g2_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.4.0b08" -load(pathJoin("esmf", esmf_ver)) +ip_ver=os.getenv("ip_ver") or "4.3.0" +load(pathJoin("ip", ip_ver)) -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" -load(pathJoin("w3emc", w3emc_ver)) +nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" +load(pathJoin("nemsio", nemsio_ver)) sp_ver=os.getenv("sp_ver") or "2.3.3" load(pathJoin("sp", sp_ver)) -ip_ver=os.getenv("ip_ver") or "4.0.0" -load(pathJoin("ip", ip_ver)) +w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" +load(pathJoin("w3emc", w3emc_ver)) -bacio_ver=os.getenv("bacio_ver") or "2.4.1" -load(pathJoin("bacio", bacio_ver)) +sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +load(pathJoin("sfcio", sfcio_ver)) sigio_ver=os.getenv("sigio_ver") or "2.3.2" load(pathJoin("sigio", sigio_ver)) -sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" -load(pathJoin("sfcio", sfcio_ver)) +zlib_ver=os.getenv("zlib_ver") or "1.2.13" +load(pathJoin("zlib", zlib_ver)) -nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" -load(pathJoin("nemsio", nemsio_ver)) +png_ver=os.getenv("png_ver") or "1.6.37" +load(pathJoin("libpng", png_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" -load(pathJoin("g2", g2_ver)) +netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +load(pathJoin("netcdf-c", netcdf_c_ver)) + +netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) -prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("prod_util", prod_util_ver)) +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" +load(pathJoin("nccmp", nccmp_ver)) + +esmf_ver=os.getenv("esmf_ver") or "8.4.2" +load(pathJoin("esmf", esmf_ver)) -nco_ver=os.getenv("nco_ver") or "4.9.3" +nco_ver=os.getenv("nco_ver") or "5.0.6" load(pathJoin("nco", nco_ver)) whatis("Description: UFS_UTILS build environment") diff --git a/modulefiles/build.orion.intel.lua b/modulefiles/build.orion.intel.lua index 18fe99840..c7f8a5cae 100644 --- a/modulefiles/build.orion.intel.lua +++ b/modulefiles/build.orion.intel.lua @@ -1,20 +1,17 @@ help([[ -Load environment to compile UFS_UTILS on Orion +Load environment to compile UFS_UTILS on Orion using Intel ]]) -cmake_ver=os.getenv("cmake_ver") or "3.22.1" -load(pathJoin("cmake", cmake_ver)) - -prepend_path("MODULEPATH", "/work/noaa/epic-ps/role-epic-ps/hpc-stack/libs/intel-2022.1.2_ncdf492/modulefiles/stack") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") -hpc_ver=os.getenv("hpc_ver") or "1.2.0" -load(pathJoin("hpc", hpc_ver)) +stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +load(pathJoin("stack-intel", stack_intel_ver)) -hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -load(pathJoin("hpc-intel", hpc_intel_ver)) +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -load(pathJoin("hpc-impi", hpc_impi_ver)) +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -22,7 +19,7 @@ load(pathJoin("bacio", bacio_ver)) g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) -ip_ver=os.getenv("ip_ver") or "4.0.0" +ip_ver=os.getenv("ip_ver") or "4.3.0" load(pathJoin("ip", ip_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -31,7 +28,7 @@ load(pathJoin("nemsio", nemsio_ver)) sp_ver=os.getenv("sp_ver") or "2.3.3" load(pathJoin("sp", sp_ver)) -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" load(pathJoin("w3emc", w3emc_ver)) sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" @@ -40,16 +37,19 @@ load(pathJoin("sfcio", sfcio_ver)) sigio_ver=os.getenv("sigio_ver") or "2.3.2" load(pathJoin("sigio", sigio_ver)) -zlib_ver=os.getenv("zlib_ver") or "1.2.11" +zlib_ver=os.getenv("zlib_ver") or "1.2.13" load(pathJoin("zlib", zlib_ver)) -hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" -load(pathJoin("hdf5", hdf5_ver)) +png_ver=os.getenv("png_ver") or "1.6.37" +load(pathJoin("libpng", png_ver)) + +netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +load(pathJoin("netcdf-c", netcdf_c_ver)) -netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" -load(pathJoin("netcdf", netcdf_ver)) +netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) -nccmp_ver=os.getenv("nccmp_ver") or "1.8.9.0" +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) esmf_ver=os.getenv("esmf_ver") or "8.4.2" diff --git a/reg_tests/chgres_cube/driver.orion.sh b/reg_tests/chgres_cube/driver.orion.sh index 66de8b85b..011dd0fba 100755 --- a/reg_tests/chgres_cube/driver.orion.sh +++ b/reg_tests/chgres_cube/driver.orion.sh @@ -102,7 +102,7 @@ TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 --mem=75G -t 0:15:00 -A LOG_FILE4=${LOG_FILE}04 export OMP_NUM_THREADS=6 # needs to match cpus-per-task -TEST4=$(sbatch --parsable --ntasks-per-node=3 --cpus-per-task=6 --nodes=2 --mem=50G -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.sigio \ +TEST4=$(sbatch --parsable --ntasks-per-node=3 --cpus-per-task=6 --nodes=2 --mem=75G -t 0:20:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.sigio \ --open-mode=append -o $LOG_FILE4 -e $LOG_FILE4 ./c96.gfs.sigio.sh) #----------------------------------------------------------------------------- diff --git a/reg_tests/cpld_gridgen/rt.sh b/reg_tests/cpld_gridgen/rt.sh index 49f353963..2b5614a92 100755 --- a/reg_tests/cpld_gridgen/rt.sh +++ b/reg_tests/cpld_gridgen/rt.sh @@ -108,7 +108,7 @@ TESTS_FILE="$PATHRT/rt.conf" export TEST_NAME= # for C3072 on hera, use WLCLK=60 and MEM="--exclusive" -WLCLK_dflt=15 +WLCLK_dflt=20 export WLCLK=$WLCLK_dflt MEM_dflt="--mem=12g" export MEM=$MEM_dflt diff --git a/util/gdas_init/driver.jet.sh b/util/gdas_init/driver.jet.sh index 3df8cd64c..cc123c154 100755 --- a/util/gdas_init/driver.jet.sh +++ b/util/gdas_init/driver.jet.sh @@ -20,6 +20,9 @@ PROJECT_CODE=hfv3gfs QUEUE=batch PARTITION=xjet +# Needed for NDATE utility +module load prod_util/1.2.2 + source config if [ $EXTRACT_DATA == yes ]; then