From d97f257acf0143209220b1a6903a7843f72ea191 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Thu, 22 Feb 2024 19:20:50 +1100 Subject: [PATCH 1/6] Adding a Fugaku system config (#118) * Adding a Fugaku system config * license * Update system_definition.yaml * adding site * fix code to increase content width of sphinx docs * Initial compilers for Fugaku * thread counts for fugaku * adding PJM scheduler instructions https://matsci.dev/blog/2023/fugaku/ * add updated table of system definitions with fugaku * renaming RIKEN to RCCS * Removing extra dir * initial draft for fugaku config (#142) * initial draft for fugaku config * Add integrator name * Removing extra dir --------- Co-authored-by: domke <673751-domke@users.noreply.gitlab.com> Co-authored-by: pearce8 * license * column width * license * updating table for docs --------- Co-authored-by: Stephanie Brink Co-authored-by: jdomke <28772296+jdomke@users.noreply.github.com> Co-authored-by: domke <673751-domke@users.noreply.gitlab.com> --- .../auxiliary_software_files/compilers.yaml | 136 ++++ .../auxiliary_software_files/packages.yaml | 586 ++++++++++++++++++ .../spack.yaml | 23 + .../system_definition.yaml | 32 + .../variables.yaml | 13 + docs/tables/current-system-definitions.csv | 40 +- experiments/amg2023/openmp/ramble.yaml | 2 +- 7 files changed, 811 insertions(+), 21 deletions(-) create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml new file mode 100644 index 00000000..25129f76 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml @@ -0,0 +1,136 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: fj@4.10.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.38/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.38/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.38/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.38/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.38/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.38/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.8.1 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.36/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.36/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.36/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.36/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.36/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.36/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.8.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.35/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.35/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.35/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.35/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.35/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.35/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.7.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.34/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.34/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.34/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.34/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.34/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.34/lib64 + extra_rpaths: [] +- compiler: + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + operating_system: rhel8 + target: aarch64 + modules: [] + environment: + unset: [] + extra_rpaths: [] + flags: {} + spec: gcc@8.5.0 +- compiler: + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + unset: [] + extra_rpaths: [] + flags: {} + spec: gcc@8.4.1 +- compiler: + spec: gcc@12.2.0 + paths: + cc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gcc + cxx: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin//g++ + f77: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gfortran + fc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@12.2.0 + paths: + cc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gcc + cxx: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/g++ + f77: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gfortran + fc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gfortran + flags: {} + operating_system: rhel8 + target: aarch64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml new file mode 100644 index 00000000..93118804 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml @@ -0,0 +1,586 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + all: + compiler: [fj, gcc] + providers: + mpi: [fujitsu-mpi, openmpi, mpich] + blas: [fujitsu-ssl2, openblas] + lapack: [fujitsu-ssl2, openblas] + scalapack: [fujitsu-ssl2, netlib-scalapack] + fftw-api: [fujitsu-fftw, fftw, rist-fftw] + permissions: + write: group + htslib: + version: [1.12] + python: + externals: + - spec: "python@3.10.8%fj +ssl arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/python-3.10.8-7q66snjvhvy7im57hncbgpirmddrb5sk + - spec: "python@3.10.8%fj +ssl+tkinter arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/python-3.10.8-5q3ncyl2my7oomopsmukduqo36u6pnkg + - spec: "python@3.10.8%gcc +ssl arch=linux-rhel8-cascadelake" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-cascadelake/gcc-12.2.0/python-3.10.8-yt6afcnywa36aebxovs3ldscknyzlva3 + - spec: "python@3.10.8%gcc +ssl+tkinter arch=linux-rhel8-cascadelake" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-cascadelake/gcc-12.2.0/python-3.10.8-a5u7ucknpzdipu2dd3d7hf2ybkdyprly + openssh: + permissions: + write: user + fujitsu-mpi: + version: [head, 4.8.1, 4.8.0, 4.7.0] + externals: + - spec: "fujitsu-mpi@head%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-latest + - spec: "fujitsu-mpi@4.8.1%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-1.2.36 + - spec: "fujitsu-mpi@4.8.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-1.2.35 + - spec: "fujitsu-mpi@4.7.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-1.2.34 + - spec: "fujitsu-mpi@head%gcc@12.2.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc12 + - spec: "fujitsu-mpi@head%gcc@10.4.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc10 + - spec: "fujitsu-mpi@head%gcc@8.5.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc8 + buildable: False + fujitsu-ssl2: + version: [head, 4.8.1, 4.8.0, 4.7.0] + externals: + - spec: "fujitsu-ssl2@head%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-latest + - spec: "fujitsu-ssl2@4.8.1%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-1.2.36 + - spec: "fujitsu-ssl2@4.8.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-1.2.35 + - spec: "fujitsu-ssl2@4.7.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-1.2.34 + buildable: False + rist-fftw: + externals: + - spec: "rist-fftw@3.3.9-272-g63d6bd70 arch=linux-rhel8-a64fx" + prefix: /vol0004/share/rist/fftw/gcc-10.3.0/3.3.9-272-g63d6bd70 + buildable: False + ntchem: + externals: + - spec: "ntchem@13.0.0%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/ntchem/aarch64/NTChem2013/13.0.0/tcsds-1.2.36/release_mpiomp + - spec: "ntchem@12.2.0%fj arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/ntchem/aarch64/NTChem2013/12.2.0/tcsds-1.2.34/release_mpiomp + buildable: False + abinitmp: + externals: + - spec: "abinitmp@1-22%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Feb2023-ABINIT-MP-VER1-REV22 + - spec: "abinitmp@2-4%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Feb2023-ABINIT-MP-VER2-REV4 + - spec: "abinitmp@2-8%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Jan2024-ABINIT-MP-VER2-REV8 + buildable: False + fds: + externals: + - spec: "fds@6.7.7%fj@4.8.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FDS/fds-FDS6.7.7/Build/mpi_fugaku + - spec: "fds@6.7.9%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FDS/fds-FDS6.7.9/Build/mpi_fugaku + buildable: False + ffvhc-ace: + externals: + - spec: "ffvhc-ace@0.1%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FFVHC-ACE/0.1/bin + buildable: False + quantum-espresso: + externals: + - spec: "quantum-espresso@6.5%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.5/bin + - spec: "quantum-espresso@6.6%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.6/bin + - spec: "quantum-espresso@6.7%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.7/bin + - spec: "quantum-espresso@6.8%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.8/bin + - spec: "quantum-espresso@7.0%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.0/bin + - spec: "quantum-espresso@7.1%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.1/bin + - spec: "quantum-espresso@7.2%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.2/bin + - spec: "quantum-espresso@7.3%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.3/bin + buildable: False + autoconf: + externals: + - spec: "autoconf@2.69 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "autoconf@2.69 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "autoconf@2.69 arch=linux-rhel8-skylake_avx512" + prefix: /usr + automake: + externals: + - spec: "automake@1.16.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "automake@1.16.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "automake@1.16.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + binutils: + externals: + - spec: "binutils@2.30 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "binutils@2.30 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "binutils@2.30 arch=linux-rhel8-skylake_avx512" + prefix: /usr + bzip2: + externals: + - spec: "bzip2@1.0.6 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "bzip2@1.0.6 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "bzip2@1.0.6 arch=linux-rhel8-skylake_avx512" + prefix: /usr + curl: + externals: + - spec: "curl@7.61.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "curl@7.61.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "curl@7.61.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + dbus: + externals: + - spec: "dbus@1.12.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "dbus@1.12.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "dbus@1.12.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + elfutils: + externals: + - spec: "elfutils@0.186 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-skylake_avx512" + prefix: /usr + expat: + externals: + - spec: "expat@2.2.5 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "expat@2.2.5 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "expat@2.2.5 arch=linux-rhel8-skylake_avx512" + prefix: /usr + findutils: + externals: + - spec: "findutils@4.6.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "findutils@4.6.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "findutils@4.6.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + fontconfig: + externals: + - spec: "fontconfig@2.13.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "fontconfig@2.13.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "fontconfig@2.13.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + freetype: + externals: + - spec: "freetype@2.9.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "freetype@2.9.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "freetype@2.9.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gmake: + externals: + - spec: "gmake@4.2.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gmake@4.2.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gmake@4.2.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gdbm: + externals: + - spec: "gdbm@1.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gdbm@1.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gdbm@1.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gettext: + externals: + - spec: "gettext@0.19.8.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gettext@0.19.8.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gettext@0.19.8.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gmp: + externals: + - spec: "gmp@6.1.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gmp@6.1.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gmp@6.1.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gnutls: + externals: + - spec: "gnutls@3.6.16 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-skylake_avx512" + prefix: /usr + hwloc: + externals: + - spec: "hwloc@2.2.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "hwloc@2.2.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "hwloc@2.2.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + jansson: + externals: + - spec: "jansson@2.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "jansson@2.11 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "jansson@2.11 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libaio: + externals: + - spec: "libaio@0.3.112 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libaio@0.3.112 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libaio@0.3.112 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libcap: + externals: + - spec: "libcap@2.48 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libcap@2.26 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libcap@2.26 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libdrm: + externals: + - spec: "libdrm@2.4.108 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libedit: + externals: + - spec: "libedit@3.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libedit@3.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libedit@3.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libevent: + externals: + - spec: "libevent@2.1.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libevent@2.1.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libevent@2.1.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libfabric: + externals: + - spec: "libfabric@1.14.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libfabric@1.11.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libfabric@1.11.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libffi: + externals: + - spec: "libffi@3.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libffi@3.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libffi@3.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libglvnd: + externals: + - spec: "libglvnd@1.3.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libglvnd@1.3.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libglvnd@1.3.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libibumad: + externals: + - spec: "libibumad@37.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libibumad@32.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libibumad@54mlnx1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libibumad@54mlnx1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libpciaccess: + externals: + - spec: "libpciaccess@0.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libpciaccess@0.14 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libpciaccess@0.14 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libpng: + externals: + - spec: "libpng@1.6.34 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libpng@1.6.34 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libpng@1.6.34 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtasn1: + externals: + - spec: "libtasn1@4.13 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtasn1@4.13 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtasn1@4.13 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtirpc: + externals: + - spec: "libtirpc@1.1.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtirpc@1.1.4 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtirpc@1.1.4 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtool: + externals: + - spec: "libtool@2.4.6 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtool@2.4.6 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtool@2.4.6 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libuuid: + externals: + - spec: "libuuid@2.32.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libuuid@2.32.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libuuid@2.32.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxcb: + externals: + - spec: "libxcb@1.13.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxcb@1.13.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxcb@1.13.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxkbcommon: + externals: + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxml2: + externals: + - spec: "libxml2@2.9.7 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxml2@2.9.7 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxml2@2.9.7 arch=linux-rhel8-skylake_avx512" + prefix: /usr + lz4: + externals: + - spec: "lz4@1.8.3 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "lz4@1.8.3 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "lz4@1.8.3 arch=linux-rhel8-skylake_avx512" + prefix: /usr + m4: + externals: + - spec: "m4@1.4.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "m4@1.4.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "m4@1.4.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + ncurses: + externals: + - spec: "ncurses@6.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ncurses@6.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "ncurses@6.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + nettle: + externals: + - spec: "nettle@3.4.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nettle@3.4.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "nettle@3.4.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + nspr: + externals: + - spec: "nspr@4.32.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + numactl: + externals: + - spec: "numactl@2.0.12 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "numactl@2.0.12 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "numactl@2.0.12 arch=linux-rhel8-skylake_avx512" + prefix: /usr + opengl: + buildable: False + externals: + - spec: opengl@4.5.0 + prefix: /usr + openssl: + buildable: False + externals: + - spec: "openssl@1.1.1k arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "openssl@1.1.1g arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "openssl@1.1.1k arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "openssl@1.1.1k arch=linux-rhel8-skylake_avx512" + prefix: /usr + papi: + externals: + - spec: "papi@5.6.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "papi@5.6.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "papi@5.6.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + pcre: + externals: + - spec: "pcre@8.42 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "pcre@8.42 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "pcre@8.42 arch=linux-rhel8-skylake_avx512" + prefix: /usr + perl: + externals: + - spec: "perl@5.26.3 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "perl@5.26.3 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "perl@5.26.3 arch=linux-rhel8-skylake_avx512" + prefix: /usr + pkgconf: + externals: + - spec: "pkgconf@1.4.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "pkgconf@1.4.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "pkgconf@1.4.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + popt: + externals: + - spec: "popt@1.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "popt@1.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "popt@1.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + readline: + externals: + - spec: "readline@7.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "readline@7.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "readline@7.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + sqlite: + externals: + - spec: "sqlite@3.26.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "sqlite@3.26.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "sqlite@3.26.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + tcl: + externals: + - spec: "tcl@8.6.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "tcl@8.6.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "tcl@8.6.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + ucx: + externals: + - spec: "ucx@1.11.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ucx@1.9.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ucx@1.11.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "ucx@1.11.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + valgrind: + externals: + - spec: "valgrind@3.18.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + xz: + externals: + - spec: "xz@5.2.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "xz@5.2.4 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "xz@5.2.4 arch=linux-rhel8-skylake_avx512" + prefix: /usr + zlib: + externals: + - spec: "zlib@1.2.11 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "zlib@1.2.11 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "zlib@1.2.11 arch=linux-rhel8-skylake_avx512" + prefix: /usr + buildable: False + # pmlib: had problems with spack. so far binary packages only. 2023/3/20 mikami + pmlib: + externals: + - spec: "pmlib@9.0-clang-precise arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-clang-precise + - spec: "pmlib@9.0-clang-power arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-clang-power + - spec: "pmlib@9.0-trad-power arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-trad-power + buildable: False diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml new file mode 100644 index 00000000..bf2cf20c --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml @@ -0,0 +1,23 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: fj@{default_fujitsu_version} + default-mpi: + spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} + compiler-gcc: + spack_spec: gcc@12.2.0 + compiler-fujitsu: + spack_spec: fj@{default_fujitsu_version} + blas: + spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} + lapack: + spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} + mpi-gcc: + spack_spec: fujitsu-mpi@head%gcc@12.2.0 + mpi-fujitsu: + spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} \ No newline at end of file diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml new file mode 100644 index 00000000..afdbd6d6 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: Fugaku + site: RIKEN Center for Computational Science + system: Fujitsu-A64FX-TofuD + integrator: + vendor: Fujitsu + name: FX1000 + processor: + vendor: Fujitsu + name: A64FX + ISA: Armv8.2-A-SVE + uArch: aarch64 + accelerator: + vendor: + name: + ISA: + uArch: + interconnect: + vendor: Fujitsu + name: TofuInterconnectD + system-tested: + site: R-CCS + name: Fugaku + installation-year: 2020 + description: top500 + top500-system-instances: + - Fugaku (R-CCS) \ No newline at end of file diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml new file mode 100644 index 00000000..046ce515 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -0,0 +1,13 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + batch_time: '02:00' + mpi_command: 'mpiexec' + batch_submit: 'pjsub {execute_experiment}' + batch_nodes: '#PJM -L "node={n_nodes}"' + batch_ranks: '#PJM --mpi proc={n_ranks}' + batch_timeout: '#PJM -L "elapse={batch_time}:00"' + default_fujitsu_version: '4.8.1' \ No newline at end of file diff --git a/docs/tables/current-system-definitions.csv b/docs/tables/current-system-definitions.csv index 990480ba..e891916b 100644 --- a/docs/tables/current-system-definitions.csv +++ b/docs/tables/current-system-definitions.csv @@ -1,20 +1,20 @@ -,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Tioga,x86_64 -**site**,LLNL,,,LLNL,LLNL, -**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,HPECray-zen3-MI250X-Slingshot,x86_64 -**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,HPECray, -**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,EX235a, -**processor.vendor**,IBM,AMD,AMD,Intel,AMD, -**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,EPYC-Zen3, -**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,x86_64,x86_64 -**processor.uArch**,power9,zen4,zen3,icelake,zen3, -**accelerator.vendor**,NVIDIA,,AMD,,AMD, -**accelerator.name**,V100,,MI250X,,MI250X, -**accelerator.ISA**,PTX,,GCN,,GCN, -**accelerator.uArch**,sm_70,,gfx90a,,gfx90a, -**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,HPECray, -**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,Slingshot11, -**system-tested.site**,LLNL,AWS,LLNL,LLNL,LLNL, -**system-tested.name**,lassen,,tioga,magma,tioga, -**system-tested.installation-year**,2018,,2022,2019,2022, -**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_, -**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", +,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Pascal,Tioga,x86_64 +**site**,LLNL,,,LLNL,RIKEN Center for Computational Science,LLNL,LLNL, +**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,HPECray-zen3-MI250X-Slingshot,x86_64 +**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,Penguin,HPECray, +**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,FX1000,,EX235a, +**processor.vendor**,IBM,AMD,AMD,Intel,Fujitsu,Intel,AMD, +**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,Xeon_E5-2695_v4,EPYC-Zen3, +**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64 +**processor.uArch**,power9,zen4,zen3,icelake,aarch64,broadwell,zen3, +**accelerator.vendor**,NVIDIA,,AMD,,,NVIDIA,AMD, +**accelerator.name**,V100,,MI250X,,,P100,MI250X, +**accelerator.ISA**,PTX,,GCN,,,PTX,GCN, +**accelerator.uArch**,sm_70,,gfx90a,,,sm_56,gfx90a, +**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,Cornelis,HPECray, +**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuInterconnectD,OmniPath,Slingshot11, +**system-tested.site**,LLNL,AWS,LLNL,LLNL,R-CCS,LLNL,LLNL, +**system-tested.name**,lassen,,tioga,magma,Fugaku,pascal,tioga, +**system-tested.installation-year**,2018,,2022,2019,2020,2018,2022, +**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_, +**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (R-CCS),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", diff --git a/experiments/amg2023/openmp/ramble.yaml b/experiments/amg2023/openmp/ramble.yaml index 23782fe5..ddd9ac9c 100644 --- a/experiments/amg2023/openmp/ramble.yaml +++ b/experiments/amg2023/openmp/ramble.yaml @@ -33,7 +33,7 @@ ramble: nz: '{n}' processes_per_node: ['8', '4'] n_nodes: ['1', '2'] - threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks + threads_per_node_core: ['4', '6', '12'] #TODO: Specify n_threads according to available n_nodes and n_ranks omp_num_threads: '{threads_per_node_core} * {n_nodes}' experiments: amg2023_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: From a4d948001b839d5f04b0dfa32ba53a1e1a462a02 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Thu, 22 Feb 2024 19:35:09 +1100 Subject: [PATCH 2/6] Updating the systems table in the docs --- .../system_definition.yaml | 4 ++ docs/tables/current-system-definitions.csv | 40 +++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/configs/CSC-Lumi-HPECray-zen3-MI250X-Slingshot/system_definition.yaml b/configs/CSC-Lumi-HPECray-zen3-MI250X-Slingshot/system_definition.yaml index cf1411e5..83f72722 100644 --- a/configs/CSC-Lumi-HPECray-zen3-MI250X-Slingshot/system_definition.yaml +++ b/configs/CSC-Lumi-HPECray-zen3-MI250X-Slingshot/system_definition.yaml @@ -28,3 +28,7 @@ system_definition: name: LUMI installation-year: 2023 description: top500 + top500-system-instances: + - Frontier (ORNL) + - Lumi (CSC) + - Tioga (LLNL) \ No newline at end of file diff --git a/docs/tables/current-system-definitions.csv b/docs/tables/current-system-definitions.csv index e891916b..4ebbc2d2 100644 --- a/docs/tables/current-system-definitions.csv +++ b/docs/tables/current-system-definitions.csv @@ -1,20 +1,20 @@ -,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Pascal,Tioga,x86_64 -**site**,LLNL,,,LLNL,RIKEN Center for Computational Science,LLNL,LLNL, -**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,HPECray-zen3-MI250X-Slingshot,x86_64 -**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,Penguin,HPECray, -**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,FX1000,,EX235a, -**processor.vendor**,IBM,AMD,AMD,Intel,Fujitsu,Intel,AMD, -**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,Xeon_E5-2695_v4,EPYC-Zen3, -**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64 -**processor.uArch**,power9,zen4,zen3,icelake,aarch64,broadwell,zen3, -**accelerator.vendor**,NVIDIA,,AMD,,,NVIDIA,AMD, -**accelerator.name**,V100,,MI250X,,,P100,MI250X, -**accelerator.ISA**,PTX,,GCN,,,PTX,GCN, -**accelerator.uArch**,sm_70,,gfx90a,,,sm_56,gfx90a, -**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,Cornelis,HPECray, -**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuInterconnectD,OmniPath,Slingshot11, -**system-tested.site**,LLNL,AWS,LLNL,LLNL,R-CCS,LLNL,LLNL, -**system-tested.name**,lassen,,tioga,magma,Fugaku,pascal,tioga, -**system-tested.installation-year**,2018,,2022,2019,2020,2018,2022, -**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_, -**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (R-CCS),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", +,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Pascal,LUMI-zen3-MI250X-Slingshot,Tioga,x86_64 +**site**,LLNL,,,LLNL,RIKEN Center for Computational Science,LLNL,CSC,LLNL, +**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,LUMI-zen3-MI250X-Slingshot,HPECray-zen3-MI250X-Slingshot,x86_64 +**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,Penguin,HPECray,HPECray, +**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,FX1000,,EX235a,EX235a, +**processor.vendor**,IBM,AMD,AMD,Intel,Fujitsu,Intel,AMD,AMD, +**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,Xeon_E5-2695_v4,EPYC-Zen3,EPYC-Zen3, +**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64,x86_64 +**processor.uArch**,power9,zen4,zen3,icelake,aarch64,broadwell,zen3,zen3, +**accelerator.vendor**,NVIDIA,,AMD,,,NVIDIA,AMD,AMD, +**accelerator.name**,V100,,MI250X,,,P100,MI250X,MI250X, +**accelerator.ISA**,PTX,,GCN,,,PTX,GCN,GCN, +**accelerator.uArch**,sm_70,,gfx90a,,,sm_56,gfx90a,gfx90a, +**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,Cornelis,HPECray,HPECray, +**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuInterconnectD,OmniPath,Slingshot11,Slingshot11, +**system-tested.site**,LLNL,AWS,LLNL,LLNL,R-CCS,LLNL,CSC,LLNL, +**system-tested.name**,lassen,,tioga,magma,Fugaku,pascal,LUMI,tioga, +**system-tested.installation-year**,2018,,2022,2019,2020,2018,2023,2022, +**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_,`top500 `_, +**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (R-CCS),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)","Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", From feabae9e1c70fc8483fce9d80a92dc84f0637278 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Fri, 23 Feb 2024 10:07:45 +1100 Subject: [PATCH 3/6] Feature gromacs openmp (#144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create gromacs/application.py * Add gromacs openmp config * Add gromacs openmp config (#62) * Update repo/gromacs/application.py Co-authored-by: Szilárd Páll * Update repo/gromacs/application.py Co-authored-by: Szilárd Páll * copyright * copyright * copyright * Draft ADAC experiment setup * CUDA experiment for LLNL-Sierra-IBM-power9-V100-Infiniband * CUDA config for LLNL-Pascal-Penguin-broadwell-P100-OmniPat * Fix target name * CUDA config for lassen * Fix cuda config * Set Openmp thread affinity to cores * Fix cuda compiler for gromacs * gromacs rocm config * Fix gromacs openmp spec --------- Co-authored-by: Riyaz Haque Co-authored-by: Szilárd Páll Co-authored-by: Riyaz Haque --- .../auxiliary_software_files/packages.yaml | 10 + .../auxiliary_software_files/compilers.yaml | 13 ++ .../auxiliary_software_files/packages.yaml | 18 +- .../spack.yaml | 10 +- .../variables.yaml | 2 +- .../auxiliary_software_files/compilers.yaml | 16 ++ .../auxiliary_software_files/packages.yaml | 9 + .../spack.yaml | 10 +- .../variables.yaml | 2 +- .../auxiliary_software_files/compilers.yaml | 18 +- .../auxiliary_software_files/packages.yaml | 19 ++ .../spack.yaml | 6 + .../gromacs/cuda/execute_experiment.tpl | 16 ++ experiments/gromacs/cuda/ramble.yaml | 61 +++++ .../gromacs/openmp/execute_experiment.tpl | 16 ++ experiments/gromacs/openmp/ramble.yaml | 59 +++++ .../gromacs/rocm/execute_experiment.tpl | 16 ++ experiments/gromacs/rocm/ramble.yaml | 61 +++++ repo/cusolver/package.py | 11 + repo/gromacs/application.py | 215 ++++++++++++++++++ repo/gromacs/package.py | 22 +- 21 files changed, 598 insertions(+), 12 deletions(-) create mode 100755 experiments/gromacs/cuda/execute_experiment.tpl create mode 100644 experiments/gromacs/cuda/ramble.yaml create mode 100755 experiments/gromacs/openmp/execute_experiment.tpl create mode 100644 experiments/gromacs/openmp/ramble.yaml create mode 100755 experiments/gromacs/rocm/execute_experiment.tpl create mode 100644 experiments/gromacs/rocm/ramble.yaml create mode 100644 repo/cusolver/package.py create mode 100644 repo/gromacs/application.py diff --git a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml index 941b8a7a..e170c70d 100644 --- a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml @@ -28,6 +28,16 @@ packages: - spec: intel-oneapi-mkl@2022.1.0 prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp buildable: false + hwloc: + externals: + - spec: hwloc@2.9.1 + prefix: /usr + buildable: false + fftw: + externals: + - spec: fftw@3.3.10 + prefix: /usr/tce/packages/fftw/fftw-3.3.10 + buildable: false mpi: externals: - spec: mvapich2@2.3.7-gcc1211 diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml index d009ca70..05a2af51 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml @@ -17,3 +17,16 @@ compilers: modules: [] environment: {} extra_rpaths: [] +- compiler: + spec: clang@=14.0.6 + paths: + cc: /usr/tce/packages/clang/clang-14.0.6/bin/clang + cxx: /usr/tce/packages/clang/clang-14.0.6/bin/clang++ + f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml index eea39472..1daf4b71 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml @@ -32,8 +32,12 @@ packages: mvapich2: buildable: false externals: - - spec: mvapich2@2.3.7%intel@2021.6.0 + - spec: mvapich2@2.3.7-gcc-11.2.1 + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-11.2.1 + - spec: mvapich2@2.3.7-intel-2021.6.0 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/ + - spec: mvapich2@2.3.7-clang-14.0.6 + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6 cuda: buildable: false externals: @@ -55,10 +59,22 @@ packages: prefix: /usr/tce/packages/cuda/cuda-11.8.0 buildable: false blas: + externals: + - spec: intel-oneapi-mkl@2022.1.0 + prefix: /usr/tce/packages/mkl/mkl-2022.1.0 + buildable: false + fftw: + externals: + - spec: intel-oneapi-mkl@2022.1.0 + prefix: /usr/tce/packages/mkl/mkl-2022.1.0 buildable: false lapack: + externals: + - spec: intel-oneapi-mkl@2022.1.0 + prefix: /usr/tce/packages/mkl/mkl-2022.1.0 buildable: false intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.1.0 prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ + buildable: false diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml index 0a1d794d..1f15be61 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml @@ -8,10 +8,18 @@ spack: default-compiler: spack_spec: gcc@=11.2.1 default-mpi: - spack_spec: mvapich2@2.3.7 + spack_spec: mvapich2@2.3.7-gcc-11.2.1 compiler-gcc: spack_spec: gcc@=11.2.1 + compiler-clang: + spack_spec: clang@=14.0.6 + mpi-clang: + spack_spec: mvapich2@2.3.7-clang-14.0.6 + blas: + spack_spec: intel-oneapi-mkl@2022.1.0 cublas-cuda: spack_spec: cublas@{default_cuda_version} lapack: spack_spec: intel-oneapi-mkl@2022.1.0 + fftw: + spack_spec: intel-oneapi-mkl@2022.1.0 diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml index 462225a5..21097fef 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml @@ -11,5 +11,5 @@ variables: mpi_command: 'srun -N {n_nodes} -n {n_ranks}' batch_submit: 'sbatch {execute_experiment}' batch_nodes: '#SBATCH -N {n_nodes}' - batch_ranks: '#SBATCH -n {n_ranks}' + batch_ranks: '#SBATCH -n {n_ranks} -G {n_ranks}' batch_timeout: '#SBATCH -t {batch_time}:00' diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml index 501522dc..d27a632c 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml @@ -65,3 +65,19 @@ compilers: modules: [] environment: {} extra_rpaths: [] +- compiler: + spec: clang@16.0.6-cuda11.8.0 + paths: + cc: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang + cxx: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang++ + f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + flags: + cflags: -g -O2 + cxxflags: -g -O2 -std=c++17 + fflags: '' + operating_system: rhel7 + target: ppc64le + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml index dd43a66b..68e274aa 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml @@ -55,6 +55,11 @@ packages: - spec: cublas@11.8.0 prefix: /usr/tce/packages/cuda/cuda-11.8.0 buildable: false + fftw: + externals: + - spec: fftw@3.3.10 + prefix: /usr/tcetmp/packages/fftw/fftw-3.3.10-xl-2023.06.28 + buildable: false lapack: externals: - spec: lapack-xl@3.9.0 @@ -70,4 +75,8 @@ packages: prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-10.1.243 extra_attributes: ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm" + - spec: spectrum-mpi@2022.08.19-clang16.0.6-cuda-11.8.0 + prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-16.0.6-cuda-11.8.0-gcc-11.2.1 + extra_attributes: + ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm" buildable: false diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml index 91d89957..ed1edcb6 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml @@ -11,7 +11,15 @@ spack: spack_spec: spectrum-mpi@2022.08.19 compiler-gcc: spack_spec: gcc@8.3.1 + compiler-clang: + spack_spec: clang@16.0.6-cuda{default_cuda_version} + mpi-clang: + spack_spec: spectrum-mpi@2022.08.19-clang16.0.6-cuda-{default_cuda_version} + blas: + spack_spec: cublas@11.8.0 cublas-cuda: - spack_spec: cublas@{default_cuda_version} + spack_spec: cublas@{default_cuda_version} lapack: spack_spec: lapack-xl@3.9.0 + fftw: + spack_spec: fftw@3.3.10 diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/variables.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/variables.yaml index e7a6b343..b85115bb 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/variables.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/variables.yaml @@ -9,7 +9,7 @@ variables: mpi_command: '/usr/tcetmp/bin/lrun -n {n_ranks} -T {processes_per_node} {gtl_flag}' batch_submit: 'bsub -q pdebug {execute_experiment}' batch_nodes: '#BSUB -nnodes {n_nodes}' - batch_ranks: '#BSUB -nnodes {n_nodes}' + batch_ranks: '' batch_timeout: '#BSUB -W {batch_time}' default_cuda_version: '11.8.0' cuda_arch: '70' diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml index e3a97b0a..d7af09cc 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml @@ -13,13 +13,29 @@ compilers: fc: /opt/cray/pe/cce/16.0.0/bin/crayftn flags: cflags: -g -O2 - cxxflags: -g -O2 -std=c++14 + cxxflags: -g -O2 -std=c++17 fflags: -g -O2 -hnopattern operating_system: rhel8 target: x86_64 modules: [] environment: {} extra_rpaths: [/opt/cray/pe/gcc-libs/] +- compiler: + spec: clang@16.0.0-rocm5.5.1 + paths: + cc: /opt/rocm-5.5.1/bin/amdclang + cxx: /opt/rocm-5.5.1/bin/amdclang++ + f77: /opt/rocm-5.5.1/bin/amdflang + fc: /opt/rocm-5.5.1/bin/amdflang + flags: + cflags: -g -O2 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr + cxxflags: -g -O2 -std=c++17 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr + fflags: -g -O2 -hnopattern + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] - compiler: spec: gcc@12.2.0 paths: diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml index 93d45cee..af604fd2 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml @@ -115,6 +115,11 @@ packages: externals: - spec: bison@3.0.4 prefix: /usr + python: + externals: + - spec: python@3.9.12 + prefix: /usr/tce/packages/python/python-3.9.12 + buildable: false rocprim: externals: - spec: rocprim@5.4.3 @@ -193,6 +198,20 @@ packages: prefix: /opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/ lapack: buildable: false + hypre: + variants: amdgpu_target=gfx90a + hwloc: + externals: + - spec: hwloc@2.9.1 + prefix: /usr + buildable: false + fftw: + buildable: false + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2023.2.0 + prefix: /opt/intel/oneapi + buildable: false mpi: buildable: false cray-mpich: diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml index eeaa9e70..0b96b4d5 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml @@ -11,10 +11,14 @@ spack: spack_spec: cray-mpich@8.1%cce ~gtl compiler-rocm: spack_spec: cce@16 + compiler-amdclang: + spack_spec: clang@16.0.0-rocm5.5.1 blas-rocm: spack_spec: rocblas@5.5 blas: spack_spec: rocblas@5.5 + lapack-rocm: + spack_spec: rocsolver@5.5 lapack: spack_spec: cray-libsci@23 mpi-rocm-gtl: @@ -23,3 +27,5 @@ spack: spack_spec: cray-mpich@8.1%cce ~gtl mpi-gcc: spack_spec: cray-mpich@8.1%gcc ~gtl + fftw: + spack_spec: intel-oneapi-mkl@2023.2.0 diff --git a/experiments/gromacs/cuda/execute_experiment.tpl b/experiments/gromacs/cuda/execute_experiment.tpl new file mode 100755 index 00000000..ab02968f --- /dev/null +++ b/experiments/gromacs/cuda/execute_experiment.tpl @@ -0,0 +1,16 @@ +#!/bin/bash + +## Copyright 2023 Lawrence Livermore National Security, LLC and other +## Benchpark Project Developers. See the top-level COPYRIGHT file for details. +## +## SPDX-License-Identifier: Apache-2.0 + +{batch_nodes} +{batch_ranks} +{batch_timeout} + +cd {experiment_run_dir} + +{experiment_setup} + +{command} diff --git a/experiments/gromacs/cuda/ramble.yaml b/experiments/gromacs/cuda/ramble.yaml new file mode 100644 index 00000000..6591db87 --- /dev/null +++ b/experiments/gromacs/cuda/ramble.yaml @@ -0,0 +1,61 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + applications: + gromacs: + workloads: + water_gmx50_adac: + env_vars: + set: + OMP_PROC_BIND: close + OMP_PLACES: cores + OMP_NUM_THREADS: '{omp_num_threads}' + variables: + experiment_setup: '' + n_ranks: '{processes_per_node} * {n_nodes}' + processes_per_node: '2' + n_nodes: '4' + omp_num_threads: '10' + target: 'gpu' + experiments: + gromacs_water_gmx50_adac_size{size}_dlb{dlb}_pin{pin}_target{target}_maxh{maxh}_nsteps{nsteps}_nstlist{nstlist}_npme{npme}: + variables: + dlb: 'no' + pin: 'off' + nb: '{target}' + pme: 'auto' + bonded: 'cpu' + update: '{target}' + maxh: '0.05' + nsteps: '1000' + nstlist: '200' + npme: '1' + size: '1536' + + spack: + concretized: true + packages: + gromacs: + spack_spec: gromacs@2023.3 +mpi+openmp+cuda~hwloc~double cuda_arch=={cuda_arch} + compiler: compiler-clang + cuda: + spack_spec: cuda@{default_cuda_version}+allow-unsupported-compilers + compiler: compiler-clang + environments: + gromacs: + packages: + - cuda + - blas + - lapack + - mpi-clang + - fftw + - gromacs diff --git a/experiments/gromacs/openmp/execute_experiment.tpl b/experiments/gromacs/openmp/execute_experiment.tpl new file mode 100755 index 00000000..ab02968f --- /dev/null +++ b/experiments/gromacs/openmp/execute_experiment.tpl @@ -0,0 +1,16 @@ +#!/bin/bash + +## Copyright 2023 Lawrence Livermore National Security, LLC and other +## Benchpark Project Developers. See the top-level COPYRIGHT file for details. +## +## SPDX-License-Identifier: Apache-2.0 + +{batch_nodes} +{batch_ranks} +{batch_timeout} + +cd {experiment_run_dir} + +{experiment_setup} + +{command} diff --git a/experiments/gromacs/openmp/ramble.yaml b/experiments/gromacs/openmp/ramble.yaml new file mode 100644 index 00000000..166bb962 --- /dev/null +++ b/experiments/gromacs/openmp/ramble.yaml @@ -0,0 +1,59 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + applications: + gromacs: + workloads: + water_gmx50_adac: + env_vars: + set: + OMP_PROC_BIND: close + OMP_PLACES: cores + OMP_NUM_THREADS: '{omp_num_threads}' + variables: + experiment_setup: '' + n_ranks: '{processes_per_node} * {n_nodes}' + processes_per_node: '4' + n_nodes: '2' + omp_num_threads: '16' + target: 'cpu' + experiments: + gromacs_water_gmx50_adac_size{size}_dlb{dlb}_pin{pin}_target{target}_maxh{maxh}_nsteps{nsteps}_nstlist{nstlist}_npme{npme}: + variables: + dlb: 'no' + pin: 'off' + nb: '{target}' + pme: 'auto' + bonded: '{target}' + update: '{target}' + maxh: '0.05' + nsteps: '1000' + nstlist: '200' + npme: '0' + size: '1536' + + spack: + concretized: true + packages: + fftw: + spack_spec: fftw@3.3.10 +mpi+openmp + compiler: default-compiler + gromacs: + spack_spec: gromacs@main +mpi+openmp~hwloc + compiler: default-compiler + environments: + gromacs: + packages: + - lapack + - default-mpi + - fftw + - gromacs diff --git a/experiments/gromacs/rocm/execute_experiment.tpl b/experiments/gromacs/rocm/execute_experiment.tpl new file mode 100755 index 00000000..ab02968f --- /dev/null +++ b/experiments/gromacs/rocm/execute_experiment.tpl @@ -0,0 +1,16 @@ +#!/bin/bash + +## Copyright 2023 Lawrence Livermore National Security, LLC and other +## Benchpark Project Developers. See the top-level COPYRIGHT file for details. +## +## SPDX-License-Identifier: Apache-2.0 + +{batch_nodes} +{batch_ranks} +{batch_timeout} + +cd {experiment_run_dir} + +{experiment_setup} + +{command} diff --git a/experiments/gromacs/rocm/ramble.yaml b/experiments/gromacs/rocm/ramble.yaml new file mode 100644 index 00000000..c1aa8368 --- /dev/null +++ b/experiments/gromacs/rocm/ramble.yaml @@ -0,0 +1,61 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + applications: + gromacs: + workloads: + water_gmx50_adac: + env_vars: + set: + OMP_PROC_BIND: close + OMP_PLACES: cores + OMP_NUM_THREADS: '{omp_num_threads}' + variables: + experiment_setup: '' + n_ranks: '{processes_per_node} * {n_nodes}' + processes_per_node: '8' + n_nodes: '1' + omp_num_threads: '8' + target: 'gpu' + experiments: + gromacs_water_gmx50_adac_size{size}_dlb{dlb}_pin{pin}_target{target}_maxh{maxh}_nsteps{nsteps}_nstlist{nstlist}_npme{npme}: + variables: + dlb: 'no' + pin: 'off' + nb: '{target}' + pme: 'auto' + bonded: '{target}' + update: '{target}' + maxh: '0.05' + nsteps: '1000' + nstlist: '200' + npme: '0' + size: '1536' + + spack: + concretized: true + packages: + hipsycl: + spack_spec: hipsycl@23.10.0 + compiler: default-compiler + gromacs: + spack_spec: gromacs@2023.3 +mpi+openmp+rocm~hwloc~double amdgpu_target={rocm_arch} + compiler: default-compiler + environments: + gromacs: + packages: + - hipsycl + - blas-rocm + - lapack-rocm + - mpi-rocm-gtl + - fftw + - gromacs diff --git a/repo/cusolver/package.py b/repo/cusolver/package.py new file mode 100644 index 00000000..b2b74dad --- /dev/null +++ b/repo/cusolver/package.py @@ -0,0 +1,11 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * + + +class Cusolver(Package): + + provides("lapack") diff --git a/repo/gromacs/application.py b/repo/gromacs/application.py new file mode 100644 index 00000000..fa5c6895 --- /dev/null +++ b/repo/gromacs/application.py @@ -0,0 +1,215 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from ramble.appkit import * +from ramble.expander import Expander + + +class Gromacs(SpackApplication): + '''Define a Gromacs application''' + name = 'gromacs' + + maintainers('douglasjacobsen') + + tags('molecular-dynamics') + + default_compiler('gcc9', spack_spec='gcc@9.3.0') + software_spec('impi2018', spack_spec='intel-mpi@2018.4.274') + software_spec('gromacs', spack_spec='gromacs@2023.3', compiler='gcc12') + + executable('pre-process', 'gmx_mpi grompp ' + + '-f {input_path}/{type}.mdp ' + + '-c {input_path}/conf.gro ' + + '-p {input_path}/topol.top ' + + '-o exp_input.tpr', use_mpi=False) + executable('execute-nsteps-gen', 'gmx_mpi mdrun -notunepme ' + + '-v -resethway -noconfout -nsteps 4000 ' + + '-s exp_input.tpr', use_mpi=True) + executable('execute-nsteps', 'gmx_mpi mdrun -notunepme ' + + '-v -resethway -noconfout -nsteps 4000 ' + + '-s {input_path}', use_mpi=True) + executable('execute-adac', 'gmx_mpi mdrun ' + + '-resethway -noconfout -dlb {dlb} -pin {pin} -nb {nb} -pme {pme} -bonded {bonded} -update {update} ' + + '-maxh {maxh} -nsteps {nsteps} -notunepme -nstlist {nstlist} -npme {npme} ' + + '-v -s exp_input.tpr', use_mpi=True) + + input_file('water_gmx50_bare', url='https://ftp.gromacs.org/pub/benchmarks/water_GMX50_bare.tar.gz', + sha256='2219c10acb97787f80f6638132bad3ff2ca1e68600eef1bc8b89d9560e74c66a', + description='') + input_file('water_bare_hbonds', url='https://ftp.gromacs.org/pub/benchmarks/water_bare_hbonds.tar.gz', + sha256='b2e09d30f5c6b00ecf1c13ea6fa715ad132747863ef89f983f6c09a872cf2776', + description='') + input_file('lignocellulose', + url='https://repository.prace-ri.eu/ueabs/GROMACS/1.2/GROMACS_TestCaseB.tar.gz', + sha256='8a12db0232465e1d47c6a4eb89f615cdbbdc8fc360a86088b131331bd462f35c', + description='A model of cellulose and lignocellulosic biomass in an aqueous ' + + 'solution. This system of 3.3M atoms is inhomogeneous, at ' + + 'least with GROMACS 4.5. This system uses reaction-field' + + 'electrostatics instead of PME and therefore should scale well.') + input_file('HECBioSim', + url='https://github.com/victorusu/GROMACS_Benchmark_Suite/archive/refs/tags/1.0.0.tar.gz', + sha256='9cb2ad61ec2a422fc33578047e7cb2fd2c37ae9a75a6162d662fa2b711e9737f', + description='https://www.hecbiosim.ac.uk/access-hpc/benchmarks') + + input_file('BenchPEP', url='https://www.mpinat.mpg.de/benchPEP.zip', + sha256='f11745201dbb9e6a29a39cb016ee8123f6b0f519b250c94660f0a9623e497b22', + description='12M Atoms, Peptides in Water, 2fs time step, all bonds constrained. https://www.mpinat.mpg.de/grubmueller/bench') + + input_file('BenchPEP_h', url='https://www.mpinat.mpg.de/benchPEP-h.zip', + sha256='3ca8902fd9a6cf005b266f83b57217397b4ba4af987b97dc01e04185bd098bce', + description='12M Atoms, Peptides in Water, 2fs time step, h-bonds constrained. https://www.mpinat.mpg.de/grubmueller/bench') + + input_file('BenchMEM', url='https://www.mpinat.mpg.de/benchMEM.zip', + sha256='3c1c8cd4f274d532f48c4668e1490d389486850d6b3b258dfad4581aa11380a4', + description='82k atoms, protein in membrane surrounded by water, 2 fs time step. https://www.mpinat.mpg.de/grubmueller/bench') + + input_file('BenchRIB', url='https://www.mpinat.mpg.de/benchRIB.zip', + sha256='39acb014a79ed9a9ff2ad6294a2c09f9b85ea6986dfc204a3639814503eeb60a', + description='2 M atoms, ribosome in water, 4 fs time step. https://www.mpinat.mpg.de/grubmueller/bench') + + input_file('JCP_benchmarks', + url='https://zenodo.org/record/3893789/files/GROMACS_heterogeneous_parallelization_benchmark_info_and_systems_JCP.tar.gz?download=1', + sha256='82449291f44f4d5b7e5c192d688b57b7c2a2e267fe8b12e7a15b5d68f96c7b20', + description='GROMACS_heterogeneous_parallelization_benchmark_info_and_systems_JCP') + + workload('water_gmx50_adac', executables=['pre-process', 'execute-adac'], + input='water_gmx50_bare') + workload('water_gmx50', executables=['pre-process', 'execute-gen'], + input='water_gmx50_bare') + workload('water_bare', executables=['pre-process', 'execute-gen'], + input='water_bare_hbonds') + workload('lignocellulose', executables=['execute'], + input='lignocellulose') + workload('hecbiosim', executables=['execute'], + input='HECBioSim') + workload('benchpep', executables=['execute'], + input='BenchPEP') + workload('benchpep_h', executables=['execute'], + input='BenchPEP_h') + workload('benchmem', executables=['execute'], + input='BenchMEM') + workload('benchrib', executables=['execute'], + input='BenchRIB') + workload('stmv_rf', executables=['pre-process', 'execute-gen'], + input='JCP_benchmarks') + workload('stmv_pme', executables=['pre-process', 'execute-gen'], + input='JCP_benchmarks') + workload('rnase_cubic', executables=['pre-process', 'execute-gen'], + input='JCP_benchmarks') + workload('ion_channel', executables=['pre-process', 'execute-gen'], + input='JCP_benchmarks') + + workload_variable('dlb', default='no', + description='Dynamic load balancing (with DD): auto, no, yes', + workloads=['water_gmx50_adac']) + workload_variable('pin', default='off', + description='Whether mdrun should try to set thread affinities: auto, on, off', + workloads=['water_gmx50_adac']) + workload_variable('nb', default='auto', + description='Calculate non-bonded interactions on: auto, cpu, gpu', + workloads=['water_gmx50_adac']) + workload_variable('pme', default='auto', + description='Perform PME calculations on: auto, cpu, gpu', + workloads=['water_gmx50_adac']) + workload_variable('bonded', default='auto', + description='Perform bonded calculations on: auto, cpu, gpu', + workloads=['water_gmx50_adac']) + workload_variable('update', default='auto', + description='Perform update and constraints on: auto, cpu, gpu', + workloads=['water_gmx50_adac']) + workload_variable('maxh', default='0.05', + description='Terminate after 0.99 times this time (hours)', + workloads=['water_gmx50_adac']) + workload_variable('nsteps', default='-1', + description='Run this number of steps (-1 means infinite, -2 means use mdp option, smaller is invalid)', + workloads=['water_gmx50_adac']) + workload_variable('nstlist', default='200', + description='Set nstlist when using a Verlet buffer tolerance (0 is guess)', + workloads=['water_gmx50_adac']) + workload_variable('npme', default='0', + description='Number of separate ranks to be used for PME, -1 is guess', + workloads=['water_gmx50_adac']) + + workload_variable('size', default='1536', + values=['0000.65', '0000.96', '0001.5', + '0003', '0006', '0012', '0024', + '0048', '0096', '0192', '0384', + '0768', '1536', '3072'], + description='Workload size', + workloads=['water_gmx50', 'water_bare', 'water_gmx50_adac']) + workload_variable('type', default='pme', + description='Workload type.', + values=['pme', 'rf'], + workloads=['water_gmx50', 'water_bare', 'water_gmx50_adac']) + workload_variable('input_path', default='{water_gmx50_bare}/{size}', + description='Input path for water GMX50', + workloads=['water_gmx50', 'water_gmx50_adac']) + workload_variable('input_path', default='{water_bare_hbonds}/{size}', + description='Input path for water bare hbonds', + workload='water_bare') + workload_variable('input_path', default='{lignocellulose}/lignocellulose-rf.tpr', + description='Input path for lignocellulose', + workload='lignocellulose') + workload_variable('type', default='Crambin', + description='Workload type. Valid values are ''Crambin'', ''Glutamine-Binding-Protein'', ''hEGFRDimer'', ''hEGFRDimerPair'', ''hEGFRDimerSmallerPL'', ''hEGFRtetramerPair''', + workload='hecbiosim') + workload_variable('input_path', default='{HECBioSim}/HECBioSim/{type}/benchmark.tpr', + description='Input path for hecbiosim', + workload='hecbiosim') + workload_variable('input_path', default='{BenchPEP}/benchPEP.tpr', + description='Input path for Bench PEP workload', + workload='benchpep') + workload_variable('input_path', default='{BenchMEM}/benchMEM.tpr', + description='Input path for Bench MEM workload', + workload='benchmem') + workload_variable('input_path', default='{BenchRIB}/benchRIB.tpr', + description='Input path for Bench RIB workload', + workload='benchrib') + workload_variable('input_path', default='{BenchPEP_h}/benchPEP-h.tpr', + description='Input path for Bench PEP-h workload', + workload='benchpep_h') + workload_variable('type', default='rf_nvt', + description='Workload type for JCP_benchmarks', + workload='stmv_rf') + workload_variable('type', default='pme_nvt', + description='Workload type for JCP_benchmarks', + workload='stmv_pme') + workload_variable('type', default='grompp', + description='Workload type for JCP_benchmarks', + workloads=['ion_channel', 'rnase_cubic']) + workload_variable('input_path', default='{JCP_benchmarks}/stmv', + description='Input path for JCP_benchmark {workload_name}', + workloads=['stmv_rf', 'stmv_pme']) + workload_variable('input_path', default='{JCP_benchmarks}/{workload_name}', + description='Input path for JCP_benchmark {workload_name}', + workloads=['ion_channel', 'rnase_cubic']) + + log_str = os.path.join(Expander.expansion_str('experiment_run_dir'), + 'md.log') + + figure_of_merit('Core Time', log_file=log_str, + fom_regex=r'\s+Time:\s+(?P[0-9]+\.[0-9]+).*', + group_name='core_time', units='s') + + figure_of_merit('Wall Time', log_file=log_str, + fom_regex=r'\s+Time:\s+[0-9]+\.[0-9]+\s+' + + r'(?P[0-9]+\.[0-9]+).*', + group_name='wall_time', units='s') + + figure_of_merit('Percent Core Time', log_file=log_str, + fom_regex=r'\s+Time:\s+[0-9]+\.[0-9]+\s+[0-9]+\.[0-9]+\s+' + + r'(?P[0-9]+\.[0-9]+).*', + group_name='perc_core_time', units='%') + + figure_of_merit('Nanosecs per day', log_file=log_str, + fom_regex=r'Performance:\s+' + + r'(?P[0-9]+\.[0-9]+).*', + group_name='ns_per_day', units='ns/day') + + figure_of_merit('Hours per nanosec', log_file=log_str, + fom_regex=r'Performance:\s+[0-9]+\.[0-9]+\s+' + + r'(?P[0-9]+\.[0-9]+).*', + group_name='hours_per_ns', units='hours/ns') diff --git a/repo/gromacs/package.py b/repo/gromacs/package.py index 95d6fe61..99f80170 100644 --- a/repo/gromacs/package.py +++ b/repo/gromacs/package.py @@ -274,6 +274,7 @@ class Gromacs(CMakePackage, CudaPackage, ROCmPackage): depends_on("sycl", when="+sycl") depends_on("lapack") depends_on("blas") + depends_on("hipsycl", when="+rocm") depends_on("gcc", when="%oneapi ~intel_provided_gcc") depends_on("gcc", when="%intel ~intel_provided_gcc") @@ -539,13 +540,20 @@ def cmake_args(self): if "+cuda" in self.spec: options.append("-DCUDA_TOOLKIT_ROOT_DIR:STRING=" + self.spec["cuda"].prefix) - options.append("-DGMX_EXTERNAL_LAPACK:BOOL=ON") - if self.spec["lapack"].libs: - options.append("-DGMX_LAPACK_USER={0}".format(self.spec["lapack"].libs.joined(";"))) + target = self.spec.target + if "+cuda" in self.spec and target.family == "ppc64le": + options.append("-DGMX_EXTERNAL_LAPACK:BOOL=OFF") + else: + options.append("-DGMX_EXTERNAL_LAPACK:BOOL=ON") + if self.spec["lapack"].libs: + options.append("-DGMX_LAPACK_USER={0}".format(self.spec["lapack"].libs.joined(";"))) - options.append("-DGMX_EXTERNAL_BLAS:BOOL=ON") - if self.spec["blas"].libs: - options.append("-DGMX_BLAS_USER={0}".format(self.spec["blas"].libs.joined(";"))) + if "+cuda" in self.spec and target.family == "ppc64le": + options.append("-DGMX_EXTERNAL_BLAS:BOOL=OFF") + else: + options.append("-DGMX_EXTERNAL_BLAS:BOOL=ON") + if self.spec["blas"].libs: + options.append("-DGMX_BLAS_USER={0}".format(self.spec["blas"].libs.joined(";"))) if "+cp2k" in self.spec: options.append("-DGMX_CP2K:BOOL=ON") @@ -585,6 +593,8 @@ def cmake_args(self): elif target >= "bulldozer": # AMD Family 15h options.append("-DGMX_SIMD=AVX_128_FMA") + elif target.family == "ppc64le": + options.append("-DGMX_SIMD=None") elif "vsx" in target: # IBM Power 7 and beyond if self.spec.satisfies("%nvhpc"): From 046f0d3567bfd713f5c3fcc1d20c128ef998f270 Mon Sep 17 00:00:00 2001 From: jdomke <28772296+jdomke@users.noreply.github.com> Date: Fri, 23 Feb 2024 12:03:32 +1100 Subject: [PATCH 4/6] Fixing Fugaku configs to enable AMG tests (#148) Co-authored-by: domke <673751-domke@users.noreply.gitlab.com> --- .../auxiliary_software_files/packages.yaml | 5 +++++ .../RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml | 18 ++++++++++-------- .../variables.yaml | 5 +++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml index 93118804..591b98eb 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml @@ -584,3 +584,8 @@ packages: - spec: "pmlib@9.0-trad-power arch=linux-rhel8-a64fx" prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-trad-power buildable: False + cmake: + externals: + - spec: "cmake@3.20.2 arch=linux-rhel8-a64fx" + prefix: /usr + diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml index bf2cf20c..81facd97 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml @@ -6,18 +6,20 @@ spack: packages: default-compiler: - spack_spec: fj@{default_fujitsu_version} + spack_spec: fj@{default_fj_version} default-mpi: - spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} + spack_spec: fujitsu-mpi@{default_fj_version}%fj arch=linux-rhel8-a64fx compiler-gcc: - spack_spec: gcc@12.2.0 + spack_spec: gcc@{default_gnu_version} compiler-fujitsu: - spack_spec: fj@{default_fujitsu_version} + spack_spec: fj@{default_fj_version} blas: - spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} + spack_spec: fujitsu-ssl2@head%fj arch=linux-rhel8-a64fx lapack: - spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} + spack_spec: fujitsu-ssl2@head%fj arch=linux-rhel8-a64fx mpi-gcc: - spack_spec: fujitsu-mpi@head%gcc@12.2.0 + spack_spec: fujitsu-mpi@head%gcc@{default_gnu_version} arch=linux-rhel8-a64fx mpi-fujitsu: - spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} \ No newline at end of file + spack_spec: fujitsu-mpi@{default_fj_version}%fj arch=linux-rhel8-a64fx + gmake: + spack_spec: gmake@4.2.1 arch=linux-rhel8-a64fx diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml index 046ce515..b4db683c 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -9,5 +9,6 @@ variables: batch_submit: 'pjsub {execute_experiment}' batch_nodes: '#PJM -L "node={n_nodes}"' batch_ranks: '#PJM --mpi proc={n_ranks}' - batch_timeout: '#PJM -L "elapse={batch_time}:00"' - default_fujitsu_version: '4.8.1' \ No newline at end of file + batch_timeout: '#PJM -L "elapse={batch_time}:00" -x PJM_LLIO_GFSCACHE="/vol0001:/vol0002:/vol0003:/vol0004:/vol0005:/vol0006"' + default_fj_version: '4.8.1' + default_gnu_version: '12.2.0' From 79fc8bbb2574bff0770ab9f4870594f9a8526588 Mon Sep 17 00:00:00 2001 From: finkandreas Date: Fri, 23 Feb 2024 06:38:46 +0100 Subject: [PATCH 5/6] Add CSCS configs for Daint and Eiger (#147) add CSCS Daint and Eiger Co-authored-by: pearce8 --- .../auxiliary_software_files/compilers.yaml | 111 ++++++++++++++++++ .../auxiliary_software_files/packages.yaml | 71 +++++++++++ .../spack.yaml | 13 ++ .../system_definition.yaml | 32 +++++ .../variables.yaml | 15 +++ .../auxiliary_software_files/packages.yaml | 14 +++ .../spack.yaml | 13 ++ .../system_definition.yaml | 25 ++++ .../variables.yaml | 12 ++ 9 files changed, 306 insertions(+) create mode 100644 configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/compilers.yaml create mode 100644 configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/packages.yaml create mode 100644 configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/spack.yaml create mode 100644 configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/system_definition.yaml create mode 100644 configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/variables.yaml create mode 100644 configs/CSCS-Eiger-HPECray-zen2-Slingshot/auxiliary_software_files/packages.yaml create mode 100644 configs/CSCS-Eiger-HPECray-zen2-Slingshot/spack.yaml create mode 100644 configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml create mode 100644 configs/CSCS-Eiger-HPECray-zen2-Slingshot/variables.yaml diff --git a/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/compilers.yaml new file mode 100644 index 00000000..3b26506a --- /dev/null +++ b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/compilers.yaml @@ -0,0 +1,111 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: cce@12.0.3 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: cnl7 + target: any + modules: + - PrgEnv-cray + - cce/12.0.3 + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@9.3.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: cnl7 + target: any + modules: + - PrgEnv-gnu + - gcc/9.3.0 + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@10.3.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: cnl7 + target: any + modules: + - PrgEnv-gnu + - gcc/10.3.0 + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@11.2.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: cnl7 + target: any + modules: + - PrgEnv-gnu + - gcc/11.2.0 + environment: {} + extra_rpaths: [] +- compiler: + spec: intel@2021.3.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: cnl7 + target: any + modules: + - PrgEnv-intel + - intel/2021.3.0 + environment: {} + extra_rpaths: [] +- compiler: + spec: nvhpc@21.3 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: cnl7 + target: any + modules: + - PrgEnv-nvidia + - nvidia/21.3 + environment: {} + extra_rpaths: [] +- compiler: + spec: pgi@20.1.1 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: cnl7 + target: any + modules: + - PrgEnv-pgi + - pgi/20.1.1 + environment: {} + extra_rpaths: [] diff --git a/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/packages.yaml b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/packages.yaml new file mode 100644 index 00000000..9d85d1a3 --- /dev/null +++ b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/auxiliary_software_files/packages.yaml @@ -0,0 +1,71 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + all: + providers: + mpi: + - cray-mpich + pkgconfig: + - pkg-config + - pkgconf + cuda: + externals: + - spec: cuda@11.0.207 + prefix: /usr/local/cuda-11.0 + - spec: cuda@11.1.0 + prefix: /usr/local/cuda-11.1 + - spec: cuda@11.2.0 + prefix: /usr/local/cuda-11.2 + - spec: cuda@10.2.89 + prefix: /opt/nvidia/cudatoolkit10.2/10.2.89_3.28-2.1__g52c0314 + pkg-config: + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + r: + externals: + - spec: r@4.1.1.0 + modules: + - cray-R/4.1.1.0 + jemalloc: + externals: + - spec: jemalloc@5.1.0.3 + modules: + - cray-jemalloc/5.1.0.3 + cray-libsci: + externals: + - spec: cray-libsci@20.09.1 + modules: + - cray-libsci/20.09.1 + cray-mpich: + externals: + - spec: cray-mpich@7.7.18 + modules: + - cray-mpich/7.7.18 + netcdf-c: + externals: + - spec: netcdf-c@4.7.4.4+mpi+parallel-netcdf + modules: + - cray-netcdf-hdf5parallel/4.7.4.4 + petsc: + externals: + - spec: petsc@3.14.5.0~complex~cuda~int64 + modules: + - cray-petsc/3.14.5.0 + - spec: petsc@3.14.5.0~complex~cuda+int64 + modules: + - cray-petsc-64/3.14.5.0 + - spec: petsc@3.14.5.0+complex~cuda~int64 + modules: + - cray-petsc-complex/3.14.5.0 + - spec: petsc@3.14.5.0+complex~cuda+int64 + modules: + - cray-petsc-complex-64/3.14.5.0 + papi: + externals: + - spec: papi@6.0.0.9 + modules: + - papi/6.0.0.9 diff --git a/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/spack.yaml b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/spack.yaml new file mode 100644 index 00000000..17e5c1e3 --- /dev/null +++ b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/spack.yaml @@ -0,0 +1,13 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: gcc@9.3.0 # cce@12.0.3 + default-mpi: + spack_spec: cray-mpich@7.7.18 + compiler-gcc: + spack_spec: gcc@9.3.0 diff --git a/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/system_definition.yaml b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/system_definition.yaml new file mode 100644 index 00000000..1535acc7 --- /dev/null +++ b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: Piz Daint + site: CSCS + system: HPECray-haswell-P100-Infiniband + integrator: + vendor: HPECray + name: + processor: + vendor: Intel + name: Xeon-E5-2650v3 + ISA: x86_64 + uArch: haswell + accelerator: + vendor: NVIDIA + name: P100 + ISA: PTX + uArch: sm_60 + interconnect: + vendor: HPECray + name: Aries + system-tested: + site: CSCS + name: daint + installation-year: 2017 + description: top500 + top500-system-instances: + - Piz Daint (CSCS) diff --git a/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/variables.yaml b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/variables.yaml new file mode 100644 index 00000000..682ab531 --- /dev/null +++ b/configs/CSCS-Daint-HPECray-haswell-P100-Infiniband/variables.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + batch_time: '02:00' + mpi_command: 'srun -N {n_nodes} -n {n_ranks}' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' + default_cuda_version: '11.2.0' + cuda_arch: '60' + enable_mps: '/usr/tcetmp/bin/enable_mps' diff --git a/configs/CSCS-Eiger-HPECray-zen2-Slingshot/auxiliary_software_files/packages.yaml b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/auxiliary_software_files/packages.yaml new file mode 100644 index 00000000..dfcbd8e4 --- /dev/null +++ b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/auxiliary_software_files/packages.yaml @@ -0,0 +1,14 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + all: + providers: + mpi: + - cray-mpich + cray-mpich: + externals: + - spec: cray-mpich@8.1.28 + prefix: /user-environment/env/default diff --git a/configs/CSCS-Eiger-HPECray-zen2-Slingshot/spack.yaml b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/spack.yaml new file mode 100644 index 00000000..1c055fe5 --- /dev/null +++ b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/spack.yaml @@ -0,0 +1,13 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: gcc@12.3.0 + default-mpi: + spack_spec: cray-mpich@8.1.28 + compiler-gcc: + spack_spec: gcc@12.3.0 diff --git a/configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml new file mode 100644 index 00000000..8c7d0840 --- /dev/null +++ b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml @@ -0,0 +1,25 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: Eiger + site: CSCS + system: HPECray-zen2-Slingshot + integrator: + vendor: HPECray + name: + processor: + vendor: AMD + name: EPYC-7742 + ISA: x86_64 + uArch: zen2 + accelerator: + vendor: + name: + ISA: + uArch: + interconnect: + vendor: HPECray + name: Slingshot diff --git a/configs/CSCS-Eiger-HPECray-zen2-Slingshot/variables.yaml b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/variables.yaml new file mode 100644 index 00000000..4e258c3b --- /dev/null +++ b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/variables.yaml @@ -0,0 +1,12 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + batch_time: '00:30' + mpi_command: 'srun -N {n_nodes} -n {n_ranks}' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' From f203cec85a59e58c39f327a13d288df7514e444f Mon Sep 17 00:00:00 2001 From: pearce8 Date: Fri, 23 Feb 2024 17:42:42 +1100 Subject: [PATCH 6/6] update system definitions doc --- .../system_definition.yaml | 7 ++++ docs/tables/current-system-definitions.csv | 40 +++++++++---------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml index 8c7d0840..68397e95 100644 --- a/configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml +++ b/configs/CSCS-Eiger-HPECray-zen2-Slingshot/system_definition.yaml @@ -23,3 +23,10 @@ system_definition: interconnect: vendor: HPECray name: Slingshot + system-tested: + site: CSCS + name: daint + installation-year: 2017 + description: top500 + top500-system-instances: + - diff --git a/docs/tables/current-system-definitions.csv b/docs/tables/current-system-definitions.csv index e891916b..ccefd6d2 100644 --- a/docs/tables/current-system-definitions.csv +++ b/docs/tables/current-system-definitions.csv @@ -1,20 +1,20 @@ -,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Pascal,Tioga,x86_64 -**site**,LLNL,,,LLNL,RIKEN Center for Computational Science,LLNL,LLNL, -**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,HPECray-zen3-MI250X-Slingshot,x86_64 -**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,Penguin,HPECray, -**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,FX1000,,EX235a, -**processor.vendor**,IBM,AMD,AMD,Intel,Fujitsu,Intel,AMD, -**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,Xeon_E5-2695_v4,EPYC-Zen3, -**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64 -**processor.uArch**,power9,zen4,zen3,icelake,aarch64,broadwell,zen3, -**accelerator.vendor**,NVIDIA,,AMD,,,NVIDIA,AMD, -**accelerator.name**,V100,,MI250X,,,P100,MI250X, -**accelerator.ISA**,PTX,,GCN,,,PTX,GCN, -**accelerator.uArch**,sm_70,,gfx90a,,,sm_56,gfx90a, -**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,Cornelis,HPECray, -**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuInterconnectD,OmniPath,Slingshot11, -**system-tested.site**,LLNL,AWS,LLNL,LLNL,R-CCS,LLNL,LLNL, -**system-tested.name**,lassen,,tioga,magma,Fugaku,pascal,tioga, -**system-tested.installation-year**,2018,,2022,2019,2020,2018,2022, -**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_, -**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (R-CCS),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", +,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Piz Daint,Fugaku,Pascal,Eiger,Tioga,x86_64 +**site**,LLNL,,,LLNL,CSCS,RIKEN Center for Computational Science,LLNL,CSCS,LLNL, +**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,HPECray-haswell-P100-Infiniband,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,HPECray-zen2-Slingshot,HPECray-zen3-MI250X-Slingshot,x86_64 +**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,HPECray,Fujitsu,Penguin,HPECray,HPECray, +**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,,FX1000,,,EX235a, +**processor.vendor**,IBM,AMD,AMD,Intel,Intel,Fujitsu,Intel,AMD,AMD, +**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,Xeon-E5-2650v3,A64FX,Xeon_E5-2695_v4,EPYC-7742,EPYC-Zen3, +**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64,x86_64 +**processor.uArch**,power9,zen4,zen3,icelake,haswell,aarch64,broadwell,zen2,zen3, +**accelerator.vendor**,NVIDIA,,AMD,,NVIDIA,,NVIDIA,,AMD, +**accelerator.name**,V100,,MI250X,,P100,,P100,,MI250X, +**accelerator.ISA**,PTX,,GCN,,PTX,,PTX,,GCN, +**accelerator.uArch**,sm_70,,gfx90a,,sm_60,,sm_56,,gfx90a, +**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,HPECray,Fujitsu,Cornelis,HPECray,HPECray, +**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,Aries,TofuInterconnectD,OmniPath,Slingshot,Slingshot11, +**system-tested.site**,LLNL,AWS,LLNL,LLNL,CSCS,R-CCS,LLNL,CSCS,LLNL, +**system-tested.name**,lassen,,tioga,magma,daint,Fugaku,pascal,daint,tioga, +**system-tested.installation-year**,2018,,2022,2019,2017,2020,2018,2017,2022, +**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_,`top500 `_, +**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Piz Daint (CSCS),Fugaku (R-CCS),,,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",