From cd5616dce9ea5c7fc270d41b2b6d2b3b2d755e0e Mon Sep 17 00:00:00 2001 From: penke3 Date: Fri, 1 Mar 2024 09:34:24 +0100 Subject: [PATCH 01/10] it compiles! --- .../auxiliary_software_files/compilers.yaml | 42 ++++++++++ .../auxiliary_software_files/packages.yaml | 84 +++++++++++++++++++ configs/JSC-JUWELS-BOOSTER/spack.yaml | 17 ++++ .../JSC-JUWELS-BOOSTER/system_definition.yaml | 32 +++++++ configs/JSC-JUWELS-BOOSTER/variables.yaml | 15 ++++ 5 files changed, 190 insertions(+) create mode 100644 configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml create mode 100644 configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml create mode 100644 configs/JSC-JUWELS-BOOSTER/spack.yaml create mode 100644 configs/JSC-JUWELS-BOOSTER/system_definition.yaml create mode 100644 configs/JSC-JUWELS-BOOSTER/variables.yaml diff --git a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml new file mode 100644 index 00000000..05bfa67a --- /dev/null +++ b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml @@ -0,0 +1,42 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: nvhpc@=23.7 + paths: + cc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvc + cxx: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvc++ + f77: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran + fc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran + # flags: + # cflags: -fPIC + # cppflags: -fPIC + # fflags: -fPIC + operating_system: rocky9 + target: aarch64 + modules: + - Stages/2024 + - NVHPC/23.7 + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=12.3.0 + paths: + cc: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gcc + cxx: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/g++ + f77: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gfortran + fc: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gfortran + # flags: + # cflags: -fPIC + # cppflags: -fPIC + # fflags: -fPIC + operating_system: rocky9 + target: aarch64 + modules: + - Stages/2024 + - GCC/12.3.0 + environment: {} + extra_rpaths: [] diff --git a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml new file mode 100644 index 00000000..112b0d93 --- /dev/null +++ b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml @@ -0,0 +1,84 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. + +# SPDX-License-Identifier: Apache-2.0 + +packages: + tar: + externals: + - spec: tar@1.30 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.26.3 + prefix: /p/software/juwelsbooster/stages/2024/software/CMake/3.26.3-GCCcore-12.3.0 + modules: + - Stages/2024 + - CMake + buildable: false + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.16.5 + prefix: /p/software/juwelsbooster/stages/2024/software/Automake/1.16.5-GCCcore-12.3.0 + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /p/software/juwelsbooster/stages/2024/software/Autoconf/2.71-GCCcore-12.3.0 + openmpi: + externals: + # - spec: openmpi@4.1.5%nvhpc@=23.7+cuda~cxx~cxx_exceptions~java~memchecker~pmi~static~wrapper-rpath + # fabrics=ucx + - spec: openmpi@4.1.5 + prefix: /p/software/juwelsbooster/stages/2024/software/OpenMPI/4.1.5-NVHPC-23.7-CUDA-12 + modules: + - Stages/2024 + - NVHPC/23.7-CUDA-12 + - OpenMPI/4.1.5 + buildable: false + cuda: + buildable: false + externals: + - spec: cuda@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + modules: + - Stages/2024 + - CUDA/12 + - NVHPC/23.7-CUDA-12 + curand: + externals: + - spec: curand@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + buildable: false + cusparse: + externals: + - spec: cusparse@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + buildable: false + cublas: + externals: + - spec: cublas@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + buildable: false + blas: + buildable: false + lapack: + buildable: false + openblas: + externals: + - spec: openblas@0.3.23%gcc@12.3.0 + prefix: /p/software/juwelsbooster/stages/2024/software/OpenBLAS/0.3.23-GCC-12.3.0 + all: + providers: + mpi: + - openmpi + zlib-api: [zlib] + zlib: + externals: + - spec: zlib@1.2.13 + prefix: /p/software/juwelsbooster/stages/2024/software/zlib/1.2.13-GCCcore-12.3.0 diff --git a/configs/JSC-JUWELS-BOOSTER/spack.yaml b/configs/JSC-JUWELS-BOOSTER/spack.yaml new file mode 100644 index 00000000..9d1c4565 --- /dev/null +++ b/configs/JSC-JUWELS-BOOSTER/spack.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: gcc@=12.3.0 + default-mpi: + spack_spec: openmpi@4.1.5 + compiler-gcc: + spack_spec: gcc@=12.3.0 + cublas-cuda: + spack_spec: cublas@{default_cuda_version} + lapack: + spack_spec: openblas@0.3.23 \ No newline at end of file diff --git a/configs/JSC-JUWELS-BOOSTER/system_definition.yaml b/configs/JSC-JUWELS-BOOSTER/system_definition.yaml new file mode 100644 index 00000000..4e2e5c97 --- /dev/null +++ b/configs/JSC-JUWELS-BOOSTER/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: JUWELS + site: + system: + integrator: + vendor: + name: + processor: + vendor: + name: + ISA: x86_64 + uArch: + accelerator: + vendor: NVIDIA + name: A100 + ISA: PTX + uArch: sm_80 + interconnect: + vendor: Mellanox + name: Infiniband + system-tested: + site: JSC + name: + installation-year: 2020 + description: + top500-system-instances: + - diff --git a/configs/JSC-JUWELS-BOOSTER/variables.yaml b/configs/JSC-JUWELS-BOOSTER/variables.yaml new file mode 100644 index 00000000..b92f093b --- /dev/null +++ b/configs/JSC-JUWELS-BOOSTER/variables.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + gtl_flag: '' # to be overwritten by tests that need GTL + cuda_arch: '80' + default_cuda_version: '12.2.0' + batch_time: '02:00' + mpi_command: 'srun -N {n_nodes} -n {n_ranks}' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' From 556289eb29b6aaa5a0fb7fd571e211ff7e660118 Mon Sep 17 00:00:00 2001 From: penke3 Date: Tue, 5 Mar 2024 15:14:22 +0100 Subject: [PATCH 02/10] hpl compiles --- .../JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml | 3 +++ configs/JSC-JUWELS-BOOSTER/spack.yaml | 2 ++ configs/JSC-JUWELS-BOOSTER/variables.yaml | 2 +- experiments/saxpy/cuda/ramble.yaml | 4 ++-- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml index 112b0d93..88c0c5f7 100644 --- a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml +++ b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml @@ -73,6 +73,9 @@ packages: externals: - spec: openblas@0.3.23%gcc@12.3.0 prefix: /p/software/juwelsbooster/stages/2024/software/OpenBLAS/0.3.23-GCC-12.3.0 + modules: + - Stages/2024 + - OpenBLAS all: providers: mpi: diff --git a/configs/JSC-JUWELS-BOOSTER/spack.yaml b/configs/JSC-JUWELS-BOOSTER/spack.yaml index 9d1c4565..6dc5bfab 100644 --- a/configs/JSC-JUWELS-BOOSTER/spack.yaml +++ b/configs/JSC-JUWELS-BOOSTER/spack.yaml @@ -13,5 +13,7 @@ spack: spack_spec: gcc@=12.3.0 cublas-cuda: spack_spec: cublas@{default_cuda_version} + blas: + spack_spec: openblas@0.3.23 lapack: spack_spec: openblas@0.3.23 \ No newline at end of file diff --git a/configs/JSC-JUWELS-BOOSTER/variables.yaml b/configs/JSC-JUWELS-BOOSTER/variables.yaml index b92f093b..70e446f8 100644 --- a/configs/JSC-JUWELS-BOOSTER/variables.yaml +++ b/configs/JSC-JUWELS-BOOSTER/variables.yaml @@ -7,7 +7,7 @@ variables: gtl_flag: '' # to be overwritten by tests that need GTL cuda_arch: '80' default_cuda_version: '12.2.0' - batch_time: '02:00' + batch_time: '00:05' mpi_command: 'srun -N {n_nodes} -n {n_ranks}' batch_submit: 'sbatch {execute_experiment}' batch_nodes: '#SBATCH -N {n_nodes}' diff --git a/experiments/saxpy/cuda/ramble.yaml b/experiments/saxpy/cuda/ramble.yaml index d9d15703..83483ffd 100644 --- a/experiments/saxpy/cuda/ramble.yaml +++ b/experiments/saxpy/cuda/ramble.yaml @@ -19,8 +19,8 @@ ramble: saxpy_{n}_{n_nodes}_{processes_per_node}_{n_ranks}: variables: processes_per_node: '4' - n_nodes: '1' - n: ['128', '256', '512', '1024'] + n_nodes: ['1','2'] + n: ['512', '1024'] matrix: - n spack: From f14d03783aa43ff7876ecdfe9dd3ebcc4f30460a Mon Sep 17 00:00:00 2001 From: penke3 Date: Wed, 6 Mar 2024 10:54:19 +0100 Subject: [PATCH 03/10] clean up JUWELS Booster --- .../auxiliary_software_files/compilers.yaml | 34 ++++++++ .../auxiliary_software_files/packages.yaml | 85 +++++++++++++++++++ .../spack.yaml | 19 +++++ .../system_definition.yaml | 32 +++++++ .../variables.yaml | 15 ++++ 5 files changed, 185 insertions(+) create mode 100644 configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml create mode 100644 configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/packages.yaml create mode 100644 configs/JSC-JUWELS-Booster-rome-A100-Infiniband/spack.yaml create mode 100644 configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml create mode 100644 configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml new file mode 100644 index 00000000..d72080d3 --- /dev/null +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml @@ -0,0 +1,34 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: nvhpc@=23.7 + paths: + cc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvc + cxx: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvc++ + f77: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran + fc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran + operating_system: rocky9 + target: aarch64 + modules: + - Stages/2024 + - NVHPC/23.7 + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=12.3.0 + paths: + cc: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gcc + cxx: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/g++ + f77: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gfortran + fc: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gfortran + operating_system: rocky9 + target: aarch64 + modules: + - Stages/2024 + - GCC/12.3.0 + environment: {} + extra_rpaths: [] diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/packages.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/packages.yaml new file mode 100644 index 00000000..724df81c --- /dev/null +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/packages.yaml @@ -0,0 +1,85 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. + +# SPDX-License-Identifier: Apache-2.0 + +packages: + tar: + externals: + - spec: tar@1.30 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.26.3 + prefix: /p/software/juwelsbooster/stages/2024/software/CMake/3.26.3-GCCcore-12.3.0 + modules: + - Stages/2024 + - CMake + buildable: false + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.16.5 + prefix: /p/software/juwelsbooster/stages/2024/software/Automake/1.16.5-GCCcore-12.3.0 + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /p/software/juwelsbooster/stages/2024/software/Autoconf/2.71-GCCcore-12.3.0 + openmpi: + externals: + - spec: openmpi@4.1.5 + prefix: /p/software/juwelsbooster/stages/2024/software/OpenMPI/4.1.5-NVHPC-23.7-CUDA-12 + modules: + - Stages/2024 + - NVHPC/23.7-CUDA-12 + - OpenMPI/4.1.5 + buildable: false + cuda: + buildable: false + externals: + - spec: cuda@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + modules: + - Stages/2024 + - CUDA/12 + - NVHPC/23.7-CUDA-12 + curand: + externals: + - spec: curand@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + buildable: false + cusparse: + externals: + - spec: cusparse@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + buildable: false + cublas: + externals: + - spec: cublas@12.2.0 + prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 + buildable: false + blas: + buildable: false + lapack: + buildable: false + openblas: + externals: + - spec: openblas@0.3.23%gcc@12.3.0 + prefix: /p/software/juwelsbooster/stages/2024/software/OpenBLAS/0.3.23-GCC-12.3.0 + modules: + - Stages/2024 + - OpenBLAS + all: + providers: + mpi: + - openmpi + zlib-api: [zlib] + zlib: + externals: + - spec: zlib@1.2.13 + prefix: /p/software/juwelsbooster/stages/2024/software/zlib/1.2.13-GCCcore-12.3.0 \ No newline at end of file diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/spack.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/spack.yaml new file mode 100644 index 00000000..6dc5bfab --- /dev/null +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/spack.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: gcc@=12.3.0 + default-mpi: + spack_spec: openmpi@4.1.5 + compiler-gcc: + spack_spec: gcc@=12.3.0 + cublas-cuda: + spack_spec: cublas@{default_cuda_version} + blas: + spack_spec: openblas@0.3.23 + lapack: + spack_spec: openblas@0.3.23 \ No newline at end of file diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml new file mode 100644 index 00000000..3355b1d7 --- /dev/null +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: JUWELS-Booster + site: JSC + system: JUWELS-Booster-rome-A100-Infiniband + integrator: + vendor: tba + name: tba + processor: + vendor: AMD + name: EPYC-Zen2 + ISA: x86_64 + uArch: zen2 + accelerator: + vendor: NVIDIA + name: A100 + ISA: PTX + uArch: sm_80 + interconnect: + vendor: Mellanox + name: Infiniband + system-tested: + site: JSC + name: JUWELS-Booster + installation-year: 2020 + description: top500 + top500-system-instances: + - JUWELS Booster (JSC) diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml new file mode 100644 index 00000000..b92f093b --- /dev/null +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + gtl_flag: '' # to be overwritten by tests that need GTL + cuda_arch: '80' + default_cuda_version: '12.2.0' + batch_time: '02:00' + mpi_command: 'srun -N {n_nodes} -n {n_ranks}' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' From ca6571e427f0ddd648628a0a954a01793ff967e0 Mon Sep 17 00:00:00 2001 From: penke3 Date: Wed, 6 Mar 2024 11:00:30 +0100 Subject: [PATCH 04/10] cleanup JUWELS Booster --- .../auxiliary_software_files/compilers.yaml | 42 --------- .../auxiliary_software_files/packages.yaml | 87 ------------------- configs/JSC-JUWELS-BOOSTER/spack.yaml | 19 ---- .../JSC-JUWELS-BOOSTER/system_definition.yaml | 32 ------- configs/JSC-JUWELS-BOOSTER/variables.yaml | 15 ---- 5 files changed, 195 deletions(-) delete mode 100644 configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml delete mode 100644 configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml delete mode 100644 configs/JSC-JUWELS-BOOSTER/spack.yaml delete mode 100644 configs/JSC-JUWELS-BOOSTER/system_definition.yaml delete mode 100644 configs/JSC-JUWELS-BOOSTER/variables.yaml diff --git a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml deleted file mode 100644 index 05bfa67a..00000000 --- a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/compilers.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -compilers: -- compiler: - spec: nvhpc@=23.7 - paths: - cc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvc - cxx: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvc++ - f77: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran - fc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran - # flags: - # cflags: -fPIC - # cppflags: -fPIC - # fflags: -fPIC - operating_system: rocky9 - target: aarch64 - modules: - - Stages/2024 - - NVHPC/23.7 - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@=12.3.0 - paths: - cc: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gcc - cxx: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/g++ - f77: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gfortran - fc: /p/software/juwelsbooster/stages/2024/software/GCCcore/12.3.0/bin/gfortran - # flags: - # cflags: -fPIC - # cppflags: -fPIC - # fflags: -fPIC - operating_system: rocky9 - target: aarch64 - modules: - - Stages/2024 - - GCC/12.3.0 - environment: {} - extra_rpaths: [] diff --git a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml b/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml deleted file mode 100644 index 88c0c5f7..00000000 --- a/configs/JSC-JUWELS-BOOSTER/auxiliary_software_files/packages.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. - -# SPDX-License-Identifier: Apache-2.0 - -packages: - tar: - externals: - - spec: tar@1.30 - prefix: /usr - buildable: false - cmake: - externals: - - spec: cmake@3.26.3 - prefix: /p/software/juwelsbooster/stages/2024/software/CMake/3.26.3-GCCcore-12.3.0 - modules: - - Stages/2024 - - CMake - buildable: false - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - buildable: false - automake: - externals: - - spec: automake@1.16.5 - prefix: /p/software/juwelsbooster/stages/2024/software/Automake/1.16.5-GCCcore-12.3.0 - autoconf: - externals: - - spec: autoconf@2.71 - prefix: /p/software/juwelsbooster/stages/2024/software/Autoconf/2.71-GCCcore-12.3.0 - openmpi: - externals: - # - spec: openmpi@4.1.5%nvhpc@=23.7+cuda~cxx~cxx_exceptions~java~memchecker~pmi~static~wrapper-rpath - # fabrics=ucx - - spec: openmpi@4.1.5 - prefix: /p/software/juwelsbooster/stages/2024/software/OpenMPI/4.1.5-NVHPC-23.7-CUDA-12 - modules: - - Stages/2024 - - NVHPC/23.7-CUDA-12 - - OpenMPI/4.1.5 - buildable: false - cuda: - buildable: false - externals: - - spec: cuda@12.2.0 - prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 - modules: - - Stages/2024 - - CUDA/12 - - NVHPC/23.7-CUDA-12 - curand: - externals: - - spec: curand@12.2.0 - prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 - buildable: false - cusparse: - externals: - - spec: cusparse@12.2.0 - prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 - buildable: false - cublas: - externals: - - spec: cublas@12.2.0 - prefix: /p/software/juwelsbooster/stages/2024/software/CUDA/12 - buildable: false - blas: - buildable: false - lapack: - buildable: false - openblas: - externals: - - spec: openblas@0.3.23%gcc@12.3.0 - prefix: /p/software/juwelsbooster/stages/2024/software/OpenBLAS/0.3.23-GCC-12.3.0 - modules: - - Stages/2024 - - OpenBLAS - all: - providers: - mpi: - - openmpi - zlib-api: [zlib] - zlib: - externals: - - spec: zlib@1.2.13 - prefix: /p/software/juwelsbooster/stages/2024/software/zlib/1.2.13-GCCcore-12.3.0 diff --git a/configs/JSC-JUWELS-BOOSTER/spack.yaml b/configs/JSC-JUWELS-BOOSTER/spack.yaml deleted file mode 100644 index 6dc5bfab..00000000 --- a/configs/JSC-JUWELS-BOOSTER/spack.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -spack: - packages: - default-compiler: - spack_spec: gcc@=12.3.0 - default-mpi: - spack_spec: openmpi@4.1.5 - compiler-gcc: - spack_spec: gcc@=12.3.0 - cublas-cuda: - spack_spec: cublas@{default_cuda_version} - blas: - spack_spec: openblas@0.3.23 - lapack: - spack_spec: openblas@0.3.23 \ No newline at end of file diff --git a/configs/JSC-JUWELS-BOOSTER/system_definition.yaml b/configs/JSC-JUWELS-BOOSTER/system_definition.yaml deleted file mode 100644 index 4e2e5c97..00000000 --- a/configs/JSC-JUWELS-BOOSTER/system_definition.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -system_definition: - name: JUWELS - site: - system: - integrator: - vendor: - name: - processor: - vendor: - name: - ISA: x86_64 - uArch: - accelerator: - vendor: NVIDIA - name: A100 - ISA: PTX - uArch: sm_80 - interconnect: - vendor: Mellanox - name: Infiniband - system-tested: - site: JSC - name: - installation-year: 2020 - description: - top500-system-instances: - - diff --git a/configs/JSC-JUWELS-BOOSTER/variables.yaml b/configs/JSC-JUWELS-BOOSTER/variables.yaml deleted file mode 100644 index 70e446f8..00000000 --- a/configs/JSC-JUWELS-BOOSTER/variables.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -variables: - gtl_flag: '' # to be overwritten by tests that need GTL - cuda_arch: '80' - default_cuda_version: '12.2.0' - batch_time: '00:05' - mpi_command: 'srun -N {n_nodes} -n {n_ranks}' - batch_submit: 'sbatch {execute_experiment}' - batch_nodes: '#SBATCH -N {n_nodes}' - batch_ranks: '#SBATCH -n {n_ranks}' - batch_timeout: '#SBATCH -t {batch_time}:00' From 45c7578b65c41b8bdde10beb52b1352cd4f460a4 Mon Sep 17 00:00:00 2001 From: penke3 Date: Wed, 6 Mar 2024 11:18:11 +0100 Subject: [PATCH 05/10] back to original experiment config --- experiments/saxpy/cuda/ramble.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/saxpy/cuda/ramble.yaml b/experiments/saxpy/cuda/ramble.yaml index 83483ffd..d9d15703 100644 --- a/experiments/saxpy/cuda/ramble.yaml +++ b/experiments/saxpy/cuda/ramble.yaml @@ -19,8 +19,8 @@ ramble: saxpy_{n}_{n_nodes}_{processes_per_node}_{n_ranks}: variables: processes_per_node: '4' - n_nodes: ['1','2'] - n: ['512', '1024'] + n_nodes: '1' + n: ['128', '256', '512', '1024'] matrix: - n spack: From 569ec41b5830177e20872ffa10ec8c8e98fa9c22 Mon Sep 17 00:00:00 2001 From: caropen <69966551+caropen@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:46:36 +0100 Subject: [PATCH 06/10] Update configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml Co-authored-by: Andreas Herten --- .../system_definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml index 3355b1d7..359e334c 100644 --- a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml @@ -8,7 +8,7 @@ system_definition: site: JSC system: JUWELS-Booster-rome-A100-Infiniband integrator: - vendor: tba + vendor: Atos name: tba processor: vendor: AMD From 24eb2fee05aa281eb0595a1319d3e370920dcb86 Mon Sep 17 00:00:00 2001 From: caropen <69966551+caropen@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:46:52 +0100 Subject: [PATCH 07/10] Update configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml Co-authored-by: SebastianAchilles --- .../auxiliary_software_files/compilers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml index d72080d3..f722368a 100644 --- a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml @@ -12,7 +12,7 @@ compilers: f77: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran fc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran operating_system: rocky9 - target: aarch64 + target: x86_64 modules: - Stages/2024 - NVHPC/23.7 From 7ab085e30bf5ec56c6111b6222b66cf20106ce84 Mon Sep 17 00:00:00 2001 From: caropen <69966551+caropen@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:47:04 +0100 Subject: [PATCH 08/10] Update configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml Co-authored-by: SebastianAchilles --- .../auxiliary_software_files/compilers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml index f722368a..00f0e47d 100644 --- a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/auxiliary_software_files/compilers.yaml @@ -11,7 +11,7 @@ compilers: cxx: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvc++ f77: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran fc: /p/software/juwelsbooster/stages/2024/software/NVHPC/23.7-CUDA-12/Linux_aarch64/23.7/compilers/bin/nvfortran - operating_system: rocky9 + operating_system: rocky8 target: x86_64 modules: - Stages/2024 From b8a2b524b9f3c5ccb558539d0ef43a843914d636 Mon Sep 17 00:00:00 2001 From: caropen <69966551+caropen@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:41:27 +0200 Subject: [PATCH 09/10] Update configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml Co-authored-by: Andreas Herten --- .../system_definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml index 359e334c..740fbff7 100644 --- a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/system_definition.yaml @@ -9,7 +9,7 @@ system_definition: system: JUWELS-Booster-rome-A100-Infiniband integrator: vendor: Atos - name: tba + name: XH2000 processor: vendor: AMD name: EPYC-Zen2 From 8d9220393e19080dcb3722389088ce6553bb7dde Mon Sep 17 00:00:00 2001 From: pearce8 Date: Thu, 20 Jun 2024 10:41:12 -0500 Subject: [PATCH 10/10] Update variables.yaml to work with the allocation modifier --- .../variables.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml index b92f093b..a07feeba 100644 --- a/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml +++ b/configs/JSC-JUWELS-Booster-rome-A100-Infiniband/variables.yaml @@ -7,9 +7,12 @@ variables: gtl_flag: '' # to be overwritten by tests that need GTL cuda_arch: '80' default_cuda_version: '12.2.0' - batch_time: '02:00' - mpi_command: 'srun -N {n_nodes} -n {n_ranks}' - batch_submit: 'sbatch {execute_experiment}' - batch_nodes: '#SBATCH -N {n_nodes}' - batch_ranks: '#SBATCH -n {n_ranks}' - batch_timeout: '#SBATCH -t {batch_time}:00' + timeout: "120" + scheduler: "slurm" + sys_cores_per_node: "48" + sys_gpus_per_node: "4" + max_request: "1000" # n_ranks/n_nodes cannot exceed this + n_ranks: '1000001' # placeholder value + n_nodes: '1000001' # placeholder value + batch_submit: "placeholder" + mpi_command: "placeholder"