From 55f849f536957d69adb68b3ba7ec38051392eba5 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Thu, 8 Feb 2024 13:11:12 +0900 Subject: [PATCH] Starting a LLNL-Pascal config (#119) * Starting a LLNL-Pascal config * initial config for pascal * add gcc (for raja-perf blt build); cuda10 components + def cuda10_version * fix indentation in compilers.yaml; add needed variables (referenced by amg2023 experiment application) * rocm_arch not needed; cuda_arch is needed * what I thought was an intel-mkl install was in fact an intel-oneapi-install; also, the config depends on cublas being defined (although I don't think it can be used given that intel-oneapi-mkl is the blas provider as well as the lapack provider) * wrong variable name * use slurm scheduler * get rid of cuda10 references; replace intel compiler with gcc * adjust arch and force cuda_arch for amg2023 --------- Co-authored-by: Peter Josef Scheibel --- .../auxiliary_software_files/compilers.yaml | 19 ++++++ .../auxiliary_software_files/packages.yaml | 64 +++++++++++++++++++ .../spack.yaml | 17 +++++ .../system_definition.yaml | 32 ++++++++++ .../variables.yaml | 15 +++++ experiments/amg2023/cuda/ramble.yaml | 2 +- 6 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml create mode 100644 configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml create mode 100644 configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml create mode 100644 configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/system_definition.yaml create mode 100644 configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml new file mode 100644 index 000000000..d009ca708 --- /dev/null +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: gcc@=11.2.1 + paths: + cc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-11.2.1/bin/g++ + f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml new file mode 100644 index 000000000..eea394721 --- /dev/null +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml @@ -0,0 +1,64 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + tar: + externals: + - spec: tar@1.30 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.23.1 + prefix: /usr/tce/packages/cmake/cmake-3.23.1 + buildable: false + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr/tcetmp/packages/gmake/gmake-4.2.1 + buildable: false + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + mpi: + buildable: false + mvapich2: + buildable: false + externals: + - spec: mvapich2@2.3.7%intel@2021.6.0 + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/ + cuda: + buildable: false + externals: + - spec: cuda@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ + curand: + externals: + - spec: curand@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0 + buildable: false + cusparse: + externals: + - spec: cusparse@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0 + buildable: false + cublas: + externals: + - spec: cublas@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0 + buildable: false + blas: + buildable: false + lapack: + buildable: false + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.1.0 + prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml new file mode 100644 index 000000000..0a1d794df --- /dev/null +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: gcc@=11.2.1 + default-mpi: + spack_spec: mvapich2@2.3.7 + compiler-gcc: + spack_spec: gcc@=11.2.1 + cublas-cuda: + spack_spec: cublas@{default_cuda_version} + lapack: + spack_spec: intel-oneapi-mkl@2022.1.0 diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/system_definition.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/system_definition.yaml new file mode 100644 index 000000000..01d1688fe --- /dev/null +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: Pascal + site: LLNL + system: Penguin-broadwell-P100-OmniPath + integrator: + vendor: Penguin + name: + processor: + vendor: Intel + name: Xeon_E5-2695_v4 + ISA: x86_64 + uArch: broadwell + accelerator: + vendor: NVIDIA + name: P100 + ISA: PTX + uArch: sm_56 + interconnect: + vendor: Cornelis + name: OmniPath + system-tested: + site: LLNL + name: pascal + installation-year: 2018 + description: + top500-system-instances: + - diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml new file mode 100644 index 000000000..462225a52 --- /dev/null +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + gtl_flag: '' # to be overwritten by tests that need GTL + cuda_arch: '60' + default_cuda_version: '11.8.0' + batch_time: '02:00' + mpi_command: 'srun -N {n_nodes} -n {n_ranks}' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' diff --git a/experiments/amg2023/cuda/ramble.yaml b/experiments/amg2023/cuda/ramble.yaml index 7594790f1..feb5fa751 100644 --- a/experiments/amg2023/cuda/ramble.yaml +++ b/experiments/amg2023/cuda/ramble.yaml @@ -46,7 +46,7 @@ ramble: spack_spec: hypre@2.28.0 +mpi+cuda+mixedint cuda_arch=={cuda_arch} ^cuda@{default_cuda_version} compiler: default-compiler amg2023: - spack_spec: amg2023@develop +mpi+cuda ^cuda@{default_cuda_version} + spack_spec: amg2023@develop +mpi+cuda cuda_arch=={cuda_arch} ^cuda@{default_cuda_version} compiler: default-compiler environments: amg2023: