Starting ElCap config

configs/LLNL-ElCapitan-HPECray-zen4-MI300A-Slingshot/auxiliary_software_files/compilers.yaml

@@ -0,0 +1,51 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc:  /opt/cray/pe/cce/16.0.0/bin/craycc
+      cxx:  /opt/cray/pe/cce/16.0.0/bin/crayCC
+      f77:  /opt/cray/pe/cce/16.0.0/bin/crayftn
+      fc:  /opt/cray/pe/cce/16.0.0/bin/crayftn
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++17
+      fflags: -g -O2 -hnopattern
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: [/opt/cray/pe/gcc-libs/]
+- compiler:
+    spec: [email protected]
+    paths:
+      cc:  /opt/rocm-5.5.1/bin/amdclang
+      cxx:  /opt/rocm-5.5.1/bin/amdclang++
+      f77: /opt/rocm-5.5.1/bin/amdflang 
+      fc:  /opt/rocm-5.5.1/bin/amdflang
+    flags:
+      cflags: -g -O2 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr
+      cxxflags: -g -O2 -std=c++17 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr
+      fflags: -g -O2 -hnopattern
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /opt/cray/pe/gcc/12.2.0/bin/gcc
+      cxx: /opt/cray/pe/gcc/12.2.0/bin/g++
+      f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran
+      fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []

configs/LLNL-ElCapitan-HPECray-zen4-MI300A-Slingshot/auxiliary_software_files/packages.yaml

@@ -0,0 +1,236 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  all:
+    require: 'target=x86_64:'
+    variants: amdgpu_target=gfx90a
+  tar:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  coreutils:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  libtool:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  flex:
+    externals:
+    - spec: [email protected]+lex
+      prefix: /usr
+  openssl:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  m4:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  groff:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  cmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+    - spec: [email protected]
+      prefix: /usr/tce
+  pkgconf:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  curl:
+    externals:
+    - spec: [email protected]+gssapi+ldap+nghttp2
+      prefix: /usr
+  gmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  subversion:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  diffutils:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  swig:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  gawk:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  binutils:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  findutils:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  git-lfs:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce
+  ccache:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  automake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  cvs:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  git:
+    externals:
+    - spec: [email protected]+tcltk
+      prefix: /usr
+    - spec: [email protected]+tcltk
+      prefix: /usr/tce
+  openssh:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  autoconf:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  texinfo:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  bison:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  python:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/python/python-3.9.12
+      buildable: false
+  rocprim:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+      buildable: false
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+      buildable: false
+  rocrand:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/hiprand
+      buildable: false
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/hiprand
+      buildable: false
+  rocsparse:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+      buildable: false
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+      buildable: false
+  rocthrust:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+    buildable: false
+  hip:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+    buildable: false
+  hsa-rocr-dev:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+      buildable: false
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+      buildable: false
+  llvm-amdgpu:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/llvm
+      buildable: false
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/llvm
+      buildable: false
+  rocblas:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+      buildable: false
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+      buildable: false
+  rocsolver:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+      buildable: false
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+      buildable: false
+  cray-libsci:
+    externals:
+    - spec: [email protected]%cce
+      prefix: /opt/cray/pe/libsci/23.05.1.4/cray/12.0/x86_64/
+    - spec: [email protected]%gcc
+      prefix: /opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/
+  lapack:
+    buildable: false
+  hypre:
+    variants: amdgpu_target=gfx90a
+  hwloc:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+      buildable: false
+  fftw:
+    buildable: false
+  intel-oneapi-mkl:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/intel/oneapi
+    buildable: false
+  mpi:
+    buildable: false
+  cray-mpich:
+    buildable: false
+    externals:
+    - spec: [email protected]%cce +gtl +wrappers
+      prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0
+      extra_attributes:
+        gtl_cutoff_size: 4096
+        fi_cxi_ats: 0
+        gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib
+        ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib -lmpi_gtl_hsa"
+    - spec: [email protected]%cce ~gtl +wrappers
+      prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0
+      extra_attributes:
+        gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib
+        ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib"
+    - spec: [email protected]%gcc ~gtl +wrappers
+      prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/10.3
+      extra_attributes:
+        gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib
+        ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib"

configs/LLNL-ElCapitan-HPECray-zen4-MI300A-Slingshot/spack.yaml

@@ -0,0 +1,31 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+spack:
+  packages:
+    default-compiler:
+      spack_spec: cce@16
+    default-mpi:
+      spack_spec: [email protected]%cce ~gtl
+    compiler-rocm:
+      spack_spec: cce@16
+    compiler-amdclang:
+      spack_spec: [email protected]
+    blas-rocm:
+      spack_spec: [email protected]
+    blas:
+      spack_spec: [email protected]
+    lapack-rocm:
+      spack_spec: [email protected]
+    lapack:
+      spack_spec: cray-libsci@23
+    mpi-rocm-gtl:
+      spack_spec: [email protected]%cce +gtl
+    mpi-rocm-no-gtl:
+      spack_spec: [email protected]%cce ~gtl
+    mpi-gcc:
+      spack_spec: [email protected]%gcc ~gtl
+    fftw:
+      spack_spec: [email protected] 

configs/LLNL-ElCapitan-HPECray-zen4-MI300A-Slingshot/system_definition.yaml

@@ -0,0 +1,32 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+system_definition:
+  name: El Capitan
+  site: LLNL
+  system: HPECray-zen4-MI300A-Slingshot
+  integrator:
+    vendor: HPECray
+    name: EX235a
+  processor:
+    vendor: AMD
+    name: EPYC-Zen4
+    ISA: x86_64
+    uArch: zen4
+  accelerator:
+    vendor: AMD
+    name: MI300A
+    ISA: GCN
+    uArch: gfx90a
+  interconnect:
+    vendor: HPECray
+    name: Slingshot11
+  system-tested:
+    site: LLNL
+    name: El Capitan
+    installation-year: 2024
+    description: top500 <https://www.top500.org/system/tbd>
+  top500-system-instances:
+    - El Capitan    (LLNL)

configs/LLNL-ElCapitan-HPECray-zen4-MI300A-Slingshot/variables.yaml

@@ -0,0 +1,14 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  gtl_flag: ''  # to be overwritten by tests that need GTL
+  rocm_arch: 'gfx90a'
+  batch_time: '120m'
+  mpi_command: 'flux run -N {n_nodes} -n {n_ranks}'
+  batch_submit: 'flux batch {execute_experiment}'
+  batch_nodes: '# flux: -N {n_nodes}'
+  batch_ranks: '# flux: -n {n_ranks}'
+  batch_timeout: '# flux: -t {batch_time}'