Merge branch 'develop' into feature_thicket_modifier

.github/workflows/run.yml

@@ -15,21 +15,21 @@ jobs:
 
       - name: Build Saxpy Workspace
         run: |
-          ./bin/benchpark setup saxpy/openmp x86_64 workspace/
+          ./bin/benchpark setup saxpy/openmp nosite-x86_64 workspace/
 
       - name: Setup Ramble & Spack
         run: |
           . workspace/setup.sh
 
           spack mirror add ci-buildcache oci://ghcr.io/llnl/benchpark-binary-cache
-          spack config add "packages:all:target:[x86_64_v3]"
+          spack config add "packages:all:target:[nosite-x86_64_v3]"
 
           env | grep SPACK >> "$GITHUB_ENV"
           env | grep RAMBLE >> "$GITHUB_ENV"
           echo "PATH=$PATH" >> "$GITHUB_ENV"
 
       - name: Setup Saxpy Workspace
-        working-directory: ./workspace/saxpy/openmp/x86_64/workspace/
+        working-directory: ./workspace/saxpy/openmp/nosite-x86_64/workspace/
         run: |
           ramble \
             --workspace-dir . \
@@ -39,7 +39,7 @@ jobs:
             workspace setup
 
       - name: Run Saxpy Experiments
-        working-directory: ./workspace/saxpy/openmp/x86_64/workspace/
+        working-directory: ./workspace/saxpy/openmp/nosite-x86_64/workspace/
         run: |
           ramble \
             -c variables:n_nodes:1 \
@@ -50,7 +50,7 @@ jobs:
             on
 
       - name: Analyze Saxpy Results
-        working-directory: ./workspace/saxpy/openmp/x86_64/workspace/
+        working-directory: ./workspace/saxpy/openmp/nosite-x86_64/workspace/
         run: |
           ramble \
             --workspace-dir . \
@@ -59,7 +59,7 @@ jobs:
             workspace analyze
 
       - name: Archive Experiment Workspace
-        working-directory: ./workspace/saxpy/openmp/x86_64/workspace/
+        working-directory: ./workspace/saxpy/openmp/nosite-x86_64/workspace/
         if: always()
         run: |
           ramble \
@@ -70,10 +70,10 @@ jobs:
 
       - name: Upload Workspace Archive as CI Artifact
         if: always()
-        uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595
+        uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6
         with:
           name: workspace-archive
-          path: './workspace/saxpy/openmp/x86_64/workspace/archive/**'
+          path: './workspace/saxpy/openmp/nosite-x86_64/workspace/archive/**'
 
       - name: Upload Binaries to CI Cache
         if: github.ref == 'refs/heads/develop'

configs/Penguin-icelake-OmniPath/system_definition.yaml → configs/LLNL-Magma-Penguin-icelake-OmniPath/system_definition.yaml

@@ -4,7 +4,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 system_definition:
-  name: Penguin-icelake-OmniPath
+  name: Magma
+  site: LLNL
+  system: Penguin-icelake-OmniPath
   integrator:
     vendor: PenguinComputing
     name: RelionCluster

configs/IBM-power9-V100-Infiniband/system_definition.yaml → configs/LLNL-Sierra-IBM-power9-V100-Infiniband/system_definition.yaml

@@ -4,7 +4,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 system_definition:
-  name: IBM-power9-V100-Infiniband
+  name: Sierra
+  site: LLNL
+  system: IBM-power9-V100-Infiniband
   integrator:
     vendor: IBM
     name: AC922

configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/system_definition.yaml

@@ -0,0 +1,34 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+system_definition:
+  name: Tioga
+  site: LLNL
+  system: HPECray-zen3-MI250X-Slingshot
+  integrator:
+    vendor: HPECray
+    name: EX235a
+  processor:
+    vendor: AMD
+    name: EPYC-Zen3
+    ISA: x86_64
+    uArch: zen3
+  accelerator:
+    vendor: AMD
+    name: MI250X
+    ISA: GCN
+    uArch: gfx90a
+  interconnect:
+    vendor: HPECray
+    name: Slingshot11
+  system-tested:
+    site: LLNL
+    name: tioga
+    installation-year: 2022
+    description: top500 <https://www.top500.org/system/180052>
+  top500-system-instances:
+    - Frontier (ORNL)
+    - Lumi     (CSC)
+    - Tioga    (LLNL)

configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/variables.yaml

@@ -0,0 +1,14 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  gtl_flag: ''  # to be overwritten by tests that need GTL
+  rocm_arch: 'gfx90a'
+  batch_time: '120m'
+  mpi_command: 'flux run -N {n_nodes} -n {n_ranks}'
+  batch_submit: 'flux batch {execute_experiment}'
+  batch_nodes: '# flux: -N {n_nodes}'
+  batch_ranks: '# flux: -n {n_ranks}'
+  batch_timeout: '# flux: -t {batch_time}'

configs/AWS_PCluster_Hpc7a-zen4-EFA/system_definition.yaml → configs/nosite-AWS_PCluster_Hpc7a-zen4-EFA/system_definition.yaml

@@ -5,6 +5,8 @@
 
 system_definition:
   name: AWS_PCluster_Hpc7a-zen4-EFA
+  site:
+  system: AWS_PCluster_Hpc7a-zen4-EFA
   integrator:
     vendor: AWS
     name: ParallelCluster3.7.2-Hpc7a

configs/nosite-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml

@@ -0,0 +1,35 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc:  /opt/cray/pe/cce/16.0.0/bin/craycc
+      cxx:  /opt/cray/pe/cce/16.0.0/bin/crayCC
+      f77:  /opt/cray/pe/cce/16.0.0/bin/crayftn
+      fc:  /opt/cray/pe/cce/16.0.0/bin/crayftn
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++14
+      fflags: -g -O2 -hnopattern
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: [/opt/cray/pe/gcc-libs/]
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /opt/cray/pe/gcc/12.2.0/bin/gcc
+      cxx: /opt/cray/pe/gcc/12.2.0/bin/g++
+      f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran
+      fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []

configs/nosite-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml

@@ -0,0 +1,17 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  all:
+    require: 'target=x86_64:'
+    variants: amdgpu_target=gfx90a
+  tar:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+
+# tar above is an example of how to (optionally) specify the external packages Spack should use (and not build)
+# you can use 'spack external find' to find the packages available on your system
+# (https://spack.readthedocs.io/en/v0.18.0/command_index.html#spack-external-find)

configs/nosite-HPECray-zen3-MI250X-Slingshot/spack.yaml

@@ -0,0 +1,25 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+spack:
+  packages:
+    default-compiler:
+      spack_spec: cce@16
+    default-mpi:
+      spack_spec: [email protected]%cce ~gtl
+    compiler-rocm:
+      spack_spec: cce@16
+    blas-rocm:
+      spack_spec: [email protected]
+    blas:
+      spack_spec: [email protected]
+    lapack:
+      spack_spec: cray-libsci@23
+    mpi-rocm-gtl:
+      spack_spec: [email protected]%cce +gtl
+    mpi-rocm-no-gtl:
+      spack_spec: [email protected]%cce ~gtl
+    mpi-gcc:
+      spack_spec: [email protected]%gcc ~gtl

configs/HPECray-zen3-MI250X-Slingshot/system_definition.yaml → configs/nosite-HPECray-zen3-MI250X-Slingshot/system_definition.yaml

@@ -5,6 +5,8 @@
 
 system_definition:
   name: HPECray-zen3-MI250X-Slingshot
+  site:
+  system: HPECray-zen3-MI250X-Slingshot
   integrator:
     vendor: HPECray
     name: EX235a

configs/nosite-HPECray-zen3-MI250X-Slingshot/variables.yaml

@@ -0,0 +1,14 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  gtl_flag: ''  # to be overwritten by tests that need GTL
+  rocm_arch: 'gfx90a'
+  batch_time: '02:00'
+  mpi_command: 'srun -N {n_nodes} -n {n_ranks}'
+  batch_submit: 'sbatch {execute_experiment}'
+  batch_nodes: '#SBATCH -N {n_nodes}'
+  batch_ranks: '#SBATCH -n {n_ranks}'
+  batch_timeout: '#SBATCH -t {batch_time}:00'

configs/x86_64/system_definition.yaml → configs/nosite-x86_64/system_definition.yaml

@@ -5,6 +5,8 @@
 
 system_definition:
   name: x86_64
+  site:
+  system: x86_64
   integrator:
     vendor:
     name:

docs/add-a-site-specific-system-config.rst

@@ -0,0 +1,14 @@
+.. Copyright 2023 Lawrence Livermore National Security, LLC and other
+   Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: Apache-2.0
+
+=============================
+Adding a Specific System Configuration
+=============================
+
+For a specific system, one can (optionally) add more information about the software installed on the system
+by adding Spack config files in ``benchpark/configs/$SITE/SYSTEMNAME-GENERICSYSTEM/auxiliary_software_files/``.
+
+- ``compilers.yaml`` defines the `compilers <https://spack.readthedocs.io/en/latest/getting_started.html#compiler-config>`_  installed on the system.
+- ``packages.yaml`` defines the pre-installed `packages <https://spack.readthedocs.io/en/latest/build_settings.html#package-settings-packages-yaml>`_   (e.g., system MPI) on the system.  One way to populate this list is to find available external packages: `spack external <https://spack.readthedocs.io/en/v0.21.0/command_index.html#spack-external>`_.

docs/add-a-system-config.rst

@@ -13,32 +13,51 @@ you can add a new directory with a name which identifies the system.
 
 The naming convention for the systems is as following::
 
-  [INTEGRATOR]-MICROARCHITECTURE[-GPU][-NETWORK]
+  SITE-[SYSTEMNAME-][INTEGRATOR]-MICROARCHITECTURE[-GPU][-NETWORK]
 
 where::
 
+  SITE = nosite | DATACENTERNAME
+
+  SYSTEMNAME = the name of the specific system
+
   INTEGRATOR = COMPANY[_PRODUCTNAME][...]
 
-Benchpark has definitions for the following systems:
+  MICROARCHITECTURE = CPU Microarchitecture
+
+  GPU = GPU Product Name
+
+  NETWORK = Network Product Name
+
+Benchpark has definitions for the following (nosite) systems:
+
+- nosite-AWS_PCluster_Hpc7a-zen4-EFA
 
-- AWS_PCluster_Hpc7a-zen4-EFA
+- nosite-HPECray-zen3-MI250X-Slingshot (same hardware as Frontier, Lumi, Tioga)
 
-- HPECray-zen3-MI250X-Slingshot	(Frontier, Lumi, Tioga)
+- nosite-x86_64 (x86 CPU only platform)
 
-- IBM-power9-V100-Infiniband	(Sierra)
 
-- Penguin-icelake-OmniPath
 
-- x86_64                        (generic x86 CPU only platform)
+Benchpark has definitions for the following site-specific systems:
 
-The following files are required for each system ``benchpark/configs/${SYSTEM}``:
+- LLNL-Magma-Penguin-icelake-OmniPath
+
+- LLNL-Sierra-IBM-power9-V100-Infiniband (Sierra, Lassen)
+
+- LLNL-Tioga-HPECray-zen3-MI250X-Slingshot
+
+
+The following files are required for each nosite system ``benchpark/configs/${SYSTEM}``:
 
 1. ``system_definition.yaml`` describes the system hardware, including the integrator (and the name of the product node or cluster type), the processor, (optionally) the accelerator, and the network; the information included here is what you will typically see recorded about the system on Top500.org.  We intend to make the system definitions in Benchpark searchable, and will add a schema to enforce consistency; until then, please copy the file and fill out all of the fields without changing the keys.  Also listed is the specific system the config was developed and tested on, as well as the known systems with the same hardware so that the users of those systems can find this system specification.
 
 .. code-block:: yaml
 
   system_definition:
-    name: HPECray-zen3-MI250X-Slingshot
+    name: HPECray-zen3-MI250X-Slingshot # or site-specific name, e.g., Frontier at ORNL
+    site:
+    system: HPECray-zen3-MI250X-Slingshot
     integrator:
       vendor: HPECray
       name: EX235a
@@ -91,8 +110,5 @@ spack section in the `Ramble configuration file
       batch_ranks: ''
       batch_timeout: ''
 
-4. Optionally, one can add more information about the software installed on the system
-by adding Spack config files in ``benchpark/configs/${SYSTEM}/auxiliary_software_files/``.
-
-- ``compilers.yaml`` defines the `compilers <https://spack.readthedocs.io/en/latest/getting_started.html#compiler-config>`_  installed on the system.
-- ``packages.yaml`` defines the pre-installed `packages <https://spack.readthedocs.io/en/latest/build_settings.html#package-settings-packages-yaml>`_   (e.g., system MPI) on the system.  One way to populate this list is to find available external packages: `spack external <https://spack.readthedocs.io/en/v0.21.0/command_index.html#spack-external>`_. 
+If defining a specific system, one can be more specific with available software versions
+and packages, as demonstrated in :doc:`add-a-specific-system-config.rst`.

docs/index.rst

@@ -32,6 +32,7 @@
    add-a-benchmark
    add-an-experiment
    add-a-system-config
+   add-a-site-specific-system-config
 
 .. toctree::
    :maxdepth: 1