New system: CTS (ruby and magma) (#349)

Also fix bug in System base class for systems that don't have GPUs

lib/benchpark/system.py

@@ -150,6 +150,7 @@ def variables_yaml(self):
                 raise ValueError(f"Missing required info: {attr}")
 
         optionals = list()
+        optionals_as_cfg = ""
         for opt in ["sys_gpus_per_node", "sys_mem_per_node", "queue"]:
             if getattr(self, opt, None):
                 optionals.append(f"{opt}: {getattr(self, opt)}")

var/sys_repo/systems/cts/compilers/gcc/00-gcc-12-compilers.yaml

@@ -0,0 +1,19 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/gcc/gcc-12.1.1/bin/gcc
+      cxx: /usr/tce/packages/gcc/gcc-12.1.1/bin/g++
+      f77: /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran
+      fc: /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []

var/sys_repo/systems/cts/compilers/intel/00-intel-2021-6-0-compilers.yaml

@@ -0,0 +1,19 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/icc
+      cxx: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/icpc
+      f77: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/ifort
+      fc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/ifort
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []

var/sys_repo/systems/cts/externals/base/00-packages.yaml

@@ -0,0 +1,50 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  papi:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/papi/papi-6.0.0.1
+  tar:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  cmake:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cmake/cmake-3.26.3
+  gmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  blas:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp
+  lapack:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp
+  python:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/python/python-3.9.12/
+  hwloc:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  fftw:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/fftw/fftw-3.3.10

var/sys_repo/systems/cts/externals/mpi/00-gcc-packages.yaml

@@ -0,0 +1,13 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  mpi:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1
+      extra_attributes:
+        ldflags: "-L/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1/lib -lmpi"

var/sys_repo/systems/cts/externals/mpi/01-intel-packages.yaml

@@ -0,0 +1,13 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  mpi:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0
+      extra_attributes:
+        ldflags: "-L/usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/lib -lmpi"

var/sys_repo/systems/cts/system.py

@@ -0,0 +1,104 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pathlib
+
+from benchpark.directives import variant
+from benchpark.system import System
+
+id_to_resources = {
+    "ruby": {
+        "sys_cores_per_node": 56,
+    },
+    "magma": {
+        "sys_cores_per_node": 96,
+    },
+}
+
+
+class Cts(System):
+
+    variant(
+        "cluster",
+        default="ruby",
+        values=("ruby", "magma"),
+        description="Which cluster to run on",
+    )
+
+    variant(
+        "compiler",
+        default="gcc",
+        values=("gcc", "intel"),
+        description="Which compiler to use",
+    )
+
+    def initialize(self):
+        super().initialize()
+
+        self.scheduler = "slurm"
+        attrs = id_to_resources.get(self.spec.variants["cluster"][0])
+        for k, v in attrs.items():
+            setattr(self, k, v)
+
+    def generate_description(self, output_dir):
+        super().generate_description(output_dir)
+
+        sw_description = pathlib.Path(output_dir) / "software.yaml"
+
+        with open(sw_description, "w") as f:
+            f.write(self.sw_description())
+
+    def external_pkg_configs(self):
+        externals = Cts.resource_location / "externals"
+
+        compiler = self.spec.variants["compiler"][0]
+
+        selections = [externals / "base" / "00-packages.yaml"]
+
+        if compiler == "gcc":
+            selections.append(externals / "mpi" / "00-gcc-packages.yaml")
+        elif compiler == "intel":
+            selections.append(externals / "mpi" / "01-intel-packages.yaml")
+
+        return selections
+
+    def compiler_configs(self):
+        compilers = Cts.resource_location / "compilers"
+
+        compiler = self.spec.variants["compiler"][0]
+
+        selections = []
+        if compiler == "gcc":
+            selections.append(compilers / "gcc" / "00-gcc-12-compilers.yaml")
+        elif compiler == "intel":
+            selections.append(compilers / "intel" / "00-intel-2021-6-0-compilers.yaml")
+
+        return selections
+
+    def sw_description(self):
+        """This is somewhat vestigial, and maybe deleted later. The experiments
+        will fail if these variables are not defined though, so for now
+        they are still generated (but with more-generic values).
+        """
+        return """\
+software:
+  packages:
+    default-compiler:
+      pkg_spec: gcc
+    default-mpi:
+      pkg_spec: mvapich2
+    compiler-gcc:
+      pkg_spec: gcc
+    compiler-intel:
+      pkg_spec: intel
+    blas:
+      pkg_spec: intel-oneapi-mkl
+    lapack:
+      pkg_spec: intel-oneapi-mkl
+    mpi-gcc:
+      pkg_spec: mvapich2
+    mpi-intel:
+      pkg_spec: mvapich2
+"""