Feature caliper base modifier (#123)

* Caliper modifier

* Caliper modifier

* Merge with develop

* Removing postprocessing

* modifier changes

* Update benchpark

* Support for topdown and cuda modes in caliper

* Update license in modifier.py

* Moving modifiers to their own repository

* Fix modifier path and filename

* Fix caliper spack specs and remove elfutils

* Implementation of caliper top-down modifier

* Implementation of caliper cuda modifier

* Remove caliper-specific references from ramble.yaml

* Remove caliper-specific references from amg2023 cuda ramble.yaml

* Fix adiak calls

* Make separate modifier config file for each caliper modifier

* caliper for x86

* Fix benchpark script

* caliper cuda modifier for amg2023, saxpy

* Use caliper base modifier

* Build base mode without papi

* Remove external papi package

* fixing lint style

* Remove topdown and cuda modifiers

* Add modifier option to benchpark setup script, define caliper config once

* Remove cuda caliper modifier from amg2023 and saxpy

* Add base caliper modifier for amg2023(cuda,rocm) and saxpy(cuda,rocm)

* lint

* lint

* Replace caliper.yaml with modifier.yaml

* Add empty default modifier

* Remove star import

* relax flake8 for modifier functionality

ignores F403 and F405 in modifier.py

'from module import *' used; unable to detect undefined names (F403)
Name may be undefined, or defined from star imports: module (F405)

* Remove zero-modifier

* Add FOM in AMG2023

---------

Co-authored-by: pearce8 <[email protected]>
Co-authored-by: Riyaz Haque <[email protected]>
Co-authored-by: Stephanie Brink <[email protected]>

.flake8

@@ -8,6 +8,10 @@
 max-line-length = 88
 select = C,E,F,W,B,B950
 ignore = E501,W503,E203
+
+per-file-ignores =
+  modifiers/*/modifier.py:F403,F405
+
 builtins = IPython
 exclude =
   .eggs,

bin/benchpark

@@ -162,6 +162,12 @@ def benchpark_setup(subparsers, actions_dict):
         type=str,
         help="Where to install packages and store results for the experiments. Benchpark expects to manage this directory, and it should be empty/nonexistent the first time you run benchpark setup experiments.",
     )
+    create_parser.add_argument(
+        "--modifier",
+        type=str,
+        default="none",
+        help="The modifier to apply to the experiment (default none)",
+    )
 
     actions_dict["setup"] = benchpark_setup_handler
 
@@ -212,6 +218,7 @@ def benchpark_setup_handler(args):
     benchmark = args.benchmark
     system = args.system
     experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
+    modifier = args.modifier
     source_dir = source_location()
     debug_print(f"source_dir = {source_dir}")
     debug_print(f"specified benchmark/ProgrammingModel = {benchmark}")
@@ -246,10 +253,12 @@ def benchpark_setup_handler(args):
 
     configs_src_dir = source_dir / "configs" / str(system)
     experiment_src_dir = source_dir / "experiments" / benchmark
+    modifier_config_dir = source_dir / "modifiers" / modifier / "configs"
 
     ramble_configs_dir.mkdir(parents=True)
     symlink_tree(configs_src_dir, ramble_configs_dir)
     symlink_tree(experiment_src_dir, ramble_configs_dir)
+    symlink_tree(modifier_config_dir, ramble_configs_dir)
 
     spack_location = experiments_root / "spack"
     ramble_location = experiments_root / "ramble"
@@ -287,6 +296,9 @@ def benchpark_setup_handler(args):
         run_command(
             f'{ramble_exe} config --scope=site add "config:disable_progress_bar:true"'
         )
+        run_command(
+            f"{ramble_exe} repo add -t modifiers --scope=site {source_dir}/modifiers"
+        )
         run_command(
             f"{ramble_exe} config --scope=site add \"config:spack:global:args:'-d'\""
         )

configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml

@@ -28,6 +28,11 @@ packages:
     - spec: [email protected]
       prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp
       buildable: false
+  python:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/python/python-3.9.12/
+      buildable: false
   hwloc:
     externals:
     - spec: [email protected]

configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml

@@ -65,6 +65,11 @@ packages:
     - spec: [email protected]
       prefix: /usr/tcetmp/packages/lapack/lapack-3.9.0-xl-2020.03.18
     buildable: false
+  python:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/python/python-3.8.2
+      buildable: false
   mpi:
     externals:
     - spec: [email protected]

experiments/amg2023/cuda/ramble.yaml

@@ -4,11 +4,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
   config:
     deprecated: true
     spack_flags:
       install: '--add --keep-stage'
       concretize: '-U -f'
+
   applications:
     amg2023:
       workloads:
@@ -43,10 +49,10 @@ ramble:
     concretized: true
     packages:
       hypre:
-        spack_spec: [email protected] +mpi+cuda+mixedint cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: [email protected] +mpi+cuda+mixedint{modifier_spack_variant} cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
       amg2023:
-        spack_spec: amg2023@develop +mpi+cuda cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: amg2023@develop +mpi+cuda{modifier_spack_variant} cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
       amg2023:
@@ -56,3 +62,4 @@ ramble:
         - default-mpi
         - hypre
         - amg2023
+        - '{modifier_package_name}'

experiments/amg2023/openmp/ramble.yaml

@@ -7,6 +7,7 @@ ramble:
   include:
   - ./configs/spack.yaml
   - ./configs/variables.yaml
+  - ./configs/modifier.yaml
 
   config:
     deprecated: true
@@ -33,24 +34,24 @@ ramble:
             nz: '{n}'
             processes_per_node: ['8', '4']
             n_nodes: ['1', '2']
-            threads_per_node_core: ['4', '6', '12'] #TODO: Specify n_threads according to available n_nodes and n_ranks
+            threads_per_node_core: ['4', '6', '12']
             omp_num_threads: '{threads_per_node_core} * {n_nodes}'
           experiments:
             amg2023_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}:
               variables:
                 env_name: amg2023-omp
               matrices:
                 - size_threads:
-                  - n # TODO: Filter matrix
-                  - threads_per_node_core # TODO: Filter matrix
+                  - n
+                  - threads_per_node_core
   spack:
     concretized: true
     packages:
       hypre-omp:
-        spack_spec: [email protected] +mpi+openmp+mixedint
+        spack_spec: [email protected] +mpi+openmp+mixedint{modifier_spack_variant}
         compiler: default-compiler
       amg2023-omp:
-        spack_spec: amg2023@develop +mpi+openmp
+        spack_spec: amg2023@develop +mpi+openmp{modifier_spack_variant}
         compiler: default-compiler
     environments:
       amg2023-omp:
@@ -59,3 +60,4 @@ ramble:
         - default-mpi
         - hypre-omp
         - amg2023-omp
+        - '{modifier_package_name}'

experiments/amg2023/rocm/ramble.yaml

@@ -4,11 +4,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
   config:
     deprecated: true
     spack_flags:
       install: '--add --keep-stage'
       concretize: '-U -f'
+
   applications:
     amg2023:
       workloads:
@@ -40,10 +46,10 @@ ramble:
       gtl: ["gtl", "no-gtl"]
     packages:
       hypre-{gtl}:
-        spack_spec: [email protected] +mpi+rocm+mixedint amdgpu_target={rocm_arch}
+        spack_spec: [email protected] +mpi+rocm+mixedint{modifier_spack_variant} amdgpu_target={rocm_arch}
         compiler: compiler-rocm
       amg2023-gpu-{gtl}:
-        spack_spec: amg2023@develop +mpi+rocm amdgpu_target={rocm_arch}
+        spack_spec: amg2023@develop +mpi+rocm{modifier_spack_variant} amdgpu_target={rocm_arch}
         compiler: compiler-rocm
     environments:
       amg2023-gpu-{gtl}:
@@ -53,3 +59,4 @@ ramble:
         - mpi-rocm-{gtl}
         - hypre-{gtl}
         - amg2023-gpu-{gtl}
+        - '{modifier_package_name}'

experiments/saxpy/cuda/ramble.yaml

@@ -4,11 +4,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
   config:
     deprecated: true
     spack_flags:
       install: '--add --keep-stage'
       concretize: '-U -f'
+
   applications:
     saxpy:
       workloads:
@@ -27,10 +33,11 @@ ramble:
     concretized: true
     packages:
       saxpy:
-        spack_spec: [email protected] +cuda cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: [email protected] +cuda{modifier_spack_variant} cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
       saxpy:
         packages:
         - default-mpi
         - saxpy
+        - '{modifier_package_name}'

experiments/saxpy/openmp/ramble.yaml

@@ -7,6 +7,7 @@ ramble:
   include:
   - ./configs/spack.yaml
   - ./configs/variables.yaml
+  - ./configs/modifier.yaml
 
   config:
     deprecated: true
@@ -39,10 +40,11 @@ ramble:
     concretized: true
     packages:
       saxpy:
-        spack_spec: [email protected] +openmp ^[email protected]
+        spack_spec: [email protected] +openmp{modifier_spack_variant} ^[email protected]
         compiler: default-compiler
     environments:
       saxpy:
         packages:
         - default-mpi
         - saxpy
+        - '{modifier_package_name}'

experiments/saxpy/rocm/ramble.yaml

@@ -7,12 +7,14 @@ ramble:
   include:
   - ./configs/spack.yaml
   - ./configs/variables.yaml
+  - ./configs/modifier.yaml
 
   config:
     deprecated: true
     spack_flags:
       install: '--add --keep-stage'
       concretize: '-U -f'
+
   applications:
     saxpy:
       workloads:
@@ -31,10 +33,11 @@ ramble:
     concretized: true
     packages:
       saxpy:
-        spack_spec: [email protected] +rocm amdgpu_target={rocm_arch}
+        spack_spec: [email protected] +rocm{modifier_spack_variant} amdgpu_target={rocm_arch}
         compiler: default-compiler
     environments:
       saxpy:
         packages:
         - default-mpi
         - saxpy
+        - '{modifier_package_name}'

modifiers/caliper/configs/modifier.yaml

@@ -0,0 +1,17 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  modifier_package_name: 'caliper'
+  modifier_spack_variant: '+caliper'
+
+modifiers:
+  - name: caliper
+    mode: time
+
+spack:
+  packages:
+    caliper:
+      spack_spec: caliper+adiak+mpi~libunwind~libdw~papi

modifiers/caliper/modifier.py

@@ -0,0 +1,33 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from ramble.modkit import *
+
+
+class Caliper(SpackModifier):
+    """Define a modifier for Caliper"""
+
+    name = "caliper"
+
+    tags("profiler", "performance-analysis")
+
+    maintainers("pearce8")
+
+    mode("time", description="Platform-independent collection of time")
+
+    _cali_datafile = "{experiment_run_dir}/{experiment_name}.cali"
+
+    env_var_modification(
+        "CALI_CONFIG",
+        "spot(output={})".format(_cali_datafile),
+        method="set",
+        modes=["time"],
+    )
+
+    archive_pattern(_cali_datafile)
+
+    software_spec("caliper", spack_spec="caliper")
+
+    required_package("caliper")

modifiers/modifier_repo.yaml

@@ -0,0 +1,3 @@
+repo:
+  namespace: benchpark
+  subdirectory: ''

modifiers/none/configs/modifier.yaml

@@ -0,0 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  modifier_package_name: ''
+  modifier_spack_variant: ''

repo/amg2023/application.py

@@ -53,5 +53,3 @@ class Amg2023(SpackApplication):
     #TODO: Fix the FOM success_criteria(...)
     success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out')
 
-    def evaluate_success(self):
-      return True