Add test suite (#384)

* Includes new command `benchpark unit-test`
  This command makes use of pytest, which can accept more arguments
  that are specified as command-line option: add a "pass-through"
  mechanism for arguments that are unknown to the command (but
  e.g. in this case are known to pytest)
* Add a `spack.yaml` environment file which makes it easier to
  install the resources needed to run unit tests (e.g. pytest);
  these are not included in Benchpark's `requirements.txt`
* Tests are added for Spec syntax and for the Experiment class.
  Coverage is currently low but that is a matter for future PRs.
* This includes a minor refactor of the logic used in the library
  code to locate Benchpark's root directory (previously
  `source_location`, now accessable as `paths.benchpark_root`)

---------

Co-authored-by: Alec Scott <[email protected]>
Co-authored-by: pearce8 <[email protected]>
Co-authored-by: Peter Josef Scheibel <[email protected]>

.envrc

@@ -0,0 +1,10 @@
+#------------------------------------------------------------------------
+# Load Development Spack Environment (If Spack is installed.)
+#
+# Run 'direnv allow' from within the cloned repository to automatically
+# load the spack environment when you enter the directory.
+#------------------------------------------------------------------------
+if type spack &>/dev/null; then
+    . $SPACK_ROOT/share/spack/setup-env.sh
+    spack env activate -d .
+fi

.github/workflows/run.yml

@@ -179,7 +179,7 @@ jobs:
           ./bin/benchpark setup kripke/rocm ./tioga-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/kripke/rocm/Tioga-d34a754/workspace \
+            --workspace-dir workspace/kripke/rocm/Tioga-975af3c/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
@@ -202,7 +202,7 @@ jobs:
           ./bin/benchpark setup ./saxpy-rocm2 ./tioga-system2 workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/saxpy-rocm2/Tioga-d34a754/workspace \
+            --workspace-dir workspace/saxpy-rocm2/Tioga-975af3c/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run

bin/benchpark

@@ -12,9 +12,9 @@ import sys
 
 
 def main():
-    basedir = pathlib.Path(os.path.abspath(__file__)).parent.parent
-    main = basedir / "lib" / "main.py"
-    subprocess.run([sys.executable, main] + sys.argv[1:], check=True)
+    basedir = pathlib.Path(__file__).resolve().parents[1]
+    main_py = basedir / "lib" / "main.py"
+    subprocess.run([sys.executable, main_py] + sys.argv[1:], check=True)
 
 
 if __name__ == "__main__":

lib/benchpark/accounting.py

@@ -5,11 +5,11 @@
 
 import os
 
-from benchpark.paths import source_location
+import benchpark.paths
 
 
 def benchpark_experiments():
-    source_dir = source_location()
+    source_dir = benchpark.paths.benchpark_root
     experiments = []
     experiments_dir = source_dir / "experiments"
     for x in os.listdir(experiments_dir):
@@ -19,15 +19,15 @@ def benchpark_experiments():
 
 
 def benchpark_modifiers():
-    source_dir = source_location()
+    source_dir = benchpark.paths.benchpark_root
     modifiers = []
     for x in os.listdir(source_dir / "modifiers"):
         modifiers.append(x)
     return modifiers
 
 
 def benchpark_systems():
-    source_dir = source_location()
+    source_dir = benchpark.paths.benchpark_root
     systems = []
     for x in os.listdir(source_dir / "configs"):
         if not (

lib/benchpark/cmd/setup.py

@@ -9,15 +9,16 @@
 import pathlib
 import shutil
 import sys
+
 import yaml
 
+import benchpark.paths
 from benchpark.accounting import (
     benchpark_experiments,
     benchpark_modifiers,
     benchpark_systems,
 )
 from benchpark.debug import debug_print
-from benchpark.paths import source_location
 from benchpark.runtime import RuntimeResources
 
 
@@ -87,7 +88,7 @@ def benchpark_check_experiment(arg_str):
         )
         raise ValueError(out_str)
 
-    experiment_src_dir = source_location() / "experiments" / str(arg_str)
+    experiment_src_dir = benchpark.paths.benchpark_root / "experiments" / str(arg_str)
     return arg_str, experiment_src_dir
 
 
@@ -115,7 +116,7 @@ def benchpark_check_system(arg_str):
         )
         raise ValueError(out_str)
 
-    configs_src_dir = source_location() / "configs" / str(arg_str)
+    configs_src_dir = benchpark.paths.benchpark_root / "configs" / str(arg_str)
     return arg_str, configs_src_dir
 
 
@@ -145,7 +146,7 @@ def command(args):
 
     experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
     modifier = args.modifier
-    source_dir = source_location()
+    source_dir = benchpark.paths.benchpark_root
     debug_print(f"source_dir = {source_dir}")
     experiment_id, experiment_src_dir = benchpark_check_experiment(args.experiment)
     debug_print(f"specified experiment (benchmark/ProgrammingModel) = {experiment_id}")

lib/benchpark/cmd/unit_test.py

@@ -0,0 +1,232 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+import argparse
+import collections
+import io
+import pytest
+import os
+import re
+import sys
+
+import llnl.util.filesystem
+import llnl.util.tty.color as color
+import llnl.util.tty.colify as colify
+
+import benchpark.paths
+
+
+def setup_parser(subparser):
+    subparser.add_argument(
+        "-H",
+        "--pytest-help",
+        action="store_true",
+        default=False,
+        help="show full pytest help, with advanced options",
+    )
+
+    subparser.add_argument(
+        "-n",
+        "--numprocesses",
+        type=int,
+        default=1,
+        help="run tests in parallel up to this wide, default 1 for sequential",
+    )
+
+    # extra arguments to list tests
+    list_group = subparser.add_argument_group("listing tests")
+    list_mutex = list_group.add_mutually_exclusive_group()
+    list_mutex.add_argument(
+        "-l",
+        "--list",
+        action="store_const",
+        default=None,
+        dest="list",
+        const="list",
+        help="list test filenames",
+    )
+    list_mutex.add_argument(
+        "-L",
+        "--list-long",
+        action="store_const",
+        default=None,
+        dest="list",
+        const="long",
+        help="list all test functions",
+    )
+    list_mutex.add_argument(
+        "-N",
+        "--list-names",
+        action="store_const",
+        default=None,
+        dest="list",
+        const="names",
+        help="list full names of all tests",
+    )
+
+    # spell out some common pytest arguments, so they'll show up in help
+    pytest_group = subparser.add_argument_group(
+        "common pytest arguments (benchpark unit-test --pytest-help for more)"
+    )
+    pytest_group.add_argument(
+        "-s",
+        action="append_const",
+        dest="parsed_args",
+        const="-s",
+        help="print output while tests run (disable capture)",
+    )
+    pytest_group.add_argument(
+        "-k",
+        action="store",
+        metavar="EXPRESSION",
+        dest="expression",
+        help="filter tests by keyword (can also use w/list options)",
+    )
+    pytest_group.add_argument(
+        "--showlocals",
+        action="append_const",
+        dest="parsed_args",
+        const="--showlocals",
+        help="show local variable values in tracebacks",
+    )
+
+    # remainder is just passed to pytest
+    subparser.add_argument(
+        "pytest_args", nargs=argparse.REMAINDER, help="arguments for pytest"
+    )
+
+
+def do_list(args, extra_args):
+    """Print a lists of tests than what pytest offers."""
+
+    def colorize(c, prefix):
+        if isinstance(prefix, tuple):
+            return "::".join(
+                color.colorize("@%s{%s}" % (c, p)) for p in prefix if p != "()"
+            )
+        return color.colorize("@%s{%s}" % (c, prefix))
+
+    # To list the files we just need to inspect the filesystem,
+    # which doesn't need to wait for pytest collection and doesn't
+    # require parsing pytest output
+    files = llnl.util.filesystem.find(
+        root=benchpark.paths.test_path, files="*.py", recursive=True
+    )
+    files = [
+        os.path.relpath(f, start=benchpark.paths.benchpark_root)
+        for f in files
+        if not f.endswith(("conftest.py", "__init__.py"))
+    ]
+
+    old_output = sys.stdout
+    try:
+        sys.stdout = output = io.StringIO()
+        pytest.main(["--collect-only"] + extra_args)
+    finally:
+        sys.stdout = old_output
+
+    lines = output.getvalue().split("\n")
+    tests = collections.defaultdict(set)
+
+    # collect tests into sections
+    node_regexp = re.compile(r"(\s*)<([^ ]*) ['\"]?([^']*)['\"]?>")
+    key_parts, name_parts = [], []
+    for line in lines:
+        match = node_regexp.match(line)
+        if not match:
+            continue
+        indent, nodetype, name = match.groups()
+
+        # strip parametrized tests
+        if "[" in name:
+            name = name[: name.index("[")]
+
+        len_indent = len(indent)
+        if os.path.isabs(name):
+            name = os.path.relpath(name, start=benchpark.paths.benchpark_root)
+
+        item = (len_indent, name, nodetype)
+
+        # Reduce the parts to the scopes that are of interest
+        name_parts = [x for x in name_parts if x[0] < len_indent]
+        key_parts = [x for x in key_parts if x[0] < len_indent]
+
+        # From version 3.X to version 6.X the output format
+        # changed a lot in pytest, and probably will change
+        # in the future - so this manipulation might be fragile
+        if nodetype.lower() == "function":
+            name_parts.append(item)
+            key_end = os.path.join(*key_parts[-1][1].split("/"))
+            key = next(f for f in files if f.endswith(key_end))
+            tests[key].add(tuple(x[1] for x in name_parts))
+        elif nodetype.lower() == "class":
+            name_parts.append(item)
+        elif nodetype.lower() in ("package", "module"):
+            key_parts.append(item)
+
+    if args.list == "list":
+        files = set(tests.keys())
+        color_files = [colorize("B", file) for file in sorted(files)]
+        colify.colify(color_files)
+
+    elif args.list == "long":
+        for prefix, functions in sorted(tests.items()):
+            path = colorize("*B", prefix) + "::"
+            functions = [colorize("c", f) for f in sorted(functions)]
+            color.cprint(path)
+            colify.colify(functions, indent=4)
+            print()
+
+    else:  # args.list == "names"
+        all_functions = [
+            colorize("*B", prefix) + "::" + colorize("c", f)
+            for prefix, functions in sorted(tests.items())
+            for f in sorted(functions)
+        ]
+        colify.colify(all_functions)
+
+
+def add_back_pytest_args(args, unknown_args):
+    """Add parsed pytest args, unknown args, and remainder together.
+
+    We add some basic pytest arguments to the Spack parser to ensure that
+    they show up in the short help, so we have to reassemble things here.
+    """
+    result = args.parsed_args or []
+    result += unknown_args or []
+    result += args.pytest_args or []
+    if args.expression:
+        result += ["-k", args.expression]
+    return result
+
+
+def command(args, unknown_args):
+    global pytest
+
+    if args.pytest_help:
+        # make the pytest.main help output more accurate
+        sys.argv[0] = "spack unit-test"
+        return pytest.main(["-h"])
+
+    # add back any parsed pytest args we need to pass to pytest
+    pytest_args = add_back_pytest_args(args, unknown_args)
+    pytest_root = benchpark.paths.benchpark_root
+
+    if args.numprocesses is not None and args.numprocesses > 1:
+        pytest_args.extend(
+            [
+                "--dist",
+                "loadfile",
+                "--tx",
+                f"{args.numprocesses}*popen//python=benchpark-tmpconfig benchpark-python",
+            ]
+        )
+
+    # pytest.ini lives in the root of the spack repository.
+    with llnl.util.filesystem.working_dir(pytest_root):
+        if args.list:
+            do_list(args, pytest_args)
+            return
+
+        return pytest.main(pytest_args)

lib/benchpark/paths.py

@@ -6,11 +6,16 @@
 import os
 import pathlib
 
+
+def _source_location() -> pathlib.Path:
+    """Return the location of the project source files directory."""
+    path_to_this_file = __file__
+    return pathlib.Path(path_to_this_file).resolve().parents[2]
+
+
+benchpark_root = _source_location()
+lib_path = benchpark_root / "lib" / "benchpark"
+test_path = lib_path / "test"
 benchpark_home = pathlib.Path(os.path.expanduser("~/.benchpark"))
 global_ramble_path = benchpark_home / "ramble"
 global_spack_path = benchpark_home / "spack"
-
-
-def source_location():
-    the_directory_with_this_file = os.path.dirname(os.path.abspath(__file__))
-    return pathlib.Path(the_directory_with_this_file).parent.parent