Starting tagging functionality (#65)

* Starting tagging functionality * Update benchpark_tags.yaml * Update benchpark_tags.yaml * copyright * trying to fix lint * Adding tags * Adding tags * Adding tags * adding tags * adding tags * adding tags * Expanding benchmark tags options * Update tags * Adding functionality to see all tags, and all experiments with a specific tag * adding more options to tag queries, and adding to the docs * fix typo CI caught * formatting * formatting * Starting table descriptions of benchmarks and tags for the docs * lint * add help menu to docs * add new page for current benchpark help menu * starting benchpark list into table - parsing application.py for tags list, split across multiple lines * starting tags list into table - parsing application.py for tags list, split across multiple lines * notes * working benchmark list table * fix page reference * use benchmark tags command * use benchpark tags command * remove commented out code * remove unused parsing script for tags * black * remove unused import * Add CI setup to install ramble for use in docs generation * add tag table generator script to makefile * run all * Add yaml dependency to docs CI * fixes * sphinxcontrib-programoutput req * Fix codspell looking in the rendered docs * remove tables/*.csv * run benchmark tags script --------- Co-authored-by: Stephanie Brink <[email protected]> Co-authored-by: Alec Scott <[email protected]>
LLNL · Mar 13, 2024 · f3cfc47 · f3cfc47
1 parent b3e09e6
commit f3cfc47
Show file tree

Hide file tree

Showing 24 changed files with 432 additions and 176 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -23,10 +23,15 @@ jobs:
     - name: Install Sphinx and Theme via Pip
       run: |
         pip install -r .github/workflows/requirements/docs.txt
+        # create a dummy workspace to get a working ramble
+        bin/benchpark setup saxpy/openmp nosite-x86_64 /tmp/workspace
+        # this is gross and we should better setup ramble for people or make it a Spack package
+        pip install -r /tmp/workspace/ramble/requirements.txt
 
     - name: Build with sphinx
       run: |
-        sphinx-build docs/ _build
+        cd docs
+        make html WORKSPACE_PATH=/tmp/workspace
 
     - name: Check for Typos using Codespell
       run: |

diff --git a/.github/workflows/requirements/docs.txt b/.github/workflows/requirements/docs.txt
@@ -2,3 +2,6 @@
 sphinx==7.2.6
 sphinx-rtd-theme==2.0.0
 codespell==2.2.6
+pandas==2.2.0
+pyyaml==6.0.1
+sphinxcontrib-programoutput==0.17
diff --git a/bin/benchpark b/bin/benchpark
@@ -39,6 +39,7 @@ def main():
     actions = {}
     benchpark_list(subparsers, actions)
     benchpark_setup(subparsers, actions)
+    benchpark_tags(subparsers, actions)
 
     args = parser.parse_args()
     no_args = True if len(sys.argv) == 1 else False
@@ -62,7 +63,6 @@ def main():
 
 def get_version():
     benchpark_version = __version__
-
     return benchpark_version
 
 
@@ -89,6 +89,15 @@ def benchpark_benchmarks():
     return benchmarks
 
 
+def benchpark_experiments():
+    source_dir = source_location()
+    experiments = []
+    experiments_dir = source_dir / "experiments"
+    for x in os.listdir(experiments_dir):
+        experiments.append(f"{x}")
+    return experiments
+
+
 def benchpark_systems():
     source_dir = source_location()
     systems = []
@@ -97,6 +106,28 @@ def benchpark_systems():
     return systems
 
 
+def benchpark_get_tags():
+    f = source_location() / "tags.yaml"
+    tags = []
+
+    with open(f, "r") as stream:
+        try:
+            data = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc)
+
+    for k0, v0 in data.items():
+        if k0 == "benchpark-tags":
+            for k, v in v0.items():
+                if isinstance(v, list):
+                    for i in v:
+                        tags.append(i)
+        else:
+            print("ERROR file does not contain benchpark-tags")
+
+    return tags
+
+
 def benchpark_list_handler(args):
     source_dir = source_location()
     sublist = args.sublist
@@ -137,6 +168,17 @@ def benchpark_check_benchmark(arg_str):
     return found
 
 
+def benchpark_check_experiment(arg_str):
+    experiments = benchpark_experiments()
+    found = arg_str in experiments
+    if not found:
+        out_str = f'Invalid benchmark/experiment "{arg_str}" - must choose one of: '
+        for experiment in experiments:
+            out_str += f"\n\t{experiment}"
+        raise ValueError(out_str)
+    return found
+
+
 def benchpark_check_system(arg_str):
     systems = benchpark_systems()
     found = arg_str in systems
@@ -148,6 +190,17 @@ def benchpark_check_system(arg_str):
     return found
 
 
+def benchpark_check_tag(arg_str):
+    tags = benchpark_get_tags()
+    found = arg_str in tags
+    if not found:
+        out_str = f'Invalid tag "{arg_str}" - must choose one of: '
+        for tag in tags:
+            out_str += f"\n\t{tag}"
+        raise ValueError(out_str)
+    return found
+
+
 def benchpark_setup(subparsers, actions_dict):
     create_parser = subparsers.add_parser(
         "setup", help="Set up an experiment and prepare it to build/run"
@@ -175,7 +228,7 @@ def benchpark_setup(subparsers, actions_dict):
 
 
 def run_command(command_str, env=None):
-    subprocess.run(
+    result = subprocess.run(
         shlex.split(command_str),
         env=env,
         check=True,
@@ -184,6 +237,30 @@ def run_command(command_str, env=None):
         text=True,
     )
 
+    return (result.stdout, result.stderr)
+
+
+def benchpark_tags(subparsers, actions_dict):
+    create_parser = subparsers.add_parser("tags", help="Tags in Benchpark experiments")
+    create_parser.add_argument(
+        "experiments_root",
+        type=str,
+        help="The experiments_root you specified during Benchpark setup.",
+    )
+    create_parser.add_argument(
+        "-a",
+        "--application",
+        action="store",
+        help="The application for which to find Benchpark tags",
+    )
+    create_parser.add_argument(
+        "-t",
+        "--tag",
+        action="store",
+        help="The tag for which to search in Benchpark experiments",
+    )
+    actions_dict["tags"] = benchpark_tags_handler
+
 
 # Note: it would be nice to vendor spack.llnl.util.link_tree, but that
 # involves pulling in most of llnl/util/ and spack/util/
@@ -375,5 +452,54 @@ Further steps are needed to build the experiments (ramble -P -D {ramble_workspac
     print(instructions)
 
 
+def helper_experiments_tags(ramble_exe, experiments):
+    # find all tags in Ramble applications (both in Ramble built-in and in Benchpark/repo)
+    (tags_stdout, tags_stderr) = run_command(f"{ramble_exe} attributes --tags --all")
+    ramble_applications_tags = {}
+    lines = tags_stdout.splitlines()
+
+    for line in lines:
+        key_value = line.split(":")
+        ramble_applications_tags[key_value[0]] = key_value[1].strip().split(",")
+
+    benchpark_experiments_tags = {}
+    for exp in experiments:
+        benchpark_experiments_tags[exp] = ramble_applications_tags[exp]
+    return benchpark_experiments_tags
+
+
+def benchpark_tags_handler(args):
+    """
+    Filter ramble tags by benchpark experiments
+    """
+    source_dir = source_location()
+    experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
+    ramble_location = experiments_root / "ramble"
+    ramble_exe = ramble_location / "bin" / "ramble"
+    experiments = benchpark_experiments()
+
+    if args.tag:
+        if benchpark_check_tag(args.tag):
+            # find all applications in Ramble that have a given tag (both in Ramble built-in and in Benchpark/repo)
+            (tag_stdout, tag_stderr) = run_command(f"{ramble_exe} list -t {args.tag}")
+            lines = tag_stdout.splitlines()
+
+            for line in lines:
+                if line in experiments:
+                    print(line)
+
+    elif args.application:
+        if benchpark_check_experiment(args.application):
+            benchpark_experiments_tags = helper_experiments_tags(
+                ramble_exe, experiments
+            )
+            print(benchpark_experiments_tags[args.application])
+    else:
+        benchpark_experiments_tags = helper_experiments_tags(ramble_exe, experiments)
+        print("All tags that exist in Benchpark experiments:")
+        for k, v in benchpark_experiments_tags.items():
+            print(k)
+
+
 if __name__ == "__main__":
     main()
diff --git a/docs/2-benchpark-list.rst b/docs/2-benchpark-list.rst
@@ -3,14 +3,43 @@
 
    SPDX-License-Identifier: Apache-2.0
 
-==============
-Benchpark List
-==============
+================
+Search Benchpark
+================
 
-To list all benchmarks and systems available in Benchpark::
+The user can search for available system and experiment specifications in Benchpark.
+
+.. list-table:: Searching for specifications in Benchpark
+   :widths: 25 25 50
+   :header-rows: 1
+
+   * - Command
+     - Description
+     - Listing in the docs
+   * - benchpark list
+     - Lists all benchmarks and systems specified in Benchpark
+     - 
+   * - benchpark list systems
+     - Lists all system specified in Benchpark
+     - :doc:`available-system-specs`
+   * - benchmark list benchmarks
+     - Lists all benchmarks specified in Benchpark
+     - 
+   * - benchpark list systems
+     - Lists all system specified in Benchpark
+     - :doc:`available-system-specs`
+   * - benchpark tags workspace
+     - Lists all tags specified in Benchpark
+     - 
+   * - benchpark tags -a application workspace
+     - Lists all tags specified for a given application in Benchpark
+     - 
+   * - benchpark tags -t tag workspace
+     - Lists all experiments in Benchpark with a given tag
+     - 
 
-    cd bin
-    benchpark list
 
 Once you have decided on a ``system`` you will use, and the ``benchmark/ProgrammingModel``
 to run, you can proceed to :doc:`4-benchpark-setup`.
+
+For a complete list of options, see the help menu in :doc:`benchpark-help`.
diff --git a/docs/Makefile b/docs/Makefile
@@ -7,6 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
+WORKSPACE_PATH ?=
 
 # Put it first so that "make" without argument is like "make help".
 help:
@@ -15,7 +16,10 @@ help:
 systemconfigs:
 	./generate-sys-defs-list.py
 
-.PHONY: help sysconfigs Makefile
+tags:
+	./generate-benchmark-list.py $(WORKSPACE_PATH)
+
+.PHONY: help systemconfigs tags Makefile
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).

diff --git a/docs/available-system-specs.rst b/docs/available-system-specs.rst
@@ -13,6 +13,6 @@ use as the ``system`` parameter in Benchpark setup. See
 :doc:`4-benchpark-setup` for more details.
 
 .. csv-table:: Current system definitions in Benchpark.
-   :file: tables/current-system-definitions.csv
+   :file: current-system-definitions.csv
    :header-rows: 1
    :align: left
diff --git a/docs/benchmark-list.rst b/docs/benchmark-list.rst
@@ -0,0 +1,13 @@
+.. Copyright 2023 Lawrence Livermore National Security, LLC and other
+   Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: Apache-2.0
+
+==============
+Benchmark List
+==============
+
+.. csv-table:: Current Benchpark tags by benchmark and tag groups.
+   :file: benchmark-list.csv
+   :header-rows: 1
+   :align: left
diff --git a/docs/benchpark-help.rst b/docs/benchpark-help.rst
@@ -0,0 +1,14 @@
+.. Copyright 2023 Lawrence Livermore National Security, LLC and other
+   Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: Apache-2.0
+
+===================
+Benchpark Help Menu
+===================
+
+Benchpark help menu::
+
+    $ benchpark --help
+
+.. program-output:: ../bin/benchpark --help
diff --git a/docs/conf.py b/docs/conf.py
@@ -15,6 +15,13 @@
     ]
 )
 
+subprocess.call(
+    [
+        "make",
+        "tags",
+    ]
+)
+
 project = "Benchpark"
 copyright = "2023, LLNS LLC"
 author = "Olga Pearce, Alec Scott, Peter Scheibel, Greg Becker, Riyaz Haque, and Nathan Hanford"
@@ -24,6 +31,7 @@
 
 extensions = [
     "sphinx_rtd_theme",
+    "sphinxcontrib.programoutput",
 ]
 
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".spack-env"]