Merge branch 'develop' into patch-removing-magic

.github/workflows/bin/license

@@ -0,0 +1,184 @@
+#!/bin/env python3
+#
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import print_function
+
+import os
+import re
+from collections import defaultdict
+import sys
+
+#: SPDX license id must appear in the first <license_lines> lines of a file
+license_lines = 7
+
+#: Benchpark's license identifier
+apache_spdx = "Apache-2.0"
+
+#: regular expressions for licensed files.
+licensed_files = [
+    r"^bin\/benchpark",
+    r"^configs\/[^\/]*\/[^\/]*",
+    r"^docs\/[^\/]*\.rst$",
+    r"^experiments\/[^\/]*\/[^\/]*",
+    r"^repo\/[^\/]*\/[^\/]*",
+]
+
+
+def _all_files(root="."):
+    """Generates root-relative paths of all files in the repository."""
+    visited = set()
+    for cur_root, folders, files in os.walk(root):
+        for filename in files:
+            path = os.path.realpath(os.path.join(cur_root, filename))
+
+            if path not in visited:
+                yield os.path.relpath(path, root)
+                visited.add(path)
+
+
+def _licensed_files(root):
+    for relpath in _all_files(root):
+        if any(regex.match(relpath) for regex in licensed_files):
+            yield relpath
+
+
+def list_files(root):
+    """List files that should have license headers"""
+    for relpath in sorted(_licensed_files(root)):
+        print(os.path.join(".", relpath))
+
+
+# Error codes for license verification. All values are chosen such that
+# bool(value) evaluates to True
+SPDX_MISMATCH, GENERAL_MISMATCH, COPYRIGHT_YEAR_MISMATCH = range(1, 4)
+
+#: Latest year that copyright applies. UPDATE THIS when bumping copyright.
+latest_year = 2023
+strict_date = r"Copyright %s" % latest_year
+
+#: regexes for valid license lines at tops of files
+license_line_regexes = [
+    r"Copyright (%d|[0-9]{4}) Lawrence Livermore National Security, LLC and other"
+    % latest_year,
+    r"Benchpark Project Developers. See the top-level COPYRIGHT file for details.",
+    r"SPDX-License-Identifier: Apache-2.0",
+]
+
+
+class LicenseError(object):
+    def __init__(self):
+        self.error_counts = defaultdict(int)
+
+    def add_error(self, error):
+        self.error_counts[error] += 1
+
+    def has_errors(self):
+        return sum(self.error_counts.values()) > 0
+
+    def error_messages(self):
+        total = sum(self.error_counts.values())
+        missing = self.error_counts[GENERAL_MISMATCH]
+        spdx_mismatch = self.error_counts[SPDX_MISMATCH]
+        copyright_year_mismatch = self.error_counts[COPYRIGHT_YEAR_MISMATCH]
+        return (
+            "%d improperly licensed files\n" % (total),
+            "files with wrong SPDX-License-Identifier:   %d\n" % spdx_mismatch,
+            "files not containing expected license:      %d\n" % missing,
+            "files with wrong copyright year:            %d\n" % copyright_year_mismatch,
+        )
+
+
+def _check_license(lines, path):
+    found = []
+
+    for line in lines:
+        line = re.sub(r"^[\s#\%\.\!\/\/]*", "", line)
+        line = line.rstrip()
+        for i, line_regex in enumerate(license_line_regexes):
+            if re.match(line_regex, line):
+                # The first line of the license contains the copyright date.
+                # We allow it to be out of date but print a warning if it is
+                # out of date.
+                if i == 0:
+                    if not re.search(strict_date, line):
+                        print("{0}: Copyright date mismatch".format(path))
+                        return COPYRIGHT_YEAR_MISMATCH
+                found.append(i)
+
+    if len(found) == len(license_line_regexes) and found == list(sorted(found)):
+        return
+
+    # If the SPDX identifier is present, then there is a mismatch (since it
+    # did not match the above regex)
+    def wrong_spdx_identifier(line, path):
+        m = re.search(r"SPDX-License-Identifier: [^\n]*", line)
+        if m and m.group(1) != apache_spdx:
+            print(
+                "{0}: SPDX license identifier mismatch"
+                "(expecting {1}, found {2})".format(path, apache_spdx, m.group(1))
+            )
+            return SPDX_MISMATCH
+        else:
+            print("{0}: SPDX license identifier missing".format(path))
+            return GENERAL_MISMATCH
+
+    checks = [wrong_spdx_identifier]
+
+    for line in lines:
+        for check in checks:
+            error = check(line, path)
+            if error:
+                return error
+
+    print(
+        "{0}: the license header at the top of the file does not match the"
+        " expected format".format(path)
+    )
+    return GENERAL_MISMATCH
+
+
+def verify(root):
+    """Verify that files have the right license header"""
+
+    license_errors = LicenseError()
+
+    for relpath in _licensed_files(root):
+        path = os.path.join(root, relpath)
+        with open(path) as f:
+            lines = [line for line in f][:license_lines]
+
+        error = _check_license(lines, path)
+        if error:
+            license_errors.add_error(error)
+
+    if license_errors.has_errors():
+        print(*license_errors.error_messages())
+        sys.exit(1)
+    else:
+        print("No license issues found.")
+        return
+
+
+if __name__ == "__main__":
+    valid_options = ["list-files", "verify"]
+
+    if len(sys.argv) != 2:
+        print("Please specify a valid option: {}".format(valid_options))
+        sys.exit()
+
+    cmd = sys.argv[1]
+    if cmd not in valid_options:
+        print("Invalid argument. Valid options are {}".format(valid_options))
+        sys.exit()
+
+    licensed_files[:] = [re.compile(regex) for regex in licensed_files]
+    root = os.path.abspath(os.curdir)
+
+    if cmd == "list-files":
+        list_files(root)
+    elif cmd == "verify":
+        verify(root)

.github/workflows/ci.yml

@@ -20,6 +20,7 @@ jobs:
       docs: ${{ steps.filter.outputs.docs }}
       style: ${{ steps.filter.outputs.style }}
       run: ${{ steps.filter.outputs.run }}
+      license: ${{ steps.filter.outputs.license }}
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
@@ -49,6 +50,13 @@ jobs:
               - 'configs/**'
               - 'experiments/**'
               - 'repo/**'
+            license:
+              - '.github/**'
+              - 'bin/**'
+              - 'configs/**'
+              - 'docs/**'
+              - 'experiments/**'
+              - 'repo/**'
 
   docs:
     if: ${{ needs.changes.outputs.docs == 'true' }}
@@ -64,3 +72,8 @@ jobs:
     if: ${{ needs.changes.outputs.run == 'true' }}
     needs: changes
     uses: ./.github/workflows/run.yml
+
+  license:
+    if: ${{ needs.changes.outputs.license == 'true' }}
+    needs: changes
+    uses: ./.github/workflows/license.yml

.github/workflows/license.yml

@@ -0,0 +1,21 @@
+name: License Checks
+on:
+  # This Workflow can be triggered manually
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  verify-license:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Verify license headers
+        run: |
+          python .github/workflows/bin/license verify

COPYRIGHT

@@ -0,0 +1,16 @@
+Intellectual Property Notice
+------------------------------
+
+Benchpark is licensed under the Apache License, Version 2.0 (LICENSE-APACHE
+or http://www.apache.org/licenses/LICENSE-2.0).
+
+Copyrights and patents in the Benchpark project are retained by contributors.
+No copyright assignment is required to contribute to Benchpark.
+
+
+SPDX usage
+------------
+
+Individual files contain SPDX tags instead of the full license text.
+This enables machine processing of license information based on the SPDX
+License Identifiers that are available here: https://spdx.org/licenses/

NOTICE

@@ -0,0 +1,21 @@
+This work was produced under the auspices of the U.S. Department of
+Energy by Lawrence Livermore National Laboratory under Contract
+DE-AC52-07NA27344.
+
+This work was prepared as an account of work sponsored by an agency of
+the United States Government. Neither the United States Government nor
+Lawrence Livermore National Security, LLC, nor any of their employees
+makes any warranty, expressed or implied, or assumes any legal liability
+or responsibility for the accuracy, completeness, or usefulness of any
+information, apparatus, product, or process disclosed, or represents that
+its use would not infringe privately owned rights.
+
+Reference herein to any specific commercial product, process, or service
+by trade name, trademark, manufacturer, or otherwise does not necessarily
+constitute or imply its endorsement, recommendation, or favoring by the
+United States Government or Lawrence Livermore National Security, LLC.
+
+The views and opinions of authors expressed herein do not necessarily
+state or reflect those of the United States Government or Lawrence
+Livermore National Security, LLC, and shall not be used for advertising
+or product endorsement purposes.

bin/benchpark

@@ -1,4 +1,9 @@
 #!/usr/bin/env python3
+#
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
 
 import argparse
 import subprocess

configs/AWS-x86-ParallelCluster-3.7.2/README.txt

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 AWS x86 ParallelCluster 3.7.2
 -----------------------------
 

configs/AWS-x86-ParallelCluster-3.7.2/auxiliary_software_files/compilers.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 compilers:
 - compiler:
     spec: [email protected]

configs/AWS-x86-ParallelCluster-3.7.2/auxiliary_software_files/packages.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 packages:
   tar:
     externals:

configs/AWS-x86-ParallelCluster-3.7.2/spack.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 spack:
   packages:
     default-compiler:

configs/AWS-x86-ParallelCluster-3.7.2/variables.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 variables:
   batch_time: '02:00'
   mpi_command: 'srun -N {n_nodes} -n {n_ranks} --mpi=pmix --export=ALL,FI_EFA_USE_DEVICE_RDMA=1,FI_PROVIDER="efa",OMPI_MCA_mtl_base_verbose=100'

configs/ats2/auxiliary_software_files/compilers.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 compilers:
 - compiler:
     spec: [email protected]

configs/ats2/auxiliary_software_files/packages.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 packages:
   tar:
     externals:

configs/ats2/spack.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 spack:
   packages:
     default-compiler:

configs/ats2/variables.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 variables:
   gtl_flag: ''  # to be overwritten by tests that need GTL
   batch_time: '02:00'

configs/ats4/auxiliary_software_files/compilers.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 compilers:
 - compiler:
     spec: [email protected]

configs/ats4/auxiliary_software_files/packages.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 packages:
   all:
     require: target=x86_64

configs/ats4/spack.yaml

@@ -1,3 +1,8 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 spack:
   packages:
     default-compiler: