Convert tox to nox

.github/workflows/flake8.yaml

@@ -11,11 +11,11 @@ jobs:
       - name: Setup python for flake8
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - uses: actions/checkout@v3
-      - name: Install tox
-        run: python -m pip install tox
+      - name: Install nox
+        run: python -m pip install nox
       - name: Setup flake8
-        run: tox --notest -e flake8
+        run: nox --force-color -e flake8 --install-only
       - name: Run flake8
-        run: tox -e flake8
+        run: nox --force-color -e flake8 -vv

.github/workflows/test.yaml

@@ -1,48 +1,78 @@
 name: test
 on:
+  workflow_dispatch:
   push:
   pull_request:
 
 jobs:
+  pre-commit:
+    name: pre-commit
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pre-commit
+          pre-commit install
+      - name: Run pre-commit
+        run: pre-commit run --all-files
+
   test:
     name: test ${{ matrix.py }} - ${{ matrix.netapi }} - ${{ matrix.salt }}
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         py:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
+          - "3.11"
         netapi:
           - "cherrypy"
           - "tornado"
         salt:
-          - "v3004.2"
-          - "v3005.1"
-          - "v3006.0"
-          - "master"
-        exclude:
-          - py: "3.10"
-            salt: "v3004.2"
-          - py: "3.10"
-            salt: "v3005.1"
+          - "v3006.5"
+
     steps:
-      - name: Setup python for test ${{ matrix.py }}
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.py }}
-      - uses: actions/checkout@v3
+
       - name: Install setuptools_scm
         run: python -m pip install setuptools_scm
-      - name: Install tox
-        run: python -m pip install tox
+
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y libc6-dev libffi-dev gcc git openssh-server libzmq3-dev
         env:
           DEBIAN_FRONTEND: noninteractive
-      - name: Setup tests
-        run: tox --notest -e py${{ matrix.py }}-${{ matrix.netapi }}-${{ matrix.salt }}
-      - name: Run tests
-        run: tox -e py${{ matrix.py }}-${{ matrix.netapi }}-${{ matrix.salt }}
+
+      - name: Install Nox
+        run: |
+          python -m pip install --upgrade pip
+          pip install nox
+
+      - name: Install Test Requirements
+        env:
+          SALT_REQUIREMENT: salt==${{ matrix.salt }}
+          PYTHON_VERSIONS: ${{ matrix.py }}
+          API_BACKEND: ${{ matrix.netapi }}
+        run: |
+          nox --force-color -e tests --install-only
+
+      - name: Test
+        env:
+          SALT_REQUIREMENT: salt==${{ matrix.salt }}
+          PYTHON_VERSIONS: ${{ matrix.py }}
+          API_BACKEND: ${{ matrix.netapi }}
+          SKIP_REQUIREMENTS_INSTALL: YES
+        run: |
+          nox --force-color -e tests -- -vv tests/

.pre-commit-config.yaml

@@ -0,0 +1,64 @@
+---
+minimum_pre_commit_version: 3.5.0
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-merge-conflict  # Check for files that contain merge conflict strings.
+      - id: trailing-whitespace   # Trims trailing whitespace.
+        args: [--markdown-linebreak-ext=md]
+      - id: mixed-line-ending     # Replaces or checks mixed line ending.
+        args: [--fix=lf]
+      - id: end-of-file-fixer     # Makes sure files end in a newline and only a newline.
+      - id: check-merge-conflict  # Check for files that contain merge conflict strings.
+      - id: check-ast             # Simply check whether files parse as valid python.
+
+  # ----- Formatting ---------------------------------------------------------------------------->
+  - repo: https://github.com/myint/autoflake
+    rev: v2.2.1
+    hooks:
+      - id: autoflake
+        name: Remove unused variables and imports
+        language: python
+        args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variables", "--expand-star-imports"]
+        files: \.py$
+
+  - repo: https://github.com/saltstack/pre-commit-remove-import-headers
+    rev: 1.1.0
+    hooks:
+      - id: remove-import-headers
+
+  - repo: https://github.com/s0undt3ch/salt-rewrite
+    # Automatically rewrite code with known rules
+    rev: 2.4.2
+    hooks:
+      - id: salt-rewrite
+        alias: rewrite-tests
+        name: Rewrite the test suite
+        files: ^tests/.*\.py$
+        args: [--silent, -E, fix_docstrings]
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+    hooks:
+      - id: pyupgrade
+        name: Rewrite Code to be Py3+
+        args: [
+          --py3-plus
+        ]
+
+  - repo: https://github.com/asottile/reorder_python_imports
+    rev: v3.12.0
+    hooks:
+      - id: reorder-python-imports
+        args: [
+          --py3-plus,
+        ]
+
+  - repo: https://github.com/psf/black
+    rev: 23.12.1
+    hooks:
+      - id: black
+        args: [-l 100]
+
+  # <---- Formatting -----------------------------------------------------------------------------

noxfile.py

@@ -0,0 +1,175 @@
+import datetime
+import os
+import pathlib
+import shutil
+
+import nox
+from nox.command import CommandFailed
+
+
+API_BACKEND = [os.environ.get("API_BACKEND")] or ["cherrypy", "tornado"]
+PYTHON_VERSIONS = [os.environ.get("PYTHON_VERSIONS")] or ["3.8", "3.9", "3.10", "3.11"]
+
+# Nox options
+# Reuse existing virtualenvs
+nox.options.reuse_existing_virtualenvs = True
+#  Don't fail on missing interpreters
+nox.options.error_on_missing_interpreters = False
+
+# Be verbose when running under a CI context
+CI_RUN = os.environ.get("CI")
+PIP_INSTALL_SILENT = CI_RUN is False
+SKIP_REQUIREMENTS_INSTALL = "SKIP_REQUIREMENTS_INSTALL" in os.environ
+EXTRA_REQUIREMENTS_INSTALL = os.environ.get("EXTRA_REQUIREMENTS_INSTALL")
+SALT_REQUIREMENT = os.environ.get("SALT_REQUIREMENT") or "salt"
+COVERAGE_VERSION_REQUIREMENT = "coverage==5.5"
+
+# Prevent Python from writing bytecode
+os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
+
+# Global Path Definitions
+REPO_ROOT = pathlib.Path(__file__).resolve().parent
+# Change current directory to REPO_ROOT
+os.chdir(str(REPO_ROOT))
+
+ARTIFACTS_DIR = REPO_ROOT / "artifacts"
+# Make sure the artifacts directory exists
+ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True)
+RUNTESTS_LOGFILE = ARTIFACTS_DIR / "runtests-{}.log".format(
+    datetime.datetime.now().strftime("%Y%m%d%H%M%S.%f")
+)
+COVERAGE_REPORT_DB = REPO_ROOT / ".coverage"
+COVERAGE_REPORT_PROJECT = ARTIFACTS_DIR.relative_to(REPO_ROOT) / "coverage-project.xml"
+COVERAGE_REPORT_TESTS = ARTIFACTS_DIR.relative_to(REPO_ROOT) / "coverage-tests.xml"
+JUNIT_REPORT = ARTIFACTS_DIR.relative_to(REPO_ROOT) / "junit-report.xml"
+
+
+def _install_requirements(
+    session,
+    *passed_requirements,  # pylint: disable=unused-argument
+    install_coverage_requirements=True,
+    install_test_requirements=True,
+    install_source=True,
+    install_salt=True,
+    install_extras=None,
+):
+    install_extras = install_extras or []
+    if SKIP_REQUIREMENTS_INSTALL is False:
+        # Always have the wheel package installed
+        session.install("--progress-bar=off", "wheel", silent=PIP_INSTALL_SILENT)
+        if install_coverage_requirements:
+            session.install(
+                "--progress-bar=off", COVERAGE_VERSION_REQUIREMENT, silent=PIP_INSTALL_SILENT
+            )
+        # Install the latest salt package
+        if install_salt:
+            session.install("--progress-bar=off", SALT_REQUIREMENT, silent=PIP_INSTALL_SILENT)
+
+        if install_test_requirements:
+            requirements_file = REPO_ROOT / "tests" / "requirements.txt"
+            install_command = [
+                "--progress-bar=off",
+                "-r",
+                str(requirements_file.relative_to(REPO_ROOT)),
+            ]
+            session.install(*install_command, silent=PIP_INSTALL_SILENT)
+
+        if EXTRA_REQUIREMENTS_INSTALL:
+            session.log(
+                "Installing the following extra requirements because the "
+                "EXTRA_REQUIREMENTS_INSTALL environment variable was set: "
+                "EXTRA_REQUIREMENTS_INSTALL='%s'",
+                EXTRA_REQUIREMENTS_INSTALL,
+            )
+            install_command = ["--progress-bar=off"]
+            install_command += [req.strip() for req in EXTRA_REQUIREMENTS_INSTALL.split()]
+            session.install(*install_command, silent=PIP_INSTALL_SILENT)
+
+        if install_source:
+            pkg = "."
+            if install_extras:
+                pkg += f"[{','.join(install_extras)}]"
+            session.install("-e", pkg, silent=PIP_INSTALL_SILENT)
+        elif install_extras:
+            pkg = f".[{','.join(install_extras)}]"
+            session.install(pkg, silent=PIP_INSTALL_SILENT)
+
+
+"""
+Nox session to run tests for corresponding python versions and salt versions
+"""
+
+
+@nox.session(python=PYTHON_VERSIONS)
+@nox.parametrize("api_backend", API_BACKEND)
+def tests(session, api_backend):
+    _install_requirements(session)
+
+    args = [
+        "--rootdir",
+        str(REPO_ROOT),
+        f"--log-file={RUNTESTS_LOGFILE.relative_to(REPO_ROOT)}",
+        "--log-file-level=debug",
+        "--show-capture=no",
+        f"--junitxml={JUNIT_REPORT}",
+        "--showlocals",
+        "-ra",
+        "-s",
+    ]
+
+    if session._runner.global_config.forcecolor:
+        args.append("--color=yes")
+    if not session.posargs:
+        args.append("tests/")
+    else:
+        for arg in session.posargs:
+            if arg.startswith("--color") and args[0].startswith("--color"):
+                args.pop(0)
+            args.append(arg)
+        for arg in session.posargs:
+            if arg.startswith("-"):
+                continue
+            if arg.startswith(f"tests{os.sep}"):
+                break
+            try:
+                pathlib.Path(arg).resolve().relative_to(REPO_ROOT / "tests")
+                break
+            except ValueError:
+                continue
+        else:
+            args.append("tests/")
+    try:
+        session.run(
+            "coverage", "run", "-m", "pytest", f"--salt-api-backend=rest_{api_backend}", *args
+        )
+    finally:
+        # Always combine and generate the XML coverage report
+        try:
+            session.run("coverage", "combine")
+        except CommandFailed:
+            # Sometimes some of the coverage files are corrupt which would
+            # trigger a CommandFailed exception
+            pass
+
+        # Generate report for salt code coverage
+        session.run(
+            "coverage",
+            "xml",
+            "-o",
+            str(COVERAGE_REPORT_PROJECT),
+        )
+        try:
+            session.run("coverage", "report", "--show-missing", "--include=pepper/*")
+        finally:
+            # Move the coverage DB to artifacts/coverage in order for it to be archived by CI
+            if COVERAGE_REPORT_DB.exists():
+                shutil.move(str(COVERAGE_REPORT_DB), str(ARTIFACTS_DIR / COVERAGE_REPORT_DB.name))
+
+
+@nox.session(python="3.10")
+def flake8(session):
+    _install_requirements(session)
+    # Install flake8
+    session.install("flake8")
+    # Run flake8
+    session.run("flake8", "tests/", "pepper/", "scripts/pepper", "setup.py")

pyproject.toml

@@ -0,0 +1,8 @@
+[tool.pytest.ini_options]
+addopts = "--showlocals --log-file /tmp/pepper-runtests.log --show-capture=no -ra"
+testpaths = ["tests"]
+norecursedirs = [".git", ".nox"]
+usefixtures = ["pepperconfig"]
+
+[tool.flake8]
+max-line-length = 119