Merge pull request #2749 from jku/test-fixes

Unit test infrastructure fixes

.github/workflows/_test.yml

@@ -74,14 +74,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_FLAG_NAME: ${{ runner.os }} / Python ${{ matrix.python-version }}
           COVERALLS_PARALLEL: true
-        # Use cp workaround to publish coverage reports with relative paths
-        # FIXME: Consider refactoring the tests to not require the test
-        # aggregation script being invoked from the `tests` directory, so
-        # that `.coverage` is written to and .coveragrc can also reside in
-        # the project root directory as is the convention.
         run: |
-          cp tests/.coverage .
-          coveralls --service=github --rcfile=tests/.coveragerc
+          coveralls --service=github
 
   coveralls-fin:
     # Always run when all 'tests' jobs have finished even if they failed

.github/workflows/specification-version-check.yml

@@ -20,7 +20,6 @@ jobs:
           python-version: "3.x"
       - id: get-version
         run: |
-          python3 -m pip install -e .
           script="from tuf.api.metadata import SPECIFICATION_VERSION; \
                   print(f\"v{'.'.join(SPECIFICATION_VERSION)}\")"
           ver=$(python3 -c "$script")

docs/CONTRIBUTING.rst

@@ -39,21 +39,18 @@ you need debug/run outside ``tox``.
 Unit tests
 ----------
 
-More specifically, the Update Framework's test suite can be executed by invoking
-the test aggregation script inside the *tests* subdirectory. ``tuf`` and its
-dependencies must already be installed.
+test suite can be executed directly as well (in this case the environment managed by tox is
+not used):
 ::
 
-     cd tests/
-     python3 aggregate_tests.py
+     python3 -m unittest
 
 
 Individual tests can also be executed. Optional ``-v`` flags can be added to
 increase log level up to DEBUG (``-vvvv``).
 ::
 
-     cd tests/
-     python3 test_updater_ng.py -v
+     python3 tests/test_updater_ng.py -v
 
 
 Coverage
@@ -64,8 +61,7 @@ invoked with the ``coverage`` tool (requires installation of ``coverage``, e.g.
 via PyPI).
 ::
 
-     cd tests/
-     coverage run aggregate_tests.py && coverage report
+     coverage run -m unittest
 
 
 Auto-formatting

pyproject.toml

@@ -70,11 +70,6 @@ include = [
   "/setup.py",
 ]
 
-[tool.hatch.build.targets.wheel]
-# The testing phase changes the current working directory to `tests` but the test scripts import
-# from `tests` so the root directory must be added to Python's path for editable installations
-dev-mode-dirs = ["."]
-
 # Ruff section
 # Read more here: https://docs.astral.sh/ruff/linter/#rule-selection
 [tool.ruff]
@@ -158,4 +153,7 @@ exclude_also = [
     "raise AssertionError",
     # imports for mypy only
     "if TYPE_CHECKING",
-]
+]
+[tool.coverage.run]
+branch = true
+omit = [ "tests/*" ]

requirements/dev.txt

@@ -1,8 +1,3 @@
-# Install tuf in editable mode and requirements for local testing with tox,
-# and also for running test suite or individual tests manually.
-# The build and tox versions specified here are also used as constraints
-# during CI and CD Github workflows
 -r build.txt
 -r test.txt
 -r lint.txt
--e .

tests/generated_data/generate_md.py

@@ -56,9 +56,6 @@
     signers.append(CryptoSigner(private_key, key))
 
 EXPIRY = datetime(2050, 1, 1, tzinfo=timezone.utc)
-OUT_DIR = "generated_data/ed25519_metadata"
-if not os.path.exists(OUT_DIR):
-    os.mkdir(OUT_DIR)
 
 SERIALIZER = JSONSerializer()
 
@@ -80,15 +77,13 @@ def verify_generation(md: Metadata, path: str) -> None:
             )
 
 
-def generate_all_files(
-    dump: bool | None = False, verify: bool | None = False
-) -> None:
-    """Generate a new repository and optionally verify it.
+def generate_all_files(dump: bool = False) -> None:
+    """Generate a new repository or verify that output has not changed.
 
     Args:
-        dump: Wheter to dump the newly generated files.
-        verify: Whether to verify the newly generated files with the
-            local staored.
+        dump: If True, new files are generated. If False, existing files
+           are compared to generated files and an exception is raised if
+           there are differences.
     """
     md_root = Metadata(Root(expires=EXPIRY))
     md_timestamp = Metadata(Timestamp(expires=EXPIRY))
@@ -103,12 +98,16 @@ def generate_all_files(
     for i, md in enumerate([md_root, md_timestamp, md_snapshot, md_targets]):
         assert isinstance(md, Metadata)
         md.sign(signers[i])
-        path = os.path.join(OUT_DIR, f"{md.signed.type}_with_ed25519.json")
-        if verify:
-            verify_generation(md, path)
-
+        path = os.path.join(
+            utils.TESTS_DIR,
+            "generated_data",
+            "ed25519_metadata",
+            f"{md.signed.type}_with_ed25519.json",
+        )
         if dump:
             md.to_file(path, SERIALIZER)
+        else:
+            verify_generation(md, path)
 
 
 if __name__ == "__main__":

tests/test_fetcher_ng.py

@@ -10,8 +10,7 @@
 import sys
 import tempfile
 import unittest
-from collections.abc import Iterator
-from typing import Any, ClassVar
+from typing import ClassVar
 from unittest.mock import Mock, patch
 
 import requests
@@ -163,11 +162,11 @@ def test_download_file_upper_length(self) -> None:
             self.assertEqual(self.file_length, temp_file.tell())
 
     # Download a file bigger than expected
-    def test_download_file_length_mismatch(self) -> Iterator[Any]:
-        with self.assertRaises(exceptions.DownloadLengthMismatchError):
-            # Force download_file to execute and raise the error since it is a
-            # context manager and returns Iterator[IO]
-            yield self.fetcher.download_file(self.url, self.file_length - 4)
+    def test_download_file_length_mismatch(self) -> None:
+        with self.assertRaises(
+            exceptions.DownloadLengthMismatchError
+        ), self.fetcher.download_file(self.url, self.file_length - 4):
+            pass  # we never get here as download_file() raises
 
 
 # Run unit test.

tests/test_metadata_generation.py

@@ -16,7 +16,7 @@ class TestMetadataGeneration(unittest.TestCase):
     @staticmethod
     def test_compare_static_md_to_generated() -> None:
         # md_generator = MetadataGenerator("generated_data/ed25519_metadata")
-        generate_all_files(dump=False, verify=True)
+        generate_all_files(dump=False)
 
 
 # Run unit test.

tests/test_utils.py

@@ -19,6 +19,7 @@
 """
 
 import logging
+import os
 import socket
 import sys
 import unittest
@@ -56,7 +57,7 @@ def test_simple_server_startup(self) -> None:
     def test_cleanup(self) -> None:
         # Test normal case
         server_process_handler = utils.TestServerProcess(
-            log=logger, server="simple_server.py"
+            log=logger, server=os.path.join(utils.TESTS_DIR, "simple_server.py")
         )
 
         server_process_handler.clean()

tox.ini

@@ -9,21 +9,13 @@ envlist = lint,docs,py
 skipsdist = true
 
 [testenv]
-# TODO: Consider refactoring the tests to not require the aggregation script
-# being invoked from the `tests` directory. This seems to be the convention and
-# would make use of other testing tools such as coverage/coveralls easier.
-changedir = tests
-
 commands =
     python3 --version
-    python3 -m coverage run aggregate_tests.py
-    python3 -m coverage report --rcfile {toxinidir}/pyproject.toml -m --fail-under 97
+    python3 -m coverage run -m unittest
+    python3 -m coverage report -m --fail-under 97
 
 deps =
     -r{toxinidir}/requirements/test.txt
-    # Install TUF in editable mode, instead of tox default virtual environment
-    # installation (see `skipsdist`), to get relative paths in coverage reports
-    --editable {toxinidir}
 
 install_command = python3 -m pip install {opts} {packages}
 
@@ -37,14 +29,12 @@ commands_pre =
     python3 -m pip install --force-reinstall git+https://github.com/secure-systems-lab/securesystemslib.git@main#egg=securesystemslib[crypto]
 
 commands =
-    python3 -m coverage run aggregate_tests.py
-    python3 -m coverage report  --rcfile {toxinidir}/pyproject.toml -m
+    python3 -m coverage run -m unittest
+    python3 -m coverage report -m
 
 [testenv:lint]
-changedir = {toxinidir}
 deps =
     -r{toxinidir}/requirements/lint.txt
-    --editable {toxinidir}
 lint_dirs = tuf examples tests verify_release .github/scripts
 passenv = RUFF_OUTPUT_FORMAT
 commands =
@@ -54,7 +44,6 @@ commands =
     mypy {[testenv:lint]lint_dirs}
 
 [testenv:fix]
-changedir = {toxinidir}
 deps = {[testenv:lint]deps}
 commands =
     ruff check --fix {[testenv:lint]lint_dirs}
@@ -64,6 +53,5 @@ commands =
 deps =
     -r{toxinidir}/requirements/docs.txt
 
-changedir = {toxinidir}
 commands =
     sphinx-build -b html docs docs/build/html -W