From ec0081f24dd1be6a14877b7f3a574084bdd82e37 Mon Sep 17 00:00:00 2001 From: Joost van Zwieten Date: Sat, 11 May 2024 13:43:26 +0200 Subject: [PATCH] WIP: unittest --- .github/workflows/test.yaml | 34 ++++++++++++------ devtools/gha/coverage_report_xml.py | 55 ----------------------------- devtools/gha/report_coverage.py | 44 +++++++++++++++++++++++ devtools/gha/unittest.py | 50 ++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 65 deletions(-) delete mode 100644 devtools/gha/coverage_report_xml.py create mode 100644 devtools/gha/report_coverage.py create mode 100644 devtools/gha/unittest.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c63886eac..b3090689c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -68,10 +68,6 @@ jobs: MKL_NUM_THREADS: 1 PYTHONHASHSEED: 0 NUTILS_TENSORIAL: ${{ matrix.tensorial }} - # Use PEP669's sys.monitoring for faster coverage analysis, if available. - # This also fixes a significant regression in coverage analysis on Python - # 3.12. Related issue: https://github.com/python/cpython/issues/107674 . - COVERAGE_CORE: sysmon steps: - name: Checkout uses: actions/checkout@v4 @@ -95,7 +91,7 @@ jobs: _numpy_version: ${{ matrix.numpy-version }} run: | python -um pip install --upgrade --upgrade-strategy eager wheel - python -um pip install --upgrade --upgrade-strategy eager coverage numpy$_numpy_version + python -um pip install --upgrade --upgrade-strategy eager numpy$_numpy_version # Install Nutils from `dist` dir created in job `build-python-package`. python -um pip install "$_wheel[import_gmsh,export_mpl]" - name: Install Scipy @@ -107,11 +103,29 @@ jobs: python -um pip install --upgrade --upgrade-strategy eager mkl python -um devtools.gha.configure_mkl - name: Test - run: python -um coverage run -m unittest discover -b -q -t . -s tests - - name: Post-process coverage - run: python -um devtools.gha.coverage_report_xml - - name: Upload coverage - uses: codecov/codecov-action@v3 + run: python -um devtools.gha.unittest + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: _coverage_${{ matrix.name }} + path: target/coverage/ + if-no-files-found: error + process-coverage: + if: ${{ always() }} + needs: test + name: 'Process coverage' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: _coverage_* + path: target/coverage + merge-multiple: true + - name: Generete summary + run: python -um devtools.gha.report_coverage test-examples: needs: build-python-package name: 'Test examples ${{ matrix.os }}' diff --git a/devtools/gha/coverage_report_xml.py b/devtools/gha/coverage_report_xml.py deleted file mode 100644 index 795bdb8c9..000000000 --- a/devtools/gha/coverage_report_xml.py +++ /dev/null @@ -1,55 +0,0 @@ -import sys -import re -import os.path -from typing import Sequence -from xml.etree import ElementTree -from pathlib import Path -from coverage import Coverage - -paths = [] -for path in sys.path: - try: - paths.append(str(Path(path).resolve()).lower()+os.path.sep) - except FileNotFoundError: - pass -paths = list(sorted(paths, key=len, reverse=True)) -unix_paths = tuple(p.replace('\\', '/') for p in paths) -packages = tuple(p.replace('/', '.') for p in unix_paths) - -dst = Path('coverage.xml') - -# Generate `coverage.xml` with absolute file and package names. -cov = Coverage() -cov.load() -cov.xml_report(outfile=str(dst)) - -# Load the report, remove the largest prefix in `packages` from attribute -# `name` of element `package`, if any, and similarly the largest prefix in -# `paths` from attribute `filename` of element `class` and from the content of -# element `source`. Matching prefixes is case insensitive for case insensitive -# file systems. - - -def remove_prefix(value: str, prefixes: Sequence[str]) -> str: - lvalue = value.lower() - for prefix in prefixes: - if lvalue.startswith(prefix): - return value[len(prefix):] - return value - - -root = ElementTree.parse(str(dst)) -for elem in root.iter('package'): - for package in packages: - name = elem.get('name') - if name: - elem.set('name', remove_prefix(name, packages)) - for elem in root.iter('class'): - filename = elem.get('filename') - if filename: - elem.set('filename', remove_prefix(filename, unix_paths)) - for elem in root.iter('source'): - text = elem.text - if text: - elem.text = remove_prefix(text, paths) -root.write('coverage.xml') diff --git a/devtools/gha/report_coverage.py b/devtools/gha/report_coverage.py new file mode 100644 index 000000000..2d40ab946 --- /dev/null +++ b/devtools/gha/report_coverage.py @@ -0,0 +1,44 @@ +import itertools +import json +import os +from pathlib import Path + +cov_dir = Path() / 'target' / 'coverage' + +# Load and merge coverage data. +coverage = {} +for part in cov_dir.glob('*.json'): + with part.open('r') as f: + part = json.load(f) + for file_name, part_file_coverage in part.items(): + coverage.setdefault(file_name, []).append(part_file_coverage) +coverage = {file_name: list(map(max, *file_coverage)) if len(file_coverage) > 1 else file_coverage[0] for file_name, file_coverage in coverage.items()} + +with open(os.environ.get('GITHUB_STEP_SUMMARY', None) or cov_dir / 'summary.md', 'w') as f: + print('| Name | Stmts | Miss | Cover |', file=f) + print('| :--- | ----: | ---: | ----: |', file=f) + total_covered = 0 + total_executable = 0 + for file_name, file_coverage in sorted(coverage.items()): + + # Annotate lines with uncovered lines. + i = 0 + while i < len(file_coverage): + j = i + if file_coverage[i] == 1: + while j + 1 < len(file_coverage) and file_coverage[j + 1] == 1: + j += 1 + if i == j: + print(f'::warning file={file_name},line={i},endLine={j},title=Uncovered lines,Line {i} is not covered by tests') + else: + print(f'::warning file={file_name},line={i},endLine={j},title=Uncovered lines,Lines {i} - {j} are not covered by tests') + i = j + 1 + + file_covered = sum(status == 2 for status in file_coverage) + file_executable = sum(status != 0 for status in file_coverage) + file_percentage = 100 * file_covered / file_executable if file_executable else 0 + print(f'| `{file_name}` | {file_executable} | {file_covered} | {file_percentage:.1f}% |', file=f) + total_covered += file_covered + total_executable += file_executable + total_percentage = 100 * total_covered / total_executable if total_executable else 0 + print(f'| TOTAL | {total_executable} | {total_covered} | {total_percentage:.1f}% |', file=f) diff --git a/devtools/gha/unittest.py b/devtools/gha/unittest.py new file mode 100644 index 000000000..2adf9c07c --- /dev/null +++ b/devtools/gha/unittest.py @@ -0,0 +1,50 @@ +import importlib +import inspect +import json +import os +from pathlib import Path +import sys +import unittest + +source = importlib.util.find_spec('nutils').origin +assert source.endswith(os.sep + '__init__.py') +source = source[:-11] +coverage = {} + +if hasattr(sys, 'monitoring'): + + def start(code, _): + if isinstance(code.co_filename, str) and code.co_filename.startswith(source) and not sys.monitoring.get_local_events(sys.monitoring.COVERAGE_ID, code): + if (file_coverage := coverage.get(code.co_filename)) is None: + with open(code.co_filename, 'rb') as f: + nlines = sum(1 for _ in f) + coverage[code.co_filename] = file_coverage = [0] * (nlines + 1) + for _, _, l in code.co_lines(): + if l: + file_coverage[l] = 1 + sys.monitoring.set_local_events(sys.monitoring.COVERAGE_ID, code, sys.monitoring.events.LINE) + for obj in code.co_consts: + if inspect.iscode(obj): + start(obj, None) + return sys.monitoring.DISABLE + + def line(code, line_number): + coverage[code.co_filename][line_number] = 2 + return sys.monitoring.DISABLE + + sys.monitoring.register_callback(sys.monitoring.COVERAGE_ID, sys.monitoring.events.PY_START, start) + sys.monitoring.register_callback(sys.monitoring.COVERAGE_ID, sys.monitoring.events.LINE, line) + sys.monitoring.use_tool_id(sys.monitoring.COVERAGE_ID, 'test') + sys.monitoring.set_events(sys.monitoring.COVERAGE_ID, sys.monitoring.events.PY_START) + +loader = unittest.TestLoader() +suite = loader.discover('tests') +runner = unittest.TextTestRunner(buffer=True) +runner.run(suite) + +coverage = {file_name[len(source) - 7:]: file_coverage for file_name, file_coverage in coverage.items()} +cov_dir = (Path() / 'target' / 'coverage') +cov_dir.mkdir(parents=True, exist_ok=True) +cov_file = cov_dir / ((os.environ.get('GITHUB_JOB', '') or 'coverage') + '.json') +with cov_file.open('w') as f: + json.dump(coverage, f)