diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 229f28821..08f5cc187 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,10 +69,6 @@ jobs: MKL_NUM_THREADS: 1 PYTHONHASHSEED: 0 NUTILS_TENSORIAL: ${{ matrix.tensorial }} - # Use PEP669's sys.monitoring for faster coverage analysis, if available. - # This also fixes a significant regression in coverage analysis on Python - # 3.12. Related issue: https://github.com/python/cpython/issues/107674 . - COVERAGE_CORE: sysmon steps: - name: Checkout uses: actions/checkout@v4 @@ -96,7 +92,7 @@ jobs: _numpy_version: ${{ matrix.numpy-version }} run: | python -um pip install --upgrade --upgrade-strategy eager wheel - python -um pip install --upgrade --upgrade-strategy eager coverage numpy$_numpy_version + python -um pip install --upgrade --upgrade-strategy eager numpy$_numpy_version # Install Nutils from `dist` dir created in job `build-python-package`. python -um pip install "$_wheel[import_gmsh,export_mpl]" - name: Install Scipy @@ -108,11 +104,29 @@ jobs: python -um pip install --upgrade --upgrade-strategy eager mkl python -um devtools.gha.configure_mkl - name: Test - run: python -um coverage run -m unittest discover -b -q -t . -s tests - - name: Post-process coverage - run: python -um devtools.gha.coverage_report_xml - - name: Upload coverage - uses: codecov/codecov-action@v3 + run: python -um devtools.gha.unittest + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: _coverage_${{ matrix.name }} + path: target/coverage/ + if-no-files-found: error + process-coverage: + if: ${{ always() }} + needs: test + name: 'Process coverage' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: _coverage_* + path: target/coverage + merge-multiple: true + - name: Generete summary + run: python -um devtools.gha.report_coverage test-examples: needs: build-python-package name: 'Test examples ${{ matrix.os }}' diff --git a/devtools/gha/coverage_report_xml.py b/devtools/gha/coverage_report_xml.py deleted file mode 100644 index 795bdb8c9..000000000 --- a/devtools/gha/coverage_report_xml.py +++ /dev/null @@ -1,55 +0,0 @@ -import sys -import re -import os.path -from typing import Sequence -from xml.etree import ElementTree -from pathlib import Path -from coverage import Coverage - -paths = [] -for path in sys.path: - try: - paths.append(str(Path(path).resolve()).lower()+os.path.sep) - except FileNotFoundError: - pass -paths = list(sorted(paths, key=len, reverse=True)) -unix_paths = tuple(p.replace('\\', '/') for p in paths) -packages = tuple(p.replace('/', '.') for p in unix_paths) - -dst = Path('coverage.xml') - -# Generate `coverage.xml` with absolute file and package names. -cov = Coverage() -cov.load() -cov.xml_report(outfile=str(dst)) - -# Load the report, remove the largest prefix in `packages` from attribute -# `name` of element `package`, if any, and similarly the largest prefix in -# `paths` from attribute `filename` of element `class` and from the content of -# element `source`. Matching prefixes is case insensitive for case insensitive -# file systems. - - -def remove_prefix(value: str, prefixes: Sequence[str]) -> str: - lvalue = value.lower() - for prefix in prefixes: - if lvalue.startswith(prefix): - return value[len(prefix):] - return value - - -root = ElementTree.parse(str(dst)) -for elem in root.iter('package'): - for package in packages: - name = elem.get('name') - if name: - elem.set('name', remove_prefix(name, packages)) - for elem in root.iter('class'): - filename = elem.get('filename') - if filename: - elem.set('filename', remove_prefix(filename, unix_paths)) - for elem in root.iter('source'): - text = elem.text - if text: - elem.text = remove_prefix(text, paths) -root.write('coverage.xml') diff --git a/devtools/gha/report_coverage.py b/devtools/gha/report_coverage.py new file mode 100644 index 000000000..931c33cff --- /dev/null +++ b/devtools/gha/report_coverage.py @@ -0,0 +1,29 @@ +import json +import os +from pathlib import Path + +coverage = {} +cov_dir = Path() / 'target' / 'coverage' + +for part in cov_dir.glob('*.json'): + with part.open('r') as f: + part = json.load(f) + for file_name, part_file_coverage in part.items(): + file_coverage = coverage.setdefault(file_name, {}) + for line, covered in part_file_coverage.items(): + file_coverage[line] = file_coverage.get(line, False) | covered + +with open(os.environ.get('GITHUB_STEP_SUMMARY', None) or cov_dir / 'summary.md', 'w') as f: + print('| Name | Stmts | Miss | Cover |', file=f) + print('| ---- | ----: | ---: | ----: |', file=f) + total_covered = 0 + total_executable = 0 + for file_name, file_coverage in sorted(coverage.items()): + file_covered = sum(file_coverage.values()) + file_executable = len(file_coverage) + file_percentage = 100 * file_covered / file_executable if file_executable else 0 + print(f'| {file_name} | {file_executable} | {file_covered} | {file_percentage:.1f}% |', file=f) + total_covered += file_covered + total_executable += file_executable + total_percentage = 100 * total_covered / total_executable if total_executable else 0 + print(f'| TOTAL | {total_executable} | {total_covered} | {total_percentage:.1f}% |', file=f) diff --git a/devtools/gha/unittest.py b/devtools/gha/unittest.py new file mode 100644 index 000000000..6d5e954f0 --- /dev/null +++ b/devtools/gha/unittest.py @@ -0,0 +1,49 @@ +import inspect +import json +import os +from pathlib import Path +import sys +import unittest + +source = str(Path().resolve() / 'nutils') + os.sep +coverage = {} + +if hasattr(sys, 'monitoring'): + + def start(code, _): + if isinstance(code.co_filename, str) and code.co_filename.startswith(source) and not sys.monitoring.get_local_events(sys.monitoring.COVERAGE_ID, code): + file_coverage = coverage.setdefault(code.co_filename, {}) + for _, _, l in code.co_lines(): + if l: + file_coverage.setdefault(l, False) + sys.monitoring.set_local_events(sys.monitoring.COVERAGE_ID, code, sys.monitoring.events.LINE) + for obj in code.co_consts: + if inspect.iscode(obj): + start(obj, None) + elif isinstance(code.co_filename, str) and 'nutils' in code.co_filename: + print(code.co_filename) + print(source) + raise ValueError + return sys.monitoring.DISABLE + + def line(code, line_number): + coverage.get(code.co_filename)[line_number] = True + return sys.monitoring.DISABLE + + sys.monitoring.register_callback(sys.monitoring.COVERAGE_ID, sys.monitoring.events.PY_START, start) + sys.monitoring.register_callback(sys.monitoring.COVERAGE_ID, sys.monitoring.events.LINE, line) + sys.monitoring.use_tool_id(sys.monitoring.COVERAGE_ID, 'test') + sys.monitoring.set_events(sys.monitoring.COVERAGE_ID, sys.monitoring.events.PY_START) + +loader = unittest.TestLoader() +#suite = loader.discover('tests') +suite = loader.loadTestsFromName('tests.test_unit') +runner = unittest.TextTestRunner(buffer=True) +runner.run(suite) + +coverage = {file_name[len(source) - 7:]: file_coverage for file_name, file_coverage in coverage.items()} +cov_dir = (Path() / 'target' / 'coverage') +cov_dir.mkdir(parents=True, exist_ok=True) +cov_file = cov_dir / ((os.environ.get('GITHUB_JOB', '') or 'coverage') + '.json') +with cov_file.open('w') as f: + json.dump(coverage, f)