From 74772280e698918dbf82477482002ff2eb241223 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Sun, 1 Mar 2026 21:14:23 +0000 Subject: [PATCH] perf(tests): optimize coverage instrumentation and reporting pipeline Replace coverage.py (sys.settrace-based) with slipcover (bytecode-based instrumentation) for significantly faster coverage collection: - Each behave-parallel worker runs under slipcover, producing per-feature JSON coverage files with unique UUIDs to avoid write contention. - After all workers finish, slipcover --merge combines per-worker data into a single build/coverage.json report. - XML report generated via slipcover --merge --xml for CI tooling. - Terminal report with --fail-under=97 threshold enforcement. - Robust JSON key-fallback logic handles both slipcover and coverage.py output formats. - CI workflow (ci.yml, nightly-quality.yml) updated with defensive key lookup instead of hardcoded coverage.py format. - Documentation updated to reflect slipcover as the coverage tool. - CHANGELOG.md updated. ISSUES CLOSED: #482 --- .forgejo/workflows/ci.yml | 12 +- .forgejo/workflows/nightly-quality.yml | 3 +- CHANGELOG.md | 5 + docs/development/testing.md | 15 ++- noxfile.py | 158 ++++++++++++++++++++----- pyproject.toml | 2 +- 6 files changed, 154 insertions(+), 41 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index b3f37eb65..4ed41a549 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -183,7 +183,9 @@ jobs: python3 -c " import json, sys with open('build/coverage.json') as f: - pct = round(json.load(f)['totals']['percent_covered'], 1) + data = json.load(f) + summary = data.get('summary') or data.get('totals') or {} + pct = round(summary.get('percent_covered', 0), 1) threshold = 97 if pct >= threshold: print(f'COVERAGE OK: {pct}% (threshold: {threshold}%)') @@ -210,7 +212,7 @@ jobs: benchmark-regression: if: forgejo.event_name == 'pull_request' runs-on: docker - container: + container: image: python:3.13-slim needs: [lint, typecheck] steps: @@ -222,7 +224,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - + - name: Compute base commit id: hash run: | @@ -245,13 +247,13 @@ jobs: python -m pip install awscli mkdir -p build/asv/results aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results" build/asv/results --delete || true - + - name: Run asv continuous via nox env: ASV_BASE_SHA: ${{ steps.hash.outputs.ASV_BASE_SHA }} run: | nox -s benchmark_regression - + - name: Archive the results run: | tar cf /tmp/asv-results.tar build/asv/results build/asv/html diff --git a/.forgejo/workflows/nightly-quality.yml b/.forgejo/workflows/nightly-quality.yml index da6bc8467..42ec1a040 100644 --- a/.forgejo/workflows/nightly-quality.yml +++ b/.forgejo/workflows/nightly-quality.yml @@ -95,7 +95,8 @@ jobs: cov_path = Path('build/reports/coverage.json') if cov_path.exists(): cov_data = json.loads(cov_path.read_text()) - report['gates']['coverage'] = cov_data.get('totals', {}).get('percent_covered', 0) + summary = cov_data.get('summary') or cov_data.get('totals') or {} + report['gates']['coverage'] = summary.get('percent_covered', 0) # Complexity cx_path = Path('build/reports/complexity.json') diff --git a/CHANGELOG.md b/CHANGELOG.md index e0054d4d7..c82c0c8f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Replaced coverage.py (sys.settrace) with slipcover (bytecode instrumentation) for faster + coverage collection. Each behave-parallel worker now produces per-feature JSON coverage files; + slipcover --merge combines them. CI workflow JSON key lookups handle both slipcover and + coverage.py output formats. Documentation updated to reflect slipcover as the coverage + tool. (#482) - Added semantic validation service with AST-based rules for syntax errors, missing imports, broken references, duplicate imports, API misuse, and missing symbols. Includes rule registry, file-hash LRU cache, severity mapping, and ValidationPipeline integration. (#448) diff --git a/docs/development/testing.md b/docs/development/testing.md index 7df9cc3b1..a76ac8ea7 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -54,11 +54,14 @@ The project enforces a **minimum 97% code coverage** at all times. This is a har ### How Coverage Is Measured -Coverage is measured by running Behave tests under `coverage.py` in serial mode: +Coverage is measured by running Behave tests under `slipcover` (bytecode-based instrumentation) in parallel via behave-parallel. Each worker produces a per-feature JSON coverage file, then slipcover merges them: ``` -coverage run --source=src -m behave -q --tags=-discovery --no-capture features/ -coverage report --show-missing --fail-under=97 +# Each behave-parallel worker runs: +python -m slipcover --json --out build/.slipcover..json --source src -m behave +# After all workers finish: +python -m slipcover --merge build/.slipcover.*.json --json --out build/coverage.json +python -m slipcover --merge build/coverage.json --fail-under=97 ``` ### Coverage Output @@ -90,7 +93,9 @@ Three report formats are generated under `build/`: ### Coverage Configuration -Coverage settings live in `pyproject.toml`: +Coverage is collected by `slipcover` (bytecode-based, faster than `coverage.py`'s `sys.settrace`). +Source and omit patterns are passed directly to slipcover via the `coverage_report` nox session. +The `pyproject.toml` `[tool.coverage.run]` section is retained for any tools that still read it: ```toml [tool.coverage.run] @@ -119,7 +124,7 @@ directory = "build/htmlcov" output = "build/coverage.xml" ``` -The `--fail-under=97` threshold is enforced in the `coverage_report` nox session (`noxfile.py`). +The `--fail-under=97` threshold is enforced in the `coverage_report` nox session (`noxfile.py`) via `slipcover --fail-under=97`. ### How to Improve Coverage diff --git a/noxfile.py b/noxfile.py index 1cb4e19cc..578de4ae8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -97,6 +97,7 @@ import os import re import subprocess import sys +import uuid from pathlib import Path from typing import Dict, Iterable, List, Tuple @@ -196,7 +197,13 @@ def _print_overall_summary(total: Dict[str, Dict[str, int] | float]) -> None: def _run_feature( base_args: List[str], feature: str ) -> Tuple[int, str, str, str, Dict[str, Dict[str, int] | float]]: - proc = subprocess.run([*base_args, feature], capture_output=True, text=True) + args = list(base_args) + if "__SLIPCOVER_OUT__" in args: + output_dir = os.environ.get("SLIPCOVER_OUTPUT_DIR", "build") + output_name = f".slipcover.{uuid.uuid4().hex}.json" + output_path = os.path.join(output_dir, output_name) + args = [output_path if arg == "__SLIPCOVER_OUT__" else arg for arg in args] + proc = subprocess.run([*args, feature], capture_output=True, text=True) combined_output = (proc.stdout or "") + "\\n" + (proc.stderr or "") summary = _parse_summary(combined_output) return proc.returncode, feature, proc.stdout or "", proc.stderr or "", summary @@ -204,10 +211,21 @@ def _run_feature( def _build_base_args(other_args: List[str]) -> List[str]: if os.environ.get("BEHAVE_PARALLEL_COVERAGE"): - rcfile = os.environ.get("COVERAGE_RCFILE") - base_args = [sys.executable, "-m", "coverage", "run", "--parallel-mode"] - if rcfile: - base_args.extend(["--rcfile", rcfile]) + source = os.environ.get("SLIPCOVER_SOURCE") + omit = os.environ.get("SLIPCOVER_OMIT") + output_path = "__SLIPCOVER_OUT__" + base_args = [ + sys.executable, + "-m", + "slipcover", + "--json", + "--out", + output_path, + ] + if source: + base_args.extend(["--source", source]) + if omit: + base_args.extend(["--omit", omit]) base_args.extend(["-m", "behave", *other_args]) return base_args return [sys.executable, "-m", "behave", *other_args] @@ -486,9 +504,9 @@ def coverage_report(session: nox.Session): """Generate coverage report from Behave tests. Runs behave tests in parallel via behave-parallel (same approach as - unit_tests) with each worker collecting coverage independently using - ``coverage run --parallel-mode``. After all workers finish, the - per-process data files are merged with ``coverage combine``. + unit_tests) with each worker collecting slipcover JSON coverage data. + After all workers finish, the data files are merged with + ``slipcover --merge``. Coverage threshold is enforced at >=97%. @@ -503,13 +521,31 @@ def coverage_report(session: nox.Session): session.env["PYTHONPATH"] = str(Path("src").resolve()) session.env["NO_COLOR"] = "1" - session.env["COVERAGE_RCFILE"] = "pyproject.toml" - session.env["COVERAGE_FILE"] = "build/.coverage" - # Tell behave-parallel workers to run under coverage --parallel-mode + source_paths = "src" + omit_patterns = ",".join( + [ + "*/tests/*", + "*/test_*", + "features/*", + "*/features/*", + "*/__pycache__/*", + "*/site-packages/*", + "*/dependency_injector/*", + "*/venv/*", + "*/.venv/*", + "*/.nox/*", + "src/cleveragents/discovery/*", + ] + ) + session.env["SLIPCOVER_SOURCE"] = source_paths + session.env["SLIPCOVER_OMIT"] = omit_patterns + session.env["SLIPCOVER_OUTPUT_DIR"] = "build" + # Tell behave-parallel workers to run under slipcover session.env["BEHAVE_PARALLEL_COVERAGE"] = "1" - # Clean up any existing coverage data - session.run("coverage", "erase") + # Clean up any existing slipcover data + for path in Path("build").glob(".slipcover.*.json"): + path.unlink() # Run behave tests in parallel; each worker collects coverage separately behave_cmd = session.bin + "/behave-parallel" @@ -537,34 +573,98 @@ def coverage_report(session: nox.Session): session.run(*args) - # Merge per-worker coverage data files into a single build/.coverage - session.run("coverage", "combine") + data_files = sorted(Path("build").glob(".slipcover.*.json")) + if not data_files: + session.error("No slipcover data files found in build/") - # Always generate HTML and XML reports before checking threshold - session.run("coverage", "html") - session.run("coverage", "xml") + # Merge per-worker coverage data into a single JSON report + session.run( + "python", + "-m", + "slipcover", + "--merge", + *[str(path) for path in data_files], + "--json", + "--out", + "build/coverage.json", + ) + + # Generate XML report for CI + session.run( + "python", + "-m", + "slipcover", + "--merge", + "build/coverage.json", + "--xml", + "--out", + "build/coverage.xml", + ) # Generate terminal report and check threshold. - # coverage report --fail-under exits non-zero if below threshold, - # but nox intercepts the exit. We run it with success_codes to capture - # the failure and emit a clear, CI-parseable single-line summary. + # slipcover exits with code 2 if below threshold; nox intercepts the exit. + report_path = "build/coverage-report.txt" session.run( - "coverage", - "report", - "--show-missing", + "python", + "-m", + "slipcover", + "--merge", + "build/coverage.json", + "--out", + report_path, + "--source", + source_paths, + "--omit", + omit_patterns, f"--fail-under={COVERAGE_THRESHOLD}", success_codes=[0, 2], silent=False, ) - # Parse the total percentage from the coverage JSON report for the summary + try: + with open(report_path) as report_file: + print(report_file.read(), end="") + except FileNotFoundError: + session.log(f"Slipcover report not found at {report_path}") + + # Parse the total percentage from the slipcover JSON report for the summary total_pct: float = 0.0 try: - session.run("coverage", "json", "-o", "build/coverage.json", silent=True) with open("build/coverage.json") as f: - total_pct = json.load(f)["totals"]["percent_covered"] - except (FileNotFoundError, KeyError, json.JSONDecodeError): - # Fall back: if json report failed, re-read xml + payload = json.load(f) + + summary = payload.get("summary") or payload.get("totals") or {} + percent_value = None + for key in ( + "percent_covered", + "percent_covered_display", + "line_coverage_percent", + ): + percent_value = summary.get(key) + if percent_value is not None: + break + + if percent_value is None: + covered = ( + summary.get("lines_covered") + or summary.get("covered_lines") + or summary.get("covered") + ) + total = ( + summary.get("lines") + or summary.get("total_lines") + or summary.get("num_lines") + or summary.get("total") + ) + if covered is not None and total: + percent_value = (float(covered) / float(total)) * 100.0 + + if isinstance(percent_value, str): + percent_value = percent_value.strip().rstrip("%") + + if percent_value is not None: + total_pct = float(percent_value) + except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError): total_pct = 0.0 rounded_pct = round(total_pct, 1) diff --git a/pyproject.toml b/pyproject.toml index a494a221b..d276475a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dev = [ ] tests = [ "behave==1.3.3", - "coverage>=7.11.0", + "slipcover>=1.0.17", "asv>=0.6.5", "robotframework>=7.3.2", "robotframework-pabot>=4.0.0",