From a5bd80471e2fae30cf3bc5b2d6b8cc1e4f28dbf9 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Sun, 1 Mar 2026 00:12:34 +0000 Subject: [PATCH 1/2] chore(coverage): speed up coverage reporting Use slipcover with per-feature JSON merge and src-only line totals to keep the 97% gate stable. --- noxfile.py | 161 ++++++++++++++++++++++++++++++++++++++++--------- pyproject.toml | 2 +- 2 files changed, 132 insertions(+), 31 deletions(-) diff --git a/noxfile.py b/noxfile.py index 1cb4e19cc..35557f2a2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -97,6 +97,7 @@ import os import re import subprocess import sys +import uuid from pathlib import Path from typing import Dict, Iterable, List, Tuple @@ -196,7 +197,13 @@ def _print_overall_summary(total: Dict[str, Dict[str, int] | float]) -> None: def _run_feature( base_args: List[str], feature: str ) -> Tuple[int, str, str, str, Dict[str, Dict[str, int] | float]]: - proc = subprocess.run([*base_args, feature], capture_output=True, text=True) + args = list(base_args) + if "__SLIPCOVER_OUT__" in args: + output_dir = os.environ.get("SLIPCOVER_OUTPUT_DIR", "build") + output_name = f".slipcover.{uuid.uuid4().hex}.json" + output_path = os.path.join(output_dir, output_name) + args = [output_path if arg == "__SLIPCOVER_OUT__" else arg for arg in args] + proc = subprocess.run([*args, feature], capture_output=True, text=True) combined_output = (proc.stdout or "") + "\\n" + (proc.stderr or "") summary = _parse_summary(combined_output) return proc.returncode, feature, proc.stdout or "", proc.stderr or "", summary @@ -204,10 +211,21 @@ def _run_feature( def _build_base_args(other_args: List[str]) -> List[str]: if os.environ.get("BEHAVE_PARALLEL_COVERAGE"): - rcfile = os.environ.get("COVERAGE_RCFILE") - base_args = [sys.executable, "-m", "coverage", "run", "--parallel-mode"] - if rcfile: - base_args.extend(["--rcfile", rcfile]) + source = os.environ.get("SLIPCOVER_SOURCE") + omit = os.environ.get("SLIPCOVER_OMIT") + output_path = "__SLIPCOVER_OUT__" + base_args = [ + sys.executable, + "-m", + "slipcover", + "--json", + "--out", + output_path, + ] + if source: + base_args.extend(["--source", source]) + if omit: + base_args.extend(["--omit", omit]) base_args.extend(["-m", "behave", *other_args]) return base_args return [sys.executable, "-m", "behave", *other_args] @@ -486,9 +504,9 @@ def coverage_report(session: nox.Session): """Generate coverage report from Behave tests. Runs behave tests in parallel via behave-parallel (same approach as - unit_tests) with each worker collecting coverage independently using - ``coverage run --parallel-mode``. After all workers finish, the - per-process data files are merged with ``coverage combine``. + unit_tests) with each worker collecting slipcover JSON coverage data. + After all workers finish, the data files are merged with + ``slipcover --merge``. Coverage threshold is enforced at >=97%. @@ -503,13 +521,31 @@ def coverage_report(session: nox.Session): session.env["PYTHONPATH"] = str(Path("src").resolve()) session.env["NO_COLOR"] = "1" - session.env["COVERAGE_RCFILE"] = "pyproject.toml" - session.env["COVERAGE_FILE"] = "build/.coverage" - # Tell behave-parallel workers to run under coverage --parallel-mode + source_paths = "src" + omit_patterns = ",".join( + [ + "*/tests/*", + "*/test_*", + "features/*", + "*/features/*", + "*/__pycache__/*", + "*/site-packages/*", + "*/dependency_injector/*", + "*/venv/*", + "*/.venv/*", + "*/.nox/*", + "src/cleveragents/discovery/*", + ] + ) + session.env["SLIPCOVER_SOURCE"] = source_paths + session.env["SLIPCOVER_OMIT"] = omit_patterns + session.env["SLIPCOVER_OUTPUT_DIR"] = "build" + # Tell behave-parallel workers to run under slipcover session.env["BEHAVE_PARALLEL_COVERAGE"] = "1" - # Clean up any existing coverage data - session.run("coverage", "erase") + # Clean up any existing slipcover data + for path in Path("build").glob(".slipcover.*.json"): + path.unlink() # Run behave tests in parallel; each worker collects coverage separately behave_cmd = session.bin + "/behave-parallel" @@ -537,34 +573,99 @@ def coverage_report(session: nox.Session): session.run(*args) - # Merge per-worker coverage data files into a single build/.coverage - session.run("coverage", "combine") + data_files = sorted(Path("build").glob(".slipcover.*.json")) + if not data_files: + session.error("No slipcover data files found in build/") - # Always generate HTML and XML reports before checking threshold - session.run("coverage", "html") - session.run("coverage", "xml") + # Merge per-worker coverage data into a single JSON report + session.run( + "python", + "-m", + "slipcover", + "--merge", + *[str(path) for path in data_files], + "--json", + "--out", + "build/coverage.json", + ) + + # Generate XML report for CI + session.run( + "python", + "-m", + "slipcover", + "--merge", + "build/coverage.json", + "--xml", + "--out", + "build/coverage.xml", + ) # Generate terminal report and check threshold. - # coverage report --fail-under exits non-zero if below threshold, - # but nox intercepts the exit. We run it with success_codes to capture - # the failure and emit a clear, CI-parseable single-line summary. + # slipcover exits with code 2 if below threshold; nox intercepts the exit. + report_path = "build/coverage-report.txt" session.run( - "coverage", - "report", - "--show-missing", - f"--fail-under={COVERAGE_THRESHOLD}", + "python", + "-m", + "slipcover", + "--merge", + "build/coverage.json", + "--out", + report_path, + "--source", + source_paths, + "--omit", + omit_patterns, + "--fail-under", + str(COVERAGE_THRESHOLD), success_codes=[0, 2], silent=False, ) - # Parse the total percentage from the coverage JSON report for the summary + try: + with open(report_path) as report_file: + print(report_file.read(), end="") + except FileNotFoundError: + session.log(f"Slipcover report not found at {report_path}") + + # Parse the total percentage from the slipcover JSON report for the summary total_pct: float = 0.0 try: - session.run("coverage", "json", "-o", "build/coverage.json", silent=True) with open("build/coverage.json") as f: - total_pct = json.load(f)["totals"]["percent_covered"] - except (FileNotFoundError, KeyError, json.JSONDecodeError): - # Fall back: if json report failed, re-read xml + payload = json.load(f) + + summary = payload.get("summary") or payload.get("totals") or {} + percent_value = None + for key in ( + "percent_covered", + "percent_covered_display", + "line_coverage_percent", + ): + percent_value = summary.get(key) + if percent_value is not None: + break + + if percent_value is None: + covered = ( + summary.get("lines_covered") + or summary.get("covered_lines") + or summary.get("covered") + ) + total = ( + summary.get("lines") + or summary.get("total_lines") + or summary.get("num_lines") + or summary.get("total") + ) + if covered is not None and total: + percent_value = (float(covered) / float(total)) * 100.0 + + if isinstance(percent_value, str): + percent_value = percent_value.strip().rstrip("%") + + if percent_value is not None: + total_pct = float(percent_value) + except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError): total_pct = 0.0 rounded_pct = round(total_pct, 1) diff --git a/pyproject.toml b/pyproject.toml index a494a221b..d276475a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dev = [ ] tests = [ "behave==1.3.3", - "coverage>=7.11.0", + "slipcover>=1.0.17", "asv>=0.6.5", "robotframework>=7.3.2", "robotframework-pabot>=4.0.0", -- 2.52.0 From 4eed203281c4d2c29a9f7a97efc9462ead522e30 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Sun, 1 Mar 2026 00:44:54 +0000 Subject: [PATCH 2/2] fix(ci): align coverage JSON key lookups with slipcover output format Slipcover writes coverage data under 'summary' not 'totals' (coverage.py format). The noxfile already had robust fallback logic but two CI workflows hardcoded the old key: - ci.yml 'Surface coverage summary' step: KeyError: 'totals' - nightly-quality.yml: silently reported 0% coverage Also use --fail-under=97 (equals form) in noxfile so the Robot integration test Coverage Threshold Is 97 In Noxfile passes its string search for '--fail-under='. --- .forgejo/workflows/ci.yml | 12 +++++++----- .forgejo/workflows/nightly-quality.yml | 3 ++- noxfile.py | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index b3f37eb65..4ed41a549 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -183,7 +183,9 @@ jobs: python3 -c " import json, sys with open('build/coverage.json') as f: - pct = round(json.load(f)['totals']['percent_covered'], 1) + data = json.load(f) + summary = data.get('summary') or data.get('totals') or {} + pct = round(summary.get('percent_covered', 0), 1) threshold = 97 if pct >= threshold: print(f'COVERAGE OK: {pct}% (threshold: {threshold}%)') @@ -210,7 +212,7 @@ jobs: benchmark-regression: if: forgejo.event_name == 'pull_request' runs-on: docker - container: + container: image: python:3.13-slim needs: [lint, typecheck] steps: @@ -222,7 +224,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - + - name: Compute base commit id: hash run: | @@ -245,13 +247,13 @@ jobs: python -m pip install awscli mkdir -p build/asv/results aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results" build/asv/results --delete || true - + - name: Run asv continuous via nox env: ASV_BASE_SHA: ${{ steps.hash.outputs.ASV_BASE_SHA }} run: | nox -s benchmark_regression - + - name: Archive the results run: | tar cf /tmp/asv-results.tar build/asv/results build/asv/html diff --git a/.forgejo/workflows/nightly-quality.yml b/.forgejo/workflows/nightly-quality.yml index da6bc8467..42ec1a040 100644 --- a/.forgejo/workflows/nightly-quality.yml +++ b/.forgejo/workflows/nightly-quality.yml @@ -95,7 +95,8 @@ jobs: cov_path = Path('build/reports/coverage.json') if cov_path.exists(): cov_data = json.loads(cov_path.read_text()) - report['gates']['coverage'] = cov_data.get('totals', {}).get('percent_covered', 0) + summary = cov_data.get('summary') or cov_data.get('totals') or {} + report['gates']['coverage'] = summary.get('percent_covered', 0) # Complexity cx_path = Path('build/reports/complexity.json') diff --git a/noxfile.py b/noxfile.py index 35557f2a2..578de4ae8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -616,8 +616,7 @@ def coverage_report(session: nox.Session): source_paths, "--omit", omit_patterns, - "--fail-under", - str(COVERAGE_THRESHOLD), + f"--fail-under={COVERAGE_THRESHOLD}", success_codes=[0, 2], silent=False, ) -- 2.52.0