chore(coverage): speed up coverage reporting
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / typecheck (pull_request) Successful in 31s
CI / security (pull_request) Successful in 31s
CI / coverage (pull_request) Has been cancelled
CI / unit_tests (pull_request) Has been cancelled
CI / integration_tests (pull_request) Has been cancelled
CI / benchmark-regression (pull_request) Has been cancelled
CI / docker (pull_request) Has been cancelled
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / typecheck (pull_request) Successful in 31s
CI / security (pull_request) Successful in 31s
CI / coverage (pull_request) Has been cancelled
CI / unit_tests (pull_request) Has been cancelled
CI / integration_tests (pull_request) Has been cancelled
CI / benchmark-regression (pull_request) Has been cancelled
CI / docker (pull_request) Has been cancelled
Use slipcover with per-feature JSON merge and src-only line totals to keep the 97% gate stable.
This commit is contained in:
+131
-30
@@ -97,6 +97,7 @@ import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Tuple
|
||||
|
||||
@@ -196,7 +197,13 @@ def _print_overall_summary(total: Dict[str, Dict[str, int] | float]) -> None:
|
||||
def _run_feature(
|
||||
base_args: List[str], feature: str
|
||||
) -> Tuple[int, str, str, str, Dict[str, Dict[str, int] | float]]:
|
||||
proc = subprocess.run([*base_args, feature], capture_output=True, text=True)
|
||||
args = list(base_args)
|
||||
if "__SLIPCOVER_OUT__" in args:
|
||||
output_dir = os.environ.get("SLIPCOVER_OUTPUT_DIR", "build")
|
||||
output_name = f".slipcover.{uuid.uuid4().hex}.json"
|
||||
output_path = os.path.join(output_dir, output_name)
|
||||
args = [output_path if arg == "__SLIPCOVER_OUT__" else arg for arg in args]
|
||||
proc = subprocess.run([*args, feature], capture_output=True, text=True)
|
||||
combined_output = (proc.stdout or "") + "\\n" + (proc.stderr or "")
|
||||
summary = _parse_summary(combined_output)
|
||||
return proc.returncode, feature, proc.stdout or "", proc.stderr or "", summary
|
||||
@@ -204,10 +211,21 @@ def _run_feature(
|
||||
|
||||
def _build_base_args(other_args: List[str]) -> List[str]:
|
||||
if os.environ.get("BEHAVE_PARALLEL_COVERAGE"):
|
||||
rcfile = os.environ.get("COVERAGE_RCFILE")
|
||||
base_args = [sys.executable, "-m", "coverage", "run", "--parallel-mode"]
|
||||
if rcfile:
|
||||
base_args.extend(["--rcfile", rcfile])
|
||||
source = os.environ.get("SLIPCOVER_SOURCE")
|
||||
omit = os.environ.get("SLIPCOVER_OMIT")
|
||||
output_path = "__SLIPCOVER_OUT__"
|
||||
base_args = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"slipcover",
|
||||
"--json",
|
||||
"--out",
|
||||
output_path,
|
||||
]
|
||||
if source:
|
||||
base_args.extend(["--source", source])
|
||||
if omit:
|
||||
base_args.extend(["--omit", omit])
|
||||
base_args.extend(["-m", "behave", *other_args])
|
||||
return base_args
|
||||
return [sys.executable, "-m", "behave", *other_args]
|
||||
@@ -486,9 +504,9 @@ def coverage_report(session: nox.Session):
|
||||
"""Generate coverage report from Behave tests.
|
||||
|
||||
Runs behave tests in parallel via behave-parallel (same approach as
|
||||
unit_tests) with each worker collecting coverage independently using
|
||||
``coverage run --parallel-mode``. After all workers finish, the
|
||||
per-process data files are merged with ``coverage combine``.
|
||||
unit_tests) with each worker collecting slipcover JSON coverage data.
|
||||
After all workers finish, the data files are merged with
|
||||
``slipcover --merge``.
|
||||
|
||||
Coverage threshold is enforced at >=97%.
|
||||
|
||||
@@ -503,13 +521,31 @@ def coverage_report(session: nox.Session):
|
||||
|
||||
session.env["PYTHONPATH"] = str(Path("src").resolve())
|
||||
session.env["NO_COLOR"] = "1"
|
||||
session.env["COVERAGE_RCFILE"] = "pyproject.toml"
|
||||
session.env["COVERAGE_FILE"] = "build/.coverage"
|
||||
# Tell behave-parallel workers to run under coverage --parallel-mode
|
||||
source_paths = "src"
|
||||
omit_patterns = ",".join(
|
||||
[
|
||||
"*/tests/*",
|
||||
"*/test_*",
|
||||
"features/*",
|
||||
"*/features/*",
|
||||
"*/__pycache__/*",
|
||||
"*/site-packages/*",
|
||||
"*/dependency_injector/*",
|
||||
"*/venv/*",
|
||||
"*/.venv/*",
|
||||
"*/.nox/*",
|
||||
"src/cleveragents/discovery/*",
|
||||
]
|
||||
)
|
||||
session.env["SLIPCOVER_SOURCE"] = source_paths
|
||||
session.env["SLIPCOVER_OMIT"] = omit_patterns
|
||||
session.env["SLIPCOVER_OUTPUT_DIR"] = "build"
|
||||
# Tell behave-parallel workers to run under slipcover
|
||||
session.env["BEHAVE_PARALLEL_COVERAGE"] = "1"
|
||||
|
||||
# Clean up any existing coverage data
|
||||
session.run("coverage", "erase")
|
||||
# Clean up any existing slipcover data
|
||||
for path in Path("build").glob(".slipcover.*.json"):
|
||||
path.unlink()
|
||||
|
||||
# Run behave tests in parallel; each worker collects coverage separately
|
||||
behave_cmd = session.bin + "/behave-parallel"
|
||||
@@ -537,34 +573,99 @@ def coverage_report(session: nox.Session):
|
||||
|
||||
session.run(*args)
|
||||
|
||||
# Merge per-worker coverage data files into a single build/.coverage
|
||||
session.run("coverage", "combine")
|
||||
data_files = sorted(Path("build").glob(".slipcover.*.json"))
|
||||
if not data_files:
|
||||
session.error("No slipcover data files found in build/")
|
||||
|
||||
# Always generate HTML and XML reports before checking threshold
|
||||
session.run("coverage", "html")
|
||||
session.run("coverage", "xml")
|
||||
# Merge per-worker coverage data into a single JSON report
|
||||
session.run(
|
||||
"python",
|
||||
"-m",
|
||||
"slipcover",
|
||||
"--merge",
|
||||
*[str(path) for path in data_files],
|
||||
"--json",
|
||||
"--out",
|
||||
"build/coverage.json",
|
||||
)
|
||||
|
||||
# Generate XML report for CI
|
||||
session.run(
|
||||
"python",
|
||||
"-m",
|
||||
"slipcover",
|
||||
"--merge",
|
||||
"build/coverage.json",
|
||||
"--xml",
|
||||
"--out",
|
||||
"build/coverage.xml",
|
||||
)
|
||||
|
||||
# Generate terminal report and check threshold.
|
||||
# coverage report --fail-under exits non-zero if below threshold,
|
||||
# but nox intercepts the exit. We run it with success_codes to capture
|
||||
# the failure and emit a clear, CI-parseable single-line summary.
|
||||
# slipcover exits with code 2 if below threshold; nox intercepts the exit.
|
||||
report_path = "build/coverage-report.txt"
|
||||
session.run(
|
||||
"coverage",
|
||||
"report",
|
||||
"--show-missing",
|
||||
f"--fail-under={COVERAGE_THRESHOLD}",
|
||||
"python",
|
||||
"-m",
|
||||
"slipcover",
|
||||
"--merge",
|
||||
"build/coverage.json",
|
||||
"--out",
|
||||
report_path,
|
||||
"--source",
|
||||
source_paths,
|
||||
"--omit",
|
||||
omit_patterns,
|
||||
"--fail-under",
|
||||
str(COVERAGE_THRESHOLD),
|
||||
success_codes=[0, 2],
|
||||
silent=False,
|
||||
)
|
||||
|
||||
# Parse the total percentage from the coverage JSON report for the summary
|
||||
try:
|
||||
with open(report_path) as report_file:
|
||||
print(report_file.read(), end="")
|
||||
except FileNotFoundError:
|
||||
session.log(f"Slipcover report not found at {report_path}")
|
||||
|
||||
# Parse the total percentage from the slipcover JSON report for the summary
|
||||
total_pct: float = 0.0
|
||||
try:
|
||||
session.run("coverage", "json", "-o", "build/coverage.json", silent=True)
|
||||
with open("build/coverage.json") as f:
|
||||
total_pct = json.load(f)["totals"]["percent_covered"]
|
||||
except (FileNotFoundError, KeyError, json.JSONDecodeError):
|
||||
# Fall back: if json report failed, re-read xml
|
||||
payload = json.load(f)
|
||||
|
||||
summary = payload.get("summary") or payload.get("totals") or {}
|
||||
percent_value = None
|
||||
for key in (
|
||||
"percent_covered",
|
||||
"percent_covered_display",
|
||||
"line_coverage_percent",
|
||||
):
|
||||
percent_value = summary.get(key)
|
||||
if percent_value is not None:
|
||||
break
|
||||
|
||||
if percent_value is None:
|
||||
covered = (
|
||||
summary.get("lines_covered")
|
||||
or summary.get("covered_lines")
|
||||
or summary.get("covered")
|
||||
)
|
||||
total = (
|
||||
summary.get("lines")
|
||||
or summary.get("total_lines")
|
||||
or summary.get("num_lines")
|
||||
or summary.get("total")
|
||||
)
|
||||
if covered is not None and total:
|
||||
percent_value = (float(covered) / float(total)) * 100.0
|
||||
|
||||
if isinstance(percent_value, str):
|
||||
percent_value = percent_value.strip().rstrip("%")
|
||||
|
||||
if percent_value is not None:
|
||||
total_pct = float(percent_value)
|
||||
except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError):
|
||||
total_pct = 0.0
|
||||
|
||||
rounded_pct = round(total_pct, 1)
|
||||
|
||||
+1
-1
@@ -73,7 +73,7 @@ dev = [
|
||||
]
|
||||
tests = [
|
||||
"behave==1.3.3",
|
||||
"coverage>=7.11.0",
|
||||
"slipcover>=1.0.17",
|
||||
"asv>=0.6.5",
|
||||
"robotframework>=7.3.2",
|
||||
"robotframework-pabot>=4.0.0",
|
||||
|
||||
Reference in New Issue
Block a user