Files
cleveragents-core/features/steps/coverage_threshold_config_steps.py
drew 4dec646e2f feat(ci): parallel coverage engine + explicit nox-gated coverage job (Phase 2)
Ports the WS1 fast/parallel coverage engine onto master and reconciles the
coverage gate to a single source. Local full-suite run on master's suite:
NOX_EXIT=0, all 12 chunks alive (no dead chunks), peak 658MB/chunk
(ceiling ~2.6GB vs the old ~1.4GB single-process that OOMs/reaper-kills),
96.465% -> 96.5 (rounded) >= 96.5 floor.

noxfile.py — coverage_report full-suite path now fans out K concurrent
slipcover processes (K=COVERAGE_PROCESSES, default 4) over N bin-packed
chunks (by scenario count), failing loud on any dead chunk (never merges
survivors) and merging per-chunk JSON. Bounds per-process peak RSS, killing
the single-process OOM/reaper collapse. Targeted (.feature posargs) runs keep
the single-process path. COVERAGE_THRESHOLD now reads
pyproject [tool.coverage.report].fail_under. Features are enumerated by direct
glob (NOT by importing run_behave_parallel, whose top level imports
behave/behave_parallel and is unavailable in the nox orchestrator process).

pyproject.toml — adds [tool.coverage.report].fail_under = 96.5 as the single
source of truth, with the evidence-gated ratchet rule (objective 97%).

.forgejo/workflows/ci.yml — coverage job: carries the skip_coverage operator
valve; propagates nox's exit EXPLICITLY (set -uo pipefail + PIPESTATUS +
exit $rc) instead of relying on the runner's implicit bash -eo pipefail;
adds timeout-minutes: 30 so a hang fails cleanly with diagnostics; deletes
the dead threshold=50 "Surface coverage summary" step; fixes the phantom-97
step label. Gating remains nox's --fail-under (sourced from pyproject).

coverage_threshold_config_steps.py — the fail-under feature step resolves the
floor from pyproject when COVERAGE_THRESHOLD delegates to the reader.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-01 16:32:53 -04:00

221 lines
9.2 KiB
Python

"""Step definitions for coverage threshold configuration feature."""
import re
from pathlib import Path
from behave import given, then
from behave.runner import Context
@given("the pyproject toml file is loaded for coverage check")
def step_load_pyproject_for_coverage(context: Context) -> None:
"""Load and parse pyproject.toml for coverage configuration checks."""
toml_path = Path(__file__).resolve().parent.parent.parent / "pyproject.toml"
if not toml_path.exists():
raise FileNotFoundError(f"pyproject.toml not found at {toml_path}")
context.pyproject_text = toml_path.read_text(encoding="utf-8")
@given("the noxfile py is loaded for coverage check")
def step_load_noxfile_for_coverage(context: Context) -> None:
"""Load noxfile.py for coverage session checks."""
nox_path = Path(__file__).resolve().parent.parent.parent / "noxfile.py"
if not nox_path.exists():
raise FileNotFoundError(f"noxfile.py not found at {nox_path}")
context.noxfile_text = nox_path.read_text(encoding="utf-8")
@given("the ci workflow file is loaded for coverage check")
def step_load_ci_workflow_for_coverage(context: Context) -> None:
"""Load CI workflow file for coverage enforcement checks."""
ci_path = (
Path(__file__).resolve().parent.parent.parent
/ ".forgejo"
/ "workflows"
/ "ci.yml"
)
if not ci_path.exists():
raise FileNotFoundError(f"CI workflow not found at {ci_path}")
context.ci_workflow_text = ci_path.read_text(encoding="utf-8")
@given("the nightly quality workflow file is loaded for coverage check")
def step_load_nightly_workflow_for_coverage(context: Context) -> None:
"""Load nightly quality workflow file for coverage threshold checks."""
nightly_path = (
Path(__file__).resolve().parent.parent.parent
/ ".forgejo"
/ "workflows"
/ "nightly-quality.yml"
)
if not nightly_path.exists():
raise FileNotFoundError(f"Nightly workflow not found at {nightly_path}")
context.nightly_workflow_text = nightly_path.read_text(encoding="utf-8")
@then("the coverage run section should exist")
def step_coverage_run_section_exists(context: Context) -> None:
"""Assert [tool.coverage.run] section is present."""
if "[tool.coverage.run]" not in context.pyproject_text:
raise AssertionError("[tool.coverage.run] section missing from pyproject.toml")
@then('the coverage source should include "{source}"')
def step_coverage_source_includes(context: Context, source: str) -> None:
"""Assert coverage source includes the given directory."""
# Parse the source list from pyproject.toml
match = re.search(r"source\s*=\s*\[([^\]]+)\]", context.pyproject_text)
if not match:
raise AssertionError("No source list found in coverage config")
sources = match.group(1)
if f'"{source}"' not in sources:
raise AssertionError(
f'Expected "{source}" in coverage source list, got: {sources}'
)
@then("the coverage branch tracking should be enabled")
def step_coverage_branch_enabled(context: Context) -> None:
"""Assert branch coverage is enabled."""
if "branch = true" not in context.pyproject_text:
raise AssertionError("branch = true not found in coverage config")
def _pyproject_fail_under() -> float | None:
"""The canonical coverage floor: pyproject ``[tool.coverage.report]``
``fail_under`` (the single source the noxfile + worker read). None if
absent/unreadable."""
import tomllib
toml_path = Path(__file__).resolve().parent.parent.parent / "pyproject.toml"
try:
data = tomllib.loads(toml_path.read_text(encoding="utf-8"))
return float(data["tool"]["coverage"]["report"]["fail_under"])
except (OSError, KeyError, TypeError, ValueError, tomllib.TOMLDecodeError):
return None
@then("the noxfile should contain a fail-under threshold of at least {threshold}")
def step_noxfile_fail_under(context: Context, threshold: str) -> None:
"""Assert noxfile enforces a fail-under >= the given threshold.
Handles three forms, newest first:
- the noxfile delegates ``COVERAGE_THRESHOLD`` to pyproject's single
source (``COVERAGE_THRESHOLD = _read_coverage_fail_under()`` +
``f"--fail-under={COVERAGE_THRESHOLD:g}"``) — resolve the effective floor
from pyproject ``[tool.coverage.report].fail_under`` (WS5);
- a literal ``--fail-under=96.5``;
- an f-string over a ``COVERAGE_THRESHOLD`` *constant*.
"""
import ast
threshold_val = float(threshold)
# Literal --fail-under=N (the delegated f-string form has no digits).
matches = re.findall(r"--fail-under=([\d.]+)", context.noxfile_text)
if matches:
max_threshold = max(float(m) for m in matches)
if max_threshold < threshold_val:
raise AssertionError(
f"fail-under={max_threshold} is below required {threshold_val}"
)
return
# f-string over COVERAGE_THRESHOLD: resolve a literal constant, else fall
# through to the pyproject single source the constant now delegates to.
if (
"fail-under" in context.noxfile_text
and "COVERAGE_THRESHOLD" in context.noxfile_text
):
tree = ast.parse(context.noxfile_text)
for node in ast.walk(tree):
if isinstance(node, ast.Assign):
for target in node.targets:
if (
isinstance(target, ast.Name)
and target.id == "COVERAGE_THRESHOLD"
and isinstance(node.value, ast.Constant)
):
value = float(str(node.value.value))
if value < threshold_val:
raise AssertionError(
f"COVERAGE_THRESHOLD={value} is below required "
f"{threshold_val}"
)
return
# WS5: COVERAGE_THRESHOLD = _read_coverage_fail_under() → pyproject.
resolved = _pyproject_fail_under()
if resolved is not None:
if resolved < threshold_val:
raise AssertionError(
f"pyproject fail_under={resolved:g} is below required "
f"{threshold_val}"
)
return
raise AssertionError("No --fail-under found in noxfile.py")
@then("the noxfile should define a coverage_report session")
def step_noxfile_coverage_session_exists(context: Context) -> None:
"""Assert that a coverage_report session is defined in noxfile."""
if "def coverage_report(" not in context.noxfile_text:
raise AssertionError("coverage_report session not found in noxfile.py")
@then('the coverage html directory should be "{directory}"')
def step_coverage_html_directory(context: Context, directory: str) -> None:
"""Assert HTML coverage output directory."""
if f'directory = "{directory}"' not in context.pyproject_text:
raise AssertionError(
f'Expected directory = "{directory}" in [tool.coverage.html]'
)
@then('the coverage xml output should be "{output}"')
def step_coverage_xml_output(context: Context, output: str) -> None:
"""Assert XML coverage output path."""
if f'output = "{output}"' not in context.pyproject_text:
raise AssertionError(f'Expected output = "{output}" in [tool.coverage.xml]')
@then('the coverage data file should be "{data_file}"')
def step_coverage_data_file(context: Context, data_file: str) -> None:
"""Assert coverage data file path."""
if f'data_file = "{data_file}"' not in context.pyproject_text:
raise AssertionError(
f'Expected data_file = "{data_file}" in [tool.coverage.run]'
)
@then('the coverage omit patterns should include "{pattern}"')
def step_coverage_omit_includes(context: Context, pattern: str) -> None:
"""Assert coverage omit list includes the given pattern."""
if f'"{pattern}"' not in context.pyproject_text:
raise AssertionError(f'Expected "{pattern}" in coverage omit patterns')
@then("the ci workflow should reference nox coverage_report session")
def step_ci_coverage_nox_session(context: Context) -> None:
"""Assert CI workflow runs nox -s coverage_report."""
if "nox -s coverage_report" not in context.ci_workflow_text:
raise AssertionError("CI workflow does not reference 'nox -s coverage_report'")
@then("the nightly workflow should use a fail-under of at least {threshold}")
def step_nightly_fail_under(context: Context, threshold: str) -> None:
"""Assert nightly workflow uses fail-under >= threshold."""
threshold_val = float(threshold)
# Check for --fail-under=N pattern (slipcover/coverage CLI)
matches = re.findall(r"--fail-under=([\d.]+)", context.nightly_workflow_text)
# Also check for --coverage-min N pattern (quality gates script)
matches += re.findall(r"--coverage-min\s+([\d.]+)", context.nightly_workflow_text)
if not matches:
raise AssertionError(
"No --fail-under or --coverage-min found in nightly workflow"
)
max_threshold = max(float(m) for m in matches)
if max_threshold < threshold_val:
raise AssertionError(
f"Nightly fail-under={max_threshold} is below required {threshold_val}"
)