4dec646e2f
Ports the WS1 fast/parallel coverage engine onto master and reconciles the coverage gate to a single source. Local full-suite run on master's suite: NOX_EXIT=0, all 12 chunks alive (no dead chunks), peak 658MB/chunk (ceiling ~2.6GB vs the old ~1.4GB single-process that OOMs/reaper-kills), 96.465% -> 96.5 (rounded) >= 96.5 floor. noxfile.py — coverage_report full-suite path now fans out K concurrent slipcover processes (K=COVERAGE_PROCESSES, default 4) over N bin-packed chunks (by scenario count), failing loud on any dead chunk (never merges survivors) and merging per-chunk JSON. Bounds per-process peak RSS, killing the single-process OOM/reaper collapse. Targeted (.feature posargs) runs keep the single-process path. COVERAGE_THRESHOLD now reads pyproject [tool.coverage.report].fail_under. Features are enumerated by direct glob (NOT by importing run_behave_parallel, whose top level imports behave/behave_parallel and is unavailable in the nox orchestrator process). pyproject.toml — adds [tool.coverage.report].fail_under = 96.5 as the single source of truth, with the evidence-gated ratchet rule (objective 97%). .forgejo/workflows/ci.yml — coverage job: carries the skip_coverage operator valve; propagates nox's exit EXPLICITLY (set -uo pipefail + PIPESTATUS + exit $rc) instead of relying on the runner's implicit bash -eo pipefail; adds timeout-minutes: 30 so a hang fails cleanly with diagnostics; deletes the dead threshold=50 "Surface coverage summary" step; fixes the phantom-97 step label. Gating remains nox's --fail-under (sourced from pyproject). coverage_threshold_config_steps.py — the fail-under feature step resolves the floor from pyproject when COVERAGE_THRESHOLD delegates to the reader. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
221 lines
9.2 KiB
Python
221 lines
9.2 KiB
Python
"""Step definitions for coverage threshold configuration feature."""
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from behave import given, then
|
|
from behave.runner import Context
|
|
|
|
|
|
@given("the pyproject toml file is loaded for coverage check")
|
|
def step_load_pyproject_for_coverage(context: Context) -> None:
|
|
"""Load and parse pyproject.toml for coverage configuration checks."""
|
|
toml_path = Path(__file__).resolve().parent.parent.parent / "pyproject.toml"
|
|
if not toml_path.exists():
|
|
raise FileNotFoundError(f"pyproject.toml not found at {toml_path}")
|
|
context.pyproject_text = toml_path.read_text(encoding="utf-8")
|
|
|
|
|
|
@given("the noxfile py is loaded for coverage check")
|
|
def step_load_noxfile_for_coverage(context: Context) -> None:
|
|
"""Load noxfile.py for coverage session checks."""
|
|
nox_path = Path(__file__).resolve().parent.parent.parent / "noxfile.py"
|
|
if not nox_path.exists():
|
|
raise FileNotFoundError(f"noxfile.py not found at {nox_path}")
|
|
context.noxfile_text = nox_path.read_text(encoding="utf-8")
|
|
|
|
|
|
@given("the ci workflow file is loaded for coverage check")
|
|
def step_load_ci_workflow_for_coverage(context: Context) -> None:
|
|
"""Load CI workflow file for coverage enforcement checks."""
|
|
ci_path = (
|
|
Path(__file__).resolve().parent.parent.parent
|
|
/ ".forgejo"
|
|
/ "workflows"
|
|
/ "ci.yml"
|
|
)
|
|
if not ci_path.exists():
|
|
raise FileNotFoundError(f"CI workflow not found at {ci_path}")
|
|
context.ci_workflow_text = ci_path.read_text(encoding="utf-8")
|
|
|
|
|
|
@given("the nightly quality workflow file is loaded for coverage check")
|
|
def step_load_nightly_workflow_for_coverage(context: Context) -> None:
|
|
"""Load nightly quality workflow file for coverage threshold checks."""
|
|
nightly_path = (
|
|
Path(__file__).resolve().parent.parent.parent
|
|
/ ".forgejo"
|
|
/ "workflows"
|
|
/ "nightly-quality.yml"
|
|
)
|
|
if not nightly_path.exists():
|
|
raise FileNotFoundError(f"Nightly workflow not found at {nightly_path}")
|
|
context.nightly_workflow_text = nightly_path.read_text(encoding="utf-8")
|
|
|
|
|
|
@then("the coverage run section should exist")
|
|
def step_coverage_run_section_exists(context: Context) -> None:
|
|
"""Assert [tool.coverage.run] section is present."""
|
|
if "[tool.coverage.run]" not in context.pyproject_text:
|
|
raise AssertionError("[tool.coverage.run] section missing from pyproject.toml")
|
|
|
|
|
|
@then('the coverage source should include "{source}"')
|
|
def step_coverage_source_includes(context: Context, source: str) -> None:
|
|
"""Assert coverage source includes the given directory."""
|
|
# Parse the source list from pyproject.toml
|
|
match = re.search(r"source\s*=\s*\[([^\]]+)\]", context.pyproject_text)
|
|
if not match:
|
|
raise AssertionError("No source list found in coverage config")
|
|
sources = match.group(1)
|
|
if f'"{source}"' not in sources:
|
|
raise AssertionError(
|
|
f'Expected "{source}" in coverage source list, got: {sources}'
|
|
)
|
|
|
|
|
|
@then("the coverage branch tracking should be enabled")
|
|
def step_coverage_branch_enabled(context: Context) -> None:
|
|
"""Assert branch coverage is enabled."""
|
|
if "branch = true" not in context.pyproject_text:
|
|
raise AssertionError("branch = true not found in coverage config")
|
|
|
|
|
|
def _pyproject_fail_under() -> float | None:
|
|
"""The canonical coverage floor: pyproject ``[tool.coverage.report]``
|
|
``fail_under`` (the single source the noxfile + worker read). None if
|
|
absent/unreadable."""
|
|
import tomllib
|
|
|
|
toml_path = Path(__file__).resolve().parent.parent.parent / "pyproject.toml"
|
|
try:
|
|
data = tomllib.loads(toml_path.read_text(encoding="utf-8"))
|
|
return float(data["tool"]["coverage"]["report"]["fail_under"])
|
|
except (OSError, KeyError, TypeError, ValueError, tomllib.TOMLDecodeError):
|
|
return None
|
|
|
|
|
|
@then("the noxfile should contain a fail-under threshold of at least {threshold}")
|
|
def step_noxfile_fail_under(context: Context, threshold: str) -> None:
|
|
"""Assert noxfile enforces a fail-under >= the given threshold.
|
|
|
|
Handles three forms, newest first:
|
|
- the noxfile delegates ``COVERAGE_THRESHOLD`` to pyproject's single
|
|
source (``COVERAGE_THRESHOLD = _read_coverage_fail_under()`` +
|
|
``f"--fail-under={COVERAGE_THRESHOLD:g}"``) — resolve the effective floor
|
|
from pyproject ``[tool.coverage.report].fail_under`` (WS5);
|
|
- a literal ``--fail-under=96.5``;
|
|
- an f-string over a ``COVERAGE_THRESHOLD`` *constant*.
|
|
"""
|
|
import ast
|
|
|
|
threshold_val = float(threshold)
|
|
|
|
# Literal --fail-under=N (the delegated f-string form has no digits).
|
|
matches = re.findall(r"--fail-under=([\d.]+)", context.noxfile_text)
|
|
if matches:
|
|
max_threshold = max(float(m) for m in matches)
|
|
if max_threshold < threshold_val:
|
|
raise AssertionError(
|
|
f"fail-under={max_threshold} is below required {threshold_val}"
|
|
)
|
|
return
|
|
|
|
# f-string over COVERAGE_THRESHOLD: resolve a literal constant, else fall
|
|
# through to the pyproject single source the constant now delegates to.
|
|
if (
|
|
"fail-under" in context.noxfile_text
|
|
and "COVERAGE_THRESHOLD" in context.noxfile_text
|
|
):
|
|
tree = ast.parse(context.noxfile_text)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Assign):
|
|
for target in node.targets:
|
|
if (
|
|
isinstance(target, ast.Name)
|
|
and target.id == "COVERAGE_THRESHOLD"
|
|
and isinstance(node.value, ast.Constant)
|
|
):
|
|
value = float(str(node.value.value))
|
|
if value < threshold_val:
|
|
raise AssertionError(
|
|
f"COVERAGE_THRESHOLD={value} is below required "
|
|
f"{threshold_val}"
|
|
)
|
|
return
|
|
# WS5: COVERAGE_THRESHOLD = _read_coverage_fail_under() → pyproject.
|
|
resolved = _pyproject_fail_under()
|
|
if resolved is not None:
|
|
if resolved < threshold_val:
|
|
raise AssertionError(
|
|
f"pyproject fail_under={resolved:g} is below required "
|
|
f"{threshold_val}"
|
|
)
|
|
return
|
|
raise AssertionError("No --fail-under found in noxfile.py")
|
|
|
|
|
|
@then("the noxfile should define a coverage_report session")
|
|
def step_noxfile_coverage_session_exists(context: Context) -> None:
|
|
"""Assert that a coverage_report session is defined in noxfile."""
|
|
if "def coverage_report(" not in context.noxfile_text:
|
|
raise AssertionError("coverage_report session not found in noxfile.py")
|
|
|
|
|
|
@then('the coverage html directory should be "{directory}"')
|
|
def step_coverage_html_directory(context: Context, directory: str) -> None:
|
|
"""Assert HTML coverage output directory."""
|
|
if f'directory = "{directory}"' not in context.pyproject_text:
|
|
raise AssertionError(
|
|
f'Expected directory = "{directory}" in [tool.coverage.html]'
|
|
)
|
|
|
|
|
|
@then('the coverage xml output should be "{output}"')
|
|
def step_coverage_xml_output(context: Context, output: str) -> None:
|
|
"""Assert XML coverage output path."""
|
|
if f'output = "{output}"' not in context.pyproject_text:
|
|
raise AssertionError(f'Expected output = "{output}" in [tool.coverage.xml]')
|
|
|
|
|
|
@then('the coverage data file should be "{data_file}"')
|
|
def step_coverage_data_file(context: Context, data_file: str) -> None:
|
|
"""Assert coverage data file path."""
|
|
if f'data_file = "{data_file}"' not in context.pyproject_text:
|
|
raise AssertionError(
|
|
f'Expected data_file = "{data_file}" in [tool.coverage.run]'
|
|
)
|
|
|
|
|
|
@then('the coverage omit patterns should include "{pattern}"')
|
|
def step_coverage_omit_includes(context: Context, pattern: str) -> None:
|
|
"""Assert coverage omit list includes the given pattern."""
|
|
if f'"{pattern}"' not in context.pyproject_text:
|
|
raise AssertionError(f'Expected "{pattern}" in coverage omit patterns')
|
|
|
|
|
|
@then("the ci workflow should reference nox coverage_report session")
|
|
def step_ci_coverage_nox_session(context: Context) -> None:
|
|
"""Assert CI workflow runs nox -s coverage_report."""
|
|
if "nox -s coverage_report" not in context.ci_workflow_text:
|
|
raise AssertionError("CI workflow does not reference 'nox -s coverage_report'")
|
|
|
|
|
|
@then("the nightly workflow should use a fail-under of at least {threshold}")
|
|
def step_nightly_fail_under(context: Context, threshold: str) -> None:
|
|
"""Assert nightly workflow uses fail-under >= threshold."""
|
|
threshold_val = float(threshold)
|
|
# Check for --fail-under=N pattern (slipcover/coverage CLI)
|
|
matches = re.findall(r"--fail-under=([\d.]+)", context.nightly_workflow_text)
|
|
# Also check for --coverage-min N pattern (quality gates script)
|
|
matches += re.findall(r"--coverage-min\s+([\d.]+)", context.nightly_workflow_text)
|
|
if not matches:
|
|
raise AssertionError(
|
|
"No --fail-under or --coverage-min found in nightly workflow"
|
|
)
|
|
max_threshold = max(float(m) for m in matches)
|
|
if max_threshold < threshold_val:
|
|
raise AssertionError(
|
|
f"Nightly fail-under={max_threshold} is below required {threshold_val}"
|
|
)
|