cleveragents-core/tools/_diff_aware_gate.py

"""Diff-aware classifier for ``local_ci_gate.sh`` quality-gate
output.

Cross-references the failing BDD scenarios reported by the gate
runner against the PR's changed-files list, splitting failures
into two buckets:

- ``related_failures`` — failures in feature files whose paths
  match (or are colocated with) files the PR touches. Almost
  always real regressions caused by the PR.
- ``unrelated_failures`` — failures in feature files the PR did
  not touch. Strongly correlated with environmental flakiness or
  CI-side intermittency — these are the failures Tier 0 historically
  bailed out on (e.g., PR #30 Run 1, 2026-05-12, where six CLI
  scenarios failed unrelated to the LangGraph fix and gpt-5-mini
  gave up rather than re-running them).

Used by the dispatcher's pre-flight pre-fetch path (Heavy flaky
pre-flight, plan §"#1 Heavy") to verify a failure is persistent
before surfacing it to the worker. Also useful as a standalone
helper the implementer worker can call mid-session.

This module is pure-Python and has no I/O of its own — callers
provide the gate output text and the list of changed files. Easy
to unit-test.
"""

from __future__ import annotations

import re
from typing import Iterable


# Pattern for failing-BDD-scenario lines in ``behave`` output:
#
#     features/path/to/file.feature:42  Scenario: did the thing
#
# behave reports failures as "Failing scenarios:" header followed by
# indented ``path:line`` lines. The trailing scenario name is
# optional and may contain anything; we only capture the file path
# and line number which are the load-bearing identifiers.
_BEHAVE_FAILURE_RE = re.compile(
    r"^\s*(?P<path>features/[^\s:]+\.feature):(?P<line>\d+)",
    re.MULTILINE,
)

# Gate status lines from ``tools/local_ci_gate.sh``. The script
# prints one of two shapes per gate (see ``local_ci_gate.sh:323+``):
#
#     ## [unit_tests] start
#     ## [unit_tests] PASS (12s)
#     ## [unit_tests] FAIL (47s)
#
# We match the second-line shape because that's the one with the
# verdict. The leading ``##`` is the script's section marker; the
# ``(Ns)`` elapsed-time tail is informational and ignored here.
#
# Historically the regex expected ``unit_tests: PASS`` colon-style
# output that no local script actually produces — every gate parsed
# as missing and the worker saw an empty roll-up. The fix is to
# match the real format. We keep ``SKIP`` in the alternation even
# though the current script doesn't emit it; future gate additions
# may.
_GATE_STATUS_RE = re.compile(
    r"^##\s*\[(?P<gate>lint|typecheck|unit_tests|integration_tests|"
    r"e2e_tests|coverage)\]\s+(?P<status>PASS|FAIL|SKIP)\b",
    re.MULTILINE | re.IGNORECASE,
)


def parse_failing_scenarios(gate_output: str) -> list[dict[str, str]]:
    """Extract failing scenario paths + line numbers from gate output.

    Returns a list of ``{"path": str, "line": str}`` dicts in the
    order they appear in the output. Duplicates are de-duplicated
    by ``(path, line)`` because a behave summary tends to list each
    failure both inline and in the "Failing scenarios:" footer.
    """
    seen: set[tuple[str, str]] = set()
    out: list[dict[str, str]] = []
    for m in _BEHAVE_FAILURE_RE.finditer(gate_output or ""):
        key = (m.group("path"), m.group("line"))
        if key in seen:
            continue
        seen.add(key)
        out.append({"path": m.group("path"), "line": m.group("line")})
    return out


def parse_gate_statuses(gate_output: str) -> dict[str, str]:
    """Map gate name → status (PASS/FAIL/SKIP) parsed from output.

    Coarse-grained — used by the dispatcher to surface a high-level
    gate roll-up in the worker prompt without exposing the full raw
    output. The agent reads this dict and can decide whether to
    drill into specific failures.
    """
    out: dict[str, str] = {}
    for m in _GATE_STATUS_RE.finditer(gate_output or ""):
        out[m.group("gate").lower()] = m.group("status").upper()
    return out


def _normalise_path(p: str) -> str:
    """Strip leading ``./`` and trailing whitespace from a path."""
    return (p or "").strip().removeprefix("./")


def _feature_likely_related(feature_path: str, changed_files: Iterable[str]) -> bool:
    """Heuristic: a feature is likely related to a PR's changes if
    EITHER the feature file itself was changed OR the feature's
    name-stem appears in a changed source-file path.

    Examples:
    - feature ``features/auto_debug_cli_coverage.feature`` is related
      to a PR touching ``src/cleveragents/auto_debug/cli.py`` (stem
      ``auto_debug`` appears in both).
    - feature ``features/plan_cli_commands_r2.feature`` is unrelated
      to a PR touching ``langgraph/graph.py`` (no overlap).

    This is intentionally simple — false positives (calling
    something related when it isn't) are cheap (we just don't
    auto-suppress that failure). False negatives (calling something
    unrelated when it IS) are also cheap because the worker still
    sees the failure list and can decide.
    """
    feature_path_norm = _normalise_path(feature_path)
    changed_norm = {_normalise_path(c) for c in changed_files}

    # Direct hit — feature file itself changed.
    if feature_path_norm in changed_norm:
        return True

    # Extract the stem of the feature name (strip extension + path).
    # ``features/auto_debug_cli_coverage.feature`` → ``auto_debug_cli_coverage``.
    stem = feature_path_norm.split("/")[-1]
    if stem.endswith(".feature"):
        stem = stem[: -len(".feature")]
    if not stem:
        return False

    # Split the stem into prefix tokens — the most-specific token
    # first. We compare just the first 2-3 tokens because feature
    # filenames are often verbose ("plan_cli_commands_r2_boost") and
    # matching the WHOLE stem against a path would miss legitimate
    # matches like ``plan_cli/commands.py``.
    #
    # Split on BOTH ``_`` and ``-`` so source files that use the
    # hyphenated convention (``plan-cli-commands.py``) match a
    # feature file using the underscored convention
    # (``plan_cli_commands.feature``). Without the hyphen split,
    # a feature ``auto-debug-cli`` (rare but legal in features/)
    # would tokenise as a single string and miss every legitimate
    # source-path hit.
    tokens = re.split(r"[_\-]", stem)
    # Take the first 2 tokens as the "salient prefix" — the bit
    # that's most likely to map to a source-tree path component.
    # Each two-token prefix is generated in BOTH the underscored
    # and hyphenated forms so a feature ``plan_cli_commands.feature``
    # matches a source path ``src/plan-cli/commands.py``.
    prefix_candidates: list[str] = []
    if tokens:
        prefix_candidates.append(tokens[0])
    if len(tokens) >= 2:
        prefix_candidates.append("_".join(tokens[:2]))
        prefix_candidates.append("-".join(tokens[:2]))

    for prefix in prefix_candidates:
        if not prefix:
            continue
        for changed in changed_norm:
            # Match against PATH SEGMENTS only — do NOT further
            # split each segment on ``[_\-]``. Earlier iterations
            # exploded each segment into its sub-tokens, which made
            # a diff in ``src/plan-cli/commands.py`` (segments
            # ``["src", "plan-cli", "commands", "py"]``) match
            # every feature whose stem started with ``plan`` OR
            # ``cli`` OR ``commands`` — including unrelated
            # ``cli_extensions.feature``. The fix: keep the segment
            # set literal, and rely on ``prefix_candidates`` already
            # spanning both underscored and hyphenated forms to
            # bridge the naming-convention gap.
            segments = changed.replace(".", "/").split("/")
            if prefix in segments:
                return True
    return False


def classify_failures(
    gate_output: str, changed_files: Iterable[str]
) -> dict[str, object]:
    """Top-level classifier: parse + relate.

    Returns a structured dict the dispatcher embeds in the worker
    prompt:

    .. code-block:: python

        {
            "gate_statuses": {"lint": "PASS", "unit_tests": "FAIL", ...},
            "failures_total": 6,
            "failures_related_to_diff": 0,
            "failures_unrelated_to_diff": 6,
            "related": [{"path": ..., "line": ...}, ...],
            "unrelated": [{"path": ..., "line": ...}, ...],
        }

    The agent's instruction (in implementation-worker / task-implementor
    prompts): if ``failures_related_to_diff == 0`` and the gate
    pre-flight already retried persistent failures, the run is
    PROBABLY flaky — focus on compliance gaps / fill-the-blanks
    rather than diagnosing the unrelated failures.
    """
    statuses = parse_gate_statuses(gate_output)
    failures = parse_failing_scenarios(gate_output)
    changed = list(changed_files)
    related: list[dict[str, str]] = []
    unrelated: list[dict[str, str]] = []
    for f in failures:
        if _feature_likely_related(f["path"], changed):
            related.append(f)
        else:
            unrelated.append(f)
    return {
        "gate_statuses": statuses,
        "failures_total": len(failures),
        "failures_related_to_diff": len(related),
        "failures_unrelated_to_diff": len(unrelated),
        "related": related,
        "unrelated": unrelated,
    }


def render_prompt_stanza(classification: dict[str, object]) -> str:
    """Render the classification result as a markdown stanza for the
    worker prompt. Kept here (next to the classifier) so the
    rendered shape stays lockstep with the data shape.

    Returns the empty string when there are zero failures to surface
    — the dispatcher embeds the stanza unconditionally when the
    classification ran, but the agent reads "empty stanza = all
    gates passed in pre-flight" without a special-case.
    """
    total = classification.get("failures_total", 0)
    related = classification.get("related") or []
    unrelated = classification.get("unrelated") or []
    statuses = classification.get("gate_statuses") or {}
    if not total and not statuses:
        return ""

    lines = ["## Pre-flight quality gates (verified persistent)\n"]
    if statuses:
        lines.append("Gate summary:")
        for gate, status in sorted(statuses.items()):
            lines.append(f"- `{gate}`: **{status}**")
        lines.append("")

    if total == 0:
        lines.append(
            "_No failures persisted after the pre-flight retry — "
            "you do NOT need to re-run `local_ci_gate.sh --fast` "
            "yourself unless you change files._"
        )
        return "\n".join(lines)

    lines.append(
        f"Failures total: **{total}** "
        f"(related to your diff: {len(related)}, "
        f"unrelated: {len(unrelated)})."
    )
    lines.append("")
    if related:
        lines.append("**Related to your diff** — investigate these:")
        for f in related:
            lines.append(f"- `{f['path']}:{f['line']}`")
        lines.append("")
    if unrelated:
        lines.append(
            "**Unrelated to your diff** — these were retried by "
            "the dispatcher's pre-flight and persisted, but they "
            "exercise code your PR did not touch. Likely "
            "environmental / pre-existing. Do NOT bail on the "
            "cycle for these — focus on the related failures "
            "above (if any) and on compliance gaps (next section). "
            "If you have time, you may re-run these locally to "
            "confirm reproducibility before exiting."
        )
        for f in unrelated:
            lines.append(f"- `{f['path']}:{f['line']}`")
    return "\n".join(lines)