cleveragents-core/tools/_implementer_gate_preflight.py

"""Heavy flaky-test pre-flight for the implementer dispatcher.

Runs ``local_ci_gate.sh --fast`` against the pre-cloned worktree
TWICE and classifies failures as persistent (failed both runs)
vs. flaky (passed the second run). The persistent-failure list is
then surfaced in the worker prompt via
:mod:`_diff_aware_gate.render_prompt_stanza`, split by
related-to-diff vs unrelated-to-diff buckets.

Motivation (plan §"#1 Heavy", 2026-05-13): in production on
2026-05-12, gpt-5-mini at Tier 0 ran ``--fast`` once on PR #30,
observed six CLI-feature scenarios failing in unit_tests, decided
they were "probably flaky or environmental" but unrelated to the
LangGraph fix in the diff — and then **gave up**. Kimi (Tier 2)
ran the SAME ``--fast`` seven hours later and got all four gates
PASS. The dispatcher running ``--fast`` twice up front would have
caught the flakiness deterministically; gpt-5-mini would have
either seen empty failures (flakes self-cleared on retry) OR seen
a clean "unrelated to your diff" classification and known not to
bail.

Gating:
- Always-off by default (no behaviour change).
- Activated via ``IMPLEMENTER_DISPATCHER_GATE_PREFLIGHT=1``.
- Independent of the in-cycle escalation flag; this is a strict
  upgrade to the worker's pre-fetched context that has no
  byte-equivalence concerns of its own (it ADDS a prompt stanza
  rather than modifying existing ones).

Cost:
- Two ``local_ci_gate.sh --fast`` invocations per dispatched PR
  cycle. First-cold-cache run is ~5 min (per the R1 entry in
  CHANGELOG); second is warm-cache, typically <1 min. Net
  pre-flight overhead: ~6 min per cycle. Buys back significantly
  more (Tier 0 attempts no longer bail on flaky unrelated tests).

The module has NO I/O of its own beyond ``subprocess.run`` calls
on the operator's local checkout — no Forgejo calls, no model
calls. Easy to unit-test by stubbing out the runner via the
``runner`` parameter on :func:`run_preflight`.
"""
from __future__ import annotations

import logging
import os
import subprocess
import sys
import time
from pathlib import Path
from typing import Callable

_TOOLS_DIR = str(Path(__file__).resolve().parent)
if _TOOLS_DIR not in sys.path:
    sys.path.insert(0, _TOOLS_DIR)
from _loader import (  # noqa: E402  type: ignore[import-not-found]
    load_sibling as _load_sibling,
)

_diff_aware_gate = _load_sibling("_diff_aware_gate", "_diff_aware_gate.py")

_logger = logging.getLogger("implementer_gate_preflight")


# Env var controlling whether the pre-flight runs at all. Default
# OFF so the dispatcher's pre-feature behaviour is byte-equivalent.
# Set ``IMPLEMENTER_DISPATCHER_GATE_PREFLIGHT=1`` to activate.
PREFLIGHT_ENABLED_ENV_VAR = "IMPLEMENTER_DISPATCHER_GATE_PREFLIGHT"

# Per-invocation timeout for ``local_ci_gate.sh --fast``. 20 minutes
# matches the R1 ``timeout: 1200000`` recommendation baked into
# the ``quality-gates`` skill — the first cold-cache run can
# legitimately take ~5 min and we want to avoid spurious aborts.
_DEFAULT_GATE_TIMEOUT_SECONDS = 1200

# Wrapper-relative path to the gate script (matches what the worker
# would also use). ``local_ci_gate.sh`` is in the repo's ``tools/``
# dir and the dispatcher's CWD is the repo root, but to keep this
# robust against different CWDs we resolve it at call time.
_GATE_SCRIPT = "tools/local_ci_gate.sh"


def is_preflight_enabled() -> bool:
    """Return ``True`` when the dispatcher should run the gate
    pre-flight. Off-by-default."""
    raw = os.environ.get(PREFLIGHT_ENABLED_ENV_VAR, "").strip().lower()
    return raw in {"1", "true", "yes", "on"}


_TRUTHY = frozenset({"1", "true", "yes", "on"})


def _run_gate_once(
    worktree: Path,
    runner: Callable[[list[str], dict[str, str], int], subprocess.CompletedProcess] | None = None,
    *,
    extra_env: dict[str, str] | None = None,
) -> tuple[int, str]:
    """Run ``local_ci_gate.sh --fast`` once against ``worktree``.

    Returns ``(returncode, combined_stdout_stderr)``. Caller decides
    what to do with the result — this function does not raise on
    non-zero exit codes (the gate exits 1 on test failure, which is
    the case we WANT to capture and parse).

    The ``runner`` indirection exists so unit tests can substitute
    a function that returns scripted output without invoking the
    real script.
    """
    cmd = [_GATE_SCRIPT, "--fast", "--repo-root", str(worktree)]
    env = {**os.environ, **(extra_env or {})}

    if runner is not None:
        result = runner(cmd, env, _DEFAULT_GATE_TIMEOUT_SECONDS)
        # subprocess.CompletedProcess fields: returncode + stdout/stderr
        out = (result.stdout or "") + (result.stderr or "")
        return result.returncode, out

    started = time.monotonic()
    try:
        result = subprocess.run(
            cmd,
            env=env,
            capture_output=True,
            text=True,
            timeout=_DEFAULT_GATE_TIMEOUT_SECONDS,
            check=False,
        )
    except subprocess.TimeoutExpired:
        elapsed = time.monotonic() - started
        _logger.warning(
            "gate pre-flight timed out after %.1fs running %s",
            elapsed, " ".join(cmd),
        )
        return -1, f"<TIMEOUT after {elapsed:.1f}s>"
    return result.returncode, (result.stdout or "") + (result.stderr or "")


def _failed_scenario_keys(gate_output: str) -> set[tuple[str, str]]:
    """Helper: extract the ``(path, line)`` set of failing scenarios
    so two runs can be intersected. Mirrors
    :func:`_diff_aware_gate.parse_failing_scenarios` but returns a
    set for membership testing."""
    failures = _diff_aware_gate.parse_failing_scenarios(gate_output)
    return {(f["path"], f["line"]) for f in failures}


def run_preflight(
    worktree: Path,
    changed_files: list[str],
    *,
    runner: Callable[[list[str], dict[str, str], int], subprocess.CompletedProcess] | None = None,
) -> dict[str, object]:
    """Run the gate twice, build a persistent-failure classification.

    Returns the dict shape :func:`_diff_aware_gate.classify_failures`
    produces, but with an additional ``"runs"`` key carrying per-run
    summaries:

    .. code-block:: python

        {
            "gate_statuses": {...},
            "failures_total": int,
            "failures_related_to_diff": int,
            "failures_unrelated_to_diff": int,
            "related": [...],
            "unrelated": [...],
            "runs": [
                {"run": 1, "returncode": int, "failures": int},
                {"run": 2, "returncode": int, "failures": int},
            ],
            "preflight_enabled": True,
        }

    Persistence rule: a scenario is "persistent" if it appears in
    the failure list of BOTH runs (same ``(path, line)`` key).
    Scenarios that fail run 1 but pass run 2 are considered flaky
    and are NOT surfaced to the agent.

    When run 1 already passes (returncode 0, no failures), run 2 is
    skipped — the gates passed and there's nothing to re-verify.
    """
    if not is_preflight_enabled():
        return {
            "preflight_enabled": False,
            "failures_total": 0,
            "failures_related_to_diff": 0,
            "failures_unrelated_to_diff": 0,
            "related": [],
            "unrelated": [],
            "gate_statuses": {},
            "runs": [],
        }

    rc1, out1 = _run_gate_once(worktree, runner=runner)
    failures1 = _failed_scenario_keys(out1)
    run_summaries: list[dict[str, object]] = [
        {"run": 1, "returncode": rc1, "failures": len(failures1)}
    ]

    if rc1 == 0 and not failures1:
        # All gates passed on the first attempt — no need to re-run.
        classification = _diff_aware_gate.classify_failures(
            out1, changed_files
        )
        classification.update({"runs": run_summaries, "preflight_enabled": True})
        return classification

    rc2, out2 = _run_gate_once(worktree, runner=runner)
    failures2 = _failed_scenario_keys(out2)
    run_summaries.append(
        {"run": 2, "returncode": rc2, "failures": len(failures2)}
    )

    # Persistent = present in BOTH runs. Synthesize a synthetic
    # "gate output" containing only the persistent failures so the
    # classifier doesn't accidentally see flaky ones too. The
    # synthetic output uses the same shape parse_failing_scenarios
    # consumes (``features/path.feature:N``).
    persistent = failures1 & failures2
    synthetic_lines = [f"  {p}:{n}" for (p, n) in sorted(persistent)]
    # Preserve the gate-status header from the second run's output
    # so the classifier still emits a gate_statuses summary.
    statuses_text = "\n".join(
        line for line in (out2 or "").splitlines()
        if any(g in line.lower() for g in (
            "lint:", "typecheck:", "unit_tests:",
            "integration_tests:", "e2e_tests:", "coverage:",
        ))
    )
    synthetic_output = (
        statuses_text
        + "\n\nFailing scenarios (persistent across two runs):\n"
        + "\n".join(synthetic_lines)
    )
    classification = _diff_aware_gate.classify_failures(
        synthetic_output, changed_files
    )
    classification.update(
        {"runs": run_summaries, "preflight_enabled": True}
    )
    return classification