cleveragents-core/tools/_implementer_gate_preflight.py

"""Heavy flaky-test pre-flight for the implementer dispatcher.

Runs ``local_ci_gate.sh --fast`` against the pre-cloned worktree
TWICE and classifies failures as persistent (failed both runs)
vs. flaky (passed the second run). The persistent-failure list is
then surfaced in the worker prompt via
:mod:`_diff_aware_gate.render_prompt_stanza`, split by
related-to-diff vs unrelated-to-diff buckets.

Motivation (plan §"#1 Heavy", 2026-05-13): in production on
2026-05-12, gpt-5-mini at Tier 0 ran ``--fast`` once on PR #30,
observed six CLI-feature scenarios failing in unit_tests, decided
they were "probably flaky or environmental" but unrelated to the
LangGraph fix in the diff — and then **gave up**. Kimi (Tier 2)
ran the SAME ``--fast`` seven hours later and got all four gates
PASS. The dispatcher running ``--fast`` twice up front would have
caught the flakiness deterministically; gpt-5-mini would have
either seen empty failures (flakes self-cleared on retry) OR seen
a clean "unrelated to your diff" classification and known not to
bail.

Gating:
- Always-off by default (no behaviour change).
- Activated via ``IMPLEMENTER_DISPATCHER_GATE_PREFLIGHT=1``.
- Independent of the in-cycle escalation flag; this is a strict
  upgrade to the worker's pre-fetched context that has no
  byte-equivalence concerns of its own (it ADDS a prompt stanza
  rather than modifying existing ones).

Cost:
- Two ``local_ci_gate.sh --fast`` invocations per dispatched PR
  cycle. First-cold-cache run is ~5 min (per the R1 entry in
  CHANGELOG); second is warm-cache, typically <1 min. Net
  pre-flight overhead: ~6 min per cycle. Buys back significantly
  more (Tier 0 attempts no longer bail on flaky unrelated tests).

The module has NO I/O of its own beyond ``subprocess.run`` calls
on the operator's local checkout — no Forgejo calls, no model
calls. Easy to unit-test by stubbing out the runner via the
``runner`` parameter on :func:`run_preflight`.
"""
from __future__ import annotations

import logging
import os
import subprocess
import sys
import time
from pathlib import Path
from typing import Callable

_TOOLS_DIR = str(Path(__file__).resolve().parent)
if _TOOLS_DIR not in sys.path:
    sys.path.insert(0, _TOOLS_DIR)
from _loader import (  # noqa: E402  type: ignore[import-not-found]
    load_sibling as _load_sibling,
)

_diff_aware_gate = _load_sibling("_diff_aware_gate", "_diff_aware_gate.py")

_logger = logging.getLogger("implementer_gate_preflight")


# Env var controlling whether the pre-flight runs at all. Default
# OFF so the dispatcher's pre-feature behaviour is byte-equivalent.
# Set ``IMPLEMENTER_DISPATCHER_GATE_PREFLIGHT=1`` to activate.
#
# **Independent of ``IMPLEMENTER_ESCALATION_ENABLED``.** The
# preflight runs whenever this flag is on, regardless of escalation
# state — the result lands in the PR-context sentinel for the
# worker to read on its first turn, which is useful even outside
# the escalation loop. Operational consequence: enabling preflight
# alone (without escalation) adds 2× ``local_ci_gate.sh --fast``
# wallclock (~6 min cold-cache, ~1 min warm) to every dispatched
# cycle. Enable both together unless you have a specific reason to
# decouple them.
#
# **Heartbeat / watchdog implication:** the preflight runs inside
# the dispatcher's ``prompt_factory`` before
# :func:`_dispatch_runtime._refresh_heartbeat` is established for
# the cycle. The cold-cache ~6-min window is silent from the
# systemd watchdog's perspective and stacks with the existing
# prefetch + preclone pre-heartbeat wallclock. Operators enabling
# this flag should ensure the dispatcher's systemd watchdog
# interval comfortably exceeds the worst-case pre-heartbeat
# budget — bumping ``WatchdogSec`` to 15-20 min when this flag is
# on is the conservative move.
PREFLIGHT_ENABLED_ENV_VAR = "IMPLEMENTER_DISPATCHER_GATE_PREFLIGHT"

# Per-invocation timeout for ``local_ci_gate.sh --fast``. 20 minutes
# matches the R1 ``timeout: 1200000`` recommendation baked into
# the ``quality-gates`` skill — the first cold-cache run can
# legitimately take ~5 min and we want to avoid spurious aborts.
_DEFAULT_GATE_TIMEOUT_SECONDS = 1200

# Wrapper-relative path to the gate script (matches what the worker
# would also use). ``local_ci_gate.sh`` is in the repo's ``tools/``
# dir and the dispatcher's CWD is the repo root, but to keep this
# robust against different CWDs we resolve it at call time.
_GATE_SCRIPT = "tools/local_ci_gate.sh"


def is_preflight_enabled() -> bool:
    """Return ``True`` when the dispatcher should run the gate
    pre-flight. Off-by-default."""
    raw = os.environ.get(PREFLIGHT_ENABLED_ENV_VAR, "").strip().lower()
    return raw in {"1", "true", "yes", "on"}


_TRUTHY = frozenset({"1", "true", "yes", "on"})

# Magic returncode sentinel for the timeout case. Distinct from
# the generic non-zero codes the gate script itself emits on test
# failure (it uses 1 / 2) AND from the negative signal-handling
# codes Python's ``subprocess`` returns when the child dies from a
# signal (e.g. ``-15`` for SIGTERM, ``-9`` for SIGKILL — values in
# roughly ``-1..-64``). ``-9999`` is well outside that range so a
# real subprocess can never produce it, eliminating the collision
# risk of the earlier ``-1`` sentinel. The orchestrator
# (:func:`run_preflight`) checks ``rc == _TIMEOUT_RETURNCODE`` to
# surface ``preflight_timeout=True`` in the classification rather
# than treating timeout-with-no-parsed-failures as "all green".
_TIMEOUT_RETURNCODE = -9999


def _run_gate_once(
    worktree: Path,
    runner: Callable[[list[str], dict[str, str], int], subprocess.CompletedProcess] | None = None,
    *,
    extra_env: dict[str, str] | None = None,
) -> tuple[int, str]:
    """Run ``local_ci_gate.sh --fast`` once against ``worktree``.

    Returns ``(returncode, combined_stdout_stderr)``. Caller decides
    what to do with the result — this function does not raise on
    non-zero exit codes (the gate exits 1 on test failure, which is
    the case we WANT to capture and parse).

    On :class:`subprocess.TimeoutExpired` the function returns
    ``(_TIMEOUT_RETURNCODE, "<TIMEOUT after Xs>")``. The marker line
    is what :func:`run_preflight` keys off to set
    ``preflight_timeout=True`` in the classification — without it,
    the classifier saw no parseable failures in the timeout text
    and (incorrectly) reported "no failures persisted", giving the
    worker a silent green light on a wedged gate run.

    The ``runner`` indirection exists so unit tests can substitute
    a function that returns scripted output without invoking the
    real script.
    """
    cmd = [_GATE_SCRIPT, "--fast", "--repo-root", str(worktree)]
    env = {**os.environ, **(extra_env or {})}

    if runner is not None:
        result = runner(cmd, env, _DEFAULT_GATE_TIMEOUT_SECONDS)
        # subprocess.CompletedProcess fields: returncode + stdout/stderr
        out = (result.stdout or "") + (result.stderr or "")
        return result.returncode, out

    started = time.monotonic()
    try:
        result = subprocess.run(
            cmd,
            env=env,
            capture_output=True,
            text=True,
            timeout=_DEFAULT_GATE_TIMEOUT_SECONDS,
            check=False,
        )
    except subprocess.TimeoutExpired:
        elapsed = time.monotonic() - started
        _logger.warning(
            "gate pre-flight timed out after %.1fs running %s",
            elapsed, " ".join(cmd),
        )
        return _TIMEOUT_RETURNCODE, f"<TIMEOUT after {elapsed:.1f}s>"
    return result.returncode, (result.stdout or "") + (result.stderr or "")


def _failed_scenario_keys(gate_output: str) -> set[tuple[str, str]]:
    """Helper: extract the ``(path, line)`` set of failing scenarios
    so two runs can be intersected. Mirrors
    :func:`_diff_aware_gate.parse_failing_scenarios` but returns a
    set for membership testing."""
    failures = _diff_aware_gate.parse_failing_scenarios(gate_output)
    return {(f["path"], f["line"]) for f in failures}


def run_preflight(
    worktree: Path,
    changed_files: list[str],
    *,
    runner: Callable[[list[str], dict[str, str], int], subprocess.CompletedProcess] | None = None,
) -> dict[str, object]:
    """Run the gate twice, build a persistent-failure classification.

    Returns the dict shape :func:`_diff_aware_gate.classify_failures`
    produces, but with an additional ``"runs"`` key carrying per-run
    summaries:

    .. code-block:: python

        {
            "gate_statuses": {...},
            "failures_total": int,
            "failures_related_to_diff": int,
            "failures_unrelated_to_diff": int,
            "related": [...],
            "unrelated": [...],
            "runs": [
                {"run": 1, "returncode": int, "failures": int},
                {"run": 2, "returncode": int, "failures": int},
            ],
            "preflight_enabled": True,
        }

    Persistence rule: a scenario is "persistent" if it appears in
    the failure list of BOTH runs (same ``(path, line)`` key).
    Scenarios that fail run 1 but pass run 2 are considered flaky
    and are NOT surfaced to the agent.

    When run 1 already passes (returncode 0, no failures), run 2 is
    skipped — the gates passed and there's nothing to re-verify.
    """
    if not is_preflight_enabled():
        return {
            "preflight_enabled": False,
            "failures_total": 0,
            "failures_related_to_diff": 0,
            "failures_unrelated_to_diff": 0,
            "related": [],
            "unrelated": [],
            "gate_statuses": {},
            "runs": [],
        }

    rc1, out1 = _run_gate_once(worktree, runner=runner)
    failures1 = _failed_scenario_keys(out1)
    run_summaries: list[dict[str, object]] = [
        {"run": 1, "returncode": rc1, "failures": len(failures1)}
    ]
    timed_out_run1 = rc1 == _TIMEOUT_RETURNCODE

    # ─── Timeout short-circuit ─────────────────────────────────
    # When the first run timed out we cannot deterministically
    # classify failures (the output is a marker line, not gate
    # text). Returning the empty classification would render as
    # "no persistent failures" to the worker — a silent green
    # light. Instead, surface ``preflight_timeout=True`` so the
    # renderer warns the worker that classification is unreliable.
    # Counts/lists are explicitly zeroed so the renderer cannot
    # show a contradictory "Persistent failures: N" line next to
    # the timeout warning — the classification IS suppressed in
    # this branch, and the payload should say so unambiguously.
    if timed_out_run1:
        return {
            "runs": run_summaries,
            "preflight_enabled": True,
            "preflight_timeout": True,
            "gate_statuses": {},
            "failures_total": 0,
            "failures_related_to_diff": 0,
            "failures_unrelated_to_diff": 0,
            "related": [],
            "unrelated": [],
        }

    if rc1 == 0 and not failures1:
        # All gates passed on the first attempt — no need to re-run.
        classification = _diff_aware_gate.classify_failures(
            out1, changed_files
        )
        classification.update({"runs": run_summaries, "preflight_enabled": True})
        return classification

    rc2, out2 = _run_gate_once(worktree, runner=runner)
    failures2 = _failed_scenario_keys(out2)
    run_summaries.append(
        {"run": 2, "returncode": rc2, "failures": len(failures2)}
    )
    timed_out_run2 = rc2 == _TIMEOUT_RETURNCODE

    if timed_out_run2:
        # Run 1 produced parseable output but run 2 timed out — we
        # can't intersect to find persistent failures, so we treat
        # the whole pre-flight as inconclusive (same reasoning as
        # the run-1 timeout branch). Surface run-1's gate_statuses
        # roll-up (it IS real data) but suppress the failure
        # counts: without a second run we cannot distinguish flakes
        # from persistent failures, and emitting the run-1 list as
        # if it were persistent would mislead the worker. The
        # ``preflight_timeout`` flag + zeroed counts give the
        # renderer one unambiguous story.
        run1_classification = _diff_aware_gate.classify_failures(
            out1, changed_files
        )
        return {
            "runs": run_summaries,
            "preflight_enabled": True,
            "preflight_timeout": True,
            "gate_statuses": run1_classification.get("gate_statuses") or {},
            "failures_total": 0,
            "failures_related_to_diff": 0,
            "failures_unrelated_to_diff": 0,
            "related": [],
            "unrelated": [],
        }

    # Persistent = present in BOTH runs. Track flake count
    # (failed run 1, passed run 2) so the worker / telemetry can
    # see how much the dispatcher filtered out.
    persistent = failures1 & failures2
    flakes_filtered = len(failures1 - failures2)
    synthetic_lines = [f"  {p}:{n}" for (p, n) in sorted(persistent)]
    # Preserve the gate-status header from the second run's output
    # so the classifier still emits a gate_statuses summary. The
    # gate emits ``## [unit_tests] PASS (12s)`` lines (see
    # _GATE_STATUS_RE in _diff_aware_gate.py); we filter out
    # non-status lines while keeping ``## [gate]`` rows.
    statuses_text = "\n".join(
        line for line in (out2 or "").splitlines()
        if line.lstrip().startswith("## [")
    )
    synthetic_output = (
        statuses_text
        + "\n\nFailing scenarios (persistent across two runs):\n"
        + "\n".join(synthetic_lines)
    )
    classification = _diff_aware_gate.classify_failures(
        synthetic_output, changed_files
    )
    classification.update({
        "runs": run_summaries,
        "preflight_enabled": True,
        "flakes_filtered": flakes_filtered,
    })
    return classification