cleveragents-core/tools/controller/master/ci_status_poll.py

"""AWAITING_CI status poll — autonomous CI exit (Phase 1k++++ trial).

Closes the last operator-intervention gap for the Phase 2 trial.
Previously, workflows that transitioned to AWAITING_CI sat there
forever (only ``ci_polling_exhausted`` fired, leading to STUCK after
the 2h timeout). The RUNBOOK had a manual SQL workaround.

This module ships an autonomous tick:

1. SELECT workflows in AWAITING_CI + their latest implementer
   attempt's ``head_sha_after`` (the SHA the implementer pushed).
2. For each: call ``get_ci_status(owner, repo, head_sha)`` — wraps
   Forgejo's ``/commits/{sha}/status`` combined-status endpoint.
3. Decide the next event based on the combined ``state`` field:
   - ``success`` → pre-review mergeable gate (one Forgejo PR-detail
     call, no LLM):
       - PR still merges cleanly → ``ci_green`` → REVIEWING
       - PR no longer merges (base advanced while CI ran) →
         ``pre_review_base_conflict`` → CONFLICT_RESOLVING. Hands the
         conflict to the LLM conflict_resolver and skips the doomed
         reviewer pass — code that must be rebased gets re-CI'd and
         re-reviewed afterwards anyway. Conservative: only an explicit
         ``mergeable=false`` diverts; an unknown bit falls through to
         ``ci_green``.
   - ``failure`` / ``error`` / ``timed_out`` → classify the failure
     (infra-vs-real) via ``ci_freshness.classify_ci_result`` against
     the failing jobs' CI **log content**:
       - ``infra_broken`` → ``ci_infra_recheck`` → DISCOVERED. The
         CI-freshness gate then re-handles the workflow: its
         rerun-budget counter accumulates and triggers another
         empty-commit rerun, or routes to STUCK once the budget is
         exhausted. This closes the gate⇄AWAITING_CI loop — without
         it a reran-but-still-infra-broken PR would send an
         implementer to flail on infra (the original incident).
       - ``fresh_real`` (or no log fetcher wired / unclassifiable)
         → ``ci_red_retry_same_tier`` → IMPLEMENTING (the scheduler
         enqueues a fresh implementer attempt at the same tier;
         pickup_guard + ci_poll_exhaustion catch infinite loops).
   - ``pending`` / ``queued`` / ``in_progress`` → no-op (wait for
     the next tick)
   - None / fetch failure → no-op (transient; reconciliation also
     re-checks PR state independently)

What this module DOES NOT do (yet):
- Per-gate CI summarization. ``ci_summarize.py`` exists but plugging
  it in would require fetching every job log per gate — heavy. For
  the trial we only need the overall state to advance the workflow.
  The next implementer attempt's prefetch picks up the failing
  gates via the existing ci_summary path.
- Escalation to ``ci_red_escalate``. Always uses retry_same_tier;
  ESCALATING is reached via the regular attempts-per-tier exhaustion
  path (pickup_guard / outcome mapper).
- Flake detection. Future ``ci_flake_retry`` integration ships when
  the flake classifier lands.
"""

from __future__ import annotations

import json
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timezone

from sqlalchemy import text
from sqlalchemy.engine import Engine

from ..db.session import session_scope
from ..state_machine import IllegalTransitionError, apply_event
from .ci_freshness import GetFailureLogsCallback, classify_ci_result
from .ci_run_status import (
    GetActionTasksCallback,
    classify_ci_run,
    terminal_verdict,
)

logger = logging.getLogger(__name__)


# get_ci_status(owner, repo, head_sha) → Forgejo combined-status
# dict ({"state": "...", "statuses": [...]}) or None on transient
# fetch failure.
GetCIStatusCallback = Callable[[str, str, str], dict | None]


# get_pr_details(owner, repo, pr_number) → the full Forgejo PR body
# (which carries the ``mergeable`` bit) or None on fetch failure.
# Used by the pre-review mergeable gate. Mirrors the callback the
# CI-freshness gate already consumes.
GetPRDetailsCallback = Callable[[str, str, int], dict | None]


# CI combined-status states that are a failure needing infra-vs-real
# classification before an event is chosen.
_CLASSIFY_STATES: frozenset[str] = frozenset(
    {
        "failure",
        "error",
        "timed_out",
    }
)


# Map Forgejo combined-status state → state-machine event.
# None = no-op (wait for next tick).
#
# For failure states (failure / error / timed_out) the entry here is
# the *fallback* event — used when the infra-vs-real classifier yields
# ``fresh_real`` (or no log fetcher is wired). An ``infra_broken``
# verdict overrides it with ``ci_infra_recheck`` (→ DISCOVERED) so the
# CI-freshness gate re-handles the workflow under its rerun budget.
# See ``_decide_event``.
#
# CA8: ``cancelled`` and ``stale`` are NOT genuine CI failures —
# ``cancelled`` is typically an operator hitting "cancel job" or CI
# system shutting down; ``stale`` is when a new push superseded the
# run on a different branch (common during force-pushes). Treating
# these as ci_red burns a ``pickup_count`` slot on a healthy PR,
# which combined with MAX_PICKUPS pushes the workflow to STUCK after
# a few cancels. Map them to None (wait for the next tick) so the
# poller picks up the eventual real status (or operator can re-run).
# ``timed_out`` IS a real failure — keep that as red.
_STATE_TO_EVENT: dict[str | None, str | None] = {
    "success": "ci_green",
    "failure": "ci_red_retry_same_tier",
    "error": "ci_red_retry_same_tier",
    "pending": None,
    "queued": None,
    "in_progress": None,
    "warning": "ci_green",  # advisory; treat as passed
    "neutral": "ci_green",
    "skipped": "ci_green",
    "cancelled": None,  # CA8 — operator/CI-system action, not a failure
    "timed_out": "ci_red_retry_same_tier",
    "action_required": None,  # human intervention needed; wait
    "stale": None,  # CA8 — superseded run, wait for current to land
    None: None,
}


def _head_sha_from_rerun_payload(raw: object) -> str | None:
    """Extract ``new_head_sha`` from a ``ci-rerun-triggered`` event
    payload.

    The CI-freshness gate (``ci_gate.py``) routes a stale-CI workflow
    DISCOVERED → AWAITING_CI after pushing an empty commit, and stashes
    the empty commit's SHA as ``new_head_sha`` in the event payload.
    A workflow that entered AWAITING_CI this way has no implementer
    attempt, so this is the only place the SHA-to-poll lives.

    ``raw`` may arrive as a dict (Postgres JSON column) or a JSON
    string (SQLite ``text()`` SELECT). Returns None for anything
    unparseable.
    """
    payload: dict | None = None
    if isinstance(raw, dict):
        payload = raw
    elif isinstance(raw, str) and raw:
        try:
            parsed = json.loads(raw)
        except (TypeError, ValueError):
            return None
        if isinstance(parsed, dict):
            payload = parsed
    if payload is None:
        return None
    sha = payload.get("new_head_sha")
    return sha if isinstance(sha, str) and sha else None


@dataclass
class CIStatusPollReport:
    """Per-sweep summary."""

    workflows_scanned: int = 0
    workflows_advanced_green: int = 0
    workflows_advanced_red: int = 0
    # Reran-but-still-infra-broken: routed AWAITING_CI → DISCOVERED so
    # the CI-freshness gate re-handles them under its rerun budget.
    workflows_infra_recheck: int = 0
    # CI passed but the PR no longer merged cleanly into base; routed
    # AWAITING_CI → CONFLICT_RESOLVING by the pre-review mergeable gate
    # (skips a doomed reviewer pass).
    workflows_pre_review_conflict: int = 0
    workflows_waiting: int = 0
    workflows_fetch_failed: int = 0
    transitions: list[tuple[int, str, str]] = field(default_factory=list)
    # Each entry: (workflow_id, ci_state, event_fired)


def _decide_failure_event(
    *,
    owner: str,
    repo: str,
    head_sha: str,
    wf_id: int,
    ci: dict,
    fallback_event: str,
    get_failure_logs: GetFailureLogsCallback | None,
) -> str:
    """Choose the state-machine event for a failure CI result.

    Fetches the failing jobs' CI log content (when a fetcher is wired)
    and classifies the failure infra-vs-real via
    ``classify_ci_result``:
      - ``infra_broken`` → ``ci_infra_recheck`` (→ DISCOVERED; the
        CI-freshness gate re-handles it under the rerun budget).
      - anything else (incl. no fetcher / unclassifiable) →
        ``fallback_event`` (the real-failure path, normally
        ``ci_red_retry_same_tier``).

    Never raises — a log-fetch failure degrades to ``fallback_event``.
    """
    failure_logs = ""
    if get_failure_logs is not None:
        try:
            fetched = get_failure_logs(owner, repo, head_sha)
            failure_logs = fetched if isinstance(fetched, str) else ""
        except Exception as exc:  # noqa: BLE001 — never abort the tick
            logger.warning(
                "ci_status_poll: get_failure_logs raised for workflow %s (head=%s): %s",
                wf_id,
                head_sha[:12],
                exc,
            )
    classification = classify_ci_result(
        head_sha=head_sha,
        forgejo_status=ci,
        failure_logs=failure_logs,
    )
    # infra_broken (log signature) OR stale (the failure is old and
    # unconfirmable) OR indeterminate (the full log carries no verdict
    # marker — a hard-kill / OOM) → bounce AWAITING_CI → DISCOVERED so
    # the CI-freshness gate re-handles it under the rerun budget.
    if classification.verdict in ("infra_broken", "stale", "indeterminate"):
        logger.info(
            "ci_status_poll: workflow %s reran CI not a real verdict "
            "(verdict=%s: %s); routing AWAITING_CI → DISCOVERED for the "
            "CI-freshness gate to re-handle (signatures: %s)",
            wf_id,
            classification.verdict,
            classification.reason,
            classification.matched_signatures,
        )
        return "ci_infra_recheck"
    return fallback_event


def _decide_green_event(
    *,
    owner: str,
    repo: str,
    pr_number: int | None,
    wf_id: int,
    get_pr_details: GetPRDetailsCallback | None,
) -> str:
    """Pre-review mergeable gate — run when CI is green, before the
    expensive LLM reviewer pass.

    One Forgejo PR-detail fetch: if the PR explicitly reports
    ``mergeable=false`` the base advanced while CI ran and the code
    must be rebased — return ``pre_review_base_conflict`` so the
    workflow routes straight to CONFLICT_RESOLVING and skips the
    doomed review (the resolved code gets re-CI'd + re-reviewed
    afterwards anyway).

    Conservative by construction — returns ``ci_green`` (no gating)
    whenever the mergeable bit can't be trusted:
      - no fetcher wired, or no PR number,
      - the fetch failed / returned a non-dict,
      - ``mergeable`` is absent or not a bool. Forgejo computes the
        bit asynchronously; right after a base push it can be stale
        or uncomputed, so only an explicit ``mergeable is False``
        diverts the workflow. A fresh PR is never false-routed.

    Never raises — any failure degrades to ``ci_green``.
    """
    if get_pr_details is None or pr_number is None:
        return "ci_green"
    try:
        pr = get_pr_details(owner, repo, int(pr_number))
    except Exception as exc:  # noqa: BLE001 — never abort the tick
        logger.warning(
            "ci_status_poll: pre-review mergeable fetch raised for "
            "workflow %s (PR #%s): %s",
            wf_id,
            pr_number,
            exc,
        )
        return "ci_green"
    if not isinstance(pr, dict):
        return "ci_green"
    # Known trade-off: Forgejo recomputes ``mergeable`` asynchronously,
    # so a base advancing *during* CI opens a small window where this
    # fetch races the recompute and reads a transient ``false``. The
    # gate runs minutes after the push (post-CI), so the bit is settled
    # in the overwhelming majority of cases. A false positive is
    # bounded and self-heals — the conflict_resolver rebases cleanly
    # and routes back to REVIEWING — but it does decrement the
    # conflict budget (a phantom conflict count escalates a future
    # *real* conflict one tier early). The ``mergeable`` field is
    # mirrored into the event payload so this rate stays observable.
    if pr.get("mergeable") is False:
        logger.info(
            "ci_status_poll: workflow %s (PR #%s) CI is green but "
            "Forgejo reports mergeable=false; routing AWAITING_CI → "
            "CONFLICT_RESOLVING (pre-review gate) to skip the doomed "
            "reviewer pass on code that needs rebasing",
            wf_id,
            pr_number,
        )
        return "pre_review_base_conflict"
    return "ci_green"


def run_ci_status_poll_tick(
    engine: Engine,
    *,
    owner: str,
    repo: str,
    get_ci_status: GetCIStatusCallback,
    get_failure_logs: GetFailureLogsCallback | None = None,
    get_pr_details: GetPRDetailsCallback | None = None,
    get_action_tasks: GetActionTasksCallback | None = None,
) -> CIStatusPollReport:
    """One sweep: poll Forgejo CI status for every AWAITING_CI
    workflow in this (owner, repo) + apply state-machine transitions.

    ``get_failure_logs`` is the CI job-log fetcher (backed by
    ``_ci_logs`` in production). On a failure CI result the poll
    fetches the failing jobs' log content and classifies the failure
    infra-vs-real: an ``infra_broken`` verdict routes the workflow
    AWAITING_CI → DISCOVERED (``ci_infra_recheck``) so the
    CI-freshness gate re-handles it under its rerun budget, instead of
    sending an implementer to flail on infra. When the fetcher is not
    wired the poll keeps the legacy ``ci_red_retry_same_tier``
    behaviour for every failure (conservative).

    ``get_pr_details`` is the Forgejo PR-detail fetcher. When wired,
    a green CI result passes through the pre-review mergeable gate
    (``_decide_green_event``): a PR that no longer merges cleanly into
    base is routed AWAITING_CI → CONFLICT_RESOLVING
    (``pre_review_base_conflict``) instead of REVIEWING, so the
    expensive LLM reviewer is not spent on code that must be rebased.
    When the fetcher is not wired every green CI advances straight to
    REVIEWING (legacy behaviour).
    """
    report = CIStatusPollReport()
    now = datetime.now(timezone.utc)

    with session_scope(engine) as session:
        # Find AWAITING_CI workflows + the SHA whose CI to poll.
        #
        # CA6: both implementer AND conflict_resolver push commits.
        # The previous query filtered ``a.role = 'implementer'`` so
        # workflows that re-entered AWAITING_CI after CONFLICT_RESOLVING
        # → IMPLEMENTING used a stale pre-conflict SHA. Union both roles
        # and pick whichever attempt has the highest attempt_number.
        #
        # PD9: filter on ``a.outcome = 'resolved'`` (implementer) or
        # ``a.outcome = 'resolved'`` (conflict_resolver) — a blocked
        # attempt has ``head_sha_after`` equal to ``head_sha_before``
        # (no push happened), so the previous query would treat the
        # stale pre-blocked SHA as the "latest push" and false-green
        # advance to REVIEWING for code that was never re-pushed.
        #
        # CI-gate fix: a workflow can also enter AWAITING_CI directly
        # from DISCOVERED via the CI-freshness gate (an empty-commit
        # CI rerun). That path has NO implementer/conflict_resolver
        # attempt, so the attempt sub-SELECT yields NULL and the
        # workflow would be skipped forever. The COALESCE second arm
        # reads the head SHA the gate stashed in the most-recent
        # ``ci-rerun-triggered`` event payload.
        #
        # RUN_CI_LOCAL: ``ci-not-ready`` is also accepted in the
        # attempt sub-SELECT. That outcome means the implementer ran
        # before the on-demand local CI had a verdict (no push); the
        # workflow is parked in AWAITING_CI specifically to poll the PR
        # head. Its ``head_sha_after`` == ``head_sha_before`` == the
        # current PR head — exactly the SHA to poll. Without it a fresh
        # PR (no resolved attempt, no ci-rerun event) would have no SHA
        # and sit in AWAITING_CI until ci_poll_exhaustion.
        rows = session.execute(
            text(
                "SELECT w.workflow_id, w.entity_number, "
                "       (SELECT a.head_sha_after FROM workflow_attempts a "
                "         WHERE a.workflow_id = w.workflow_id "
                "           AND a.role IN ('implementer', 'conflict_resolver') "
                "           AND a.status = 'complete' "
                "           AND a.outcome IN ('resolved', 'ci-not-ready') "
                "           AND a.head_sha_after IS NOT NULL "
                "         ORDER BY a.attempt_number DESC LIMIT 1) AS attempt_sha, "
                "       (SELECT e.payload FROM controller_events e "
                "         WHERE e.workflow_id = w.workflow_id "
                "           AND e.event_type = 'ci-rerun-triggered' "
                "         ORDER BY e.event_id DESC LIMIT 1) AS rerun_payload "
                "  FROM workflows w "
                " WHERE w.current_state = 'AWAITING_CI' "
                "   AND w.owner = :owner AND w.repo = :repo "
                "   AND w.kind = 'pr'"
            ),
            {"owner": owner, "repo": repo},
        ).all()
        report.workflows_scanned = len(rows)

        for row in rows:
            wf_id = row.workflow_id
            head_sha = row.attempt_sha
            if not head_sha:
                # No attempt-derived SHA — try the CI-gate rerun event.
                head_sha = _head_sha_from_rerun_payload(row.rerun_payload)
            if not head_sha:
                logger.warning(
                    "ci_status_poll: workflow %s in AWAITING_CI without "
                    "a head_sha (no completed implementer attempt and "
                    "no ci-rerun-triggered event); skipping",
                    wf_id,
                )
                report.workflows_waiting += 1
                continue

            try:
                ci = get_ci_status(owner, repo, head_sha)
            except Exception as exc:  # noqa: BLE001 — transient
                logger.warning(
                    "ci_status_poll: fetch failed for workflow %s (head=%s): %s",
                    wf_id,
                    head_sha[:12],
                    exc,
                )
                report.workflows_fetch_failed += 1
                continue

            if ci is None:
                report.workflows_fetch_failed += 1
                continue

            # Don't fire ci_green / ci_red until the run is actually
            # done. Forgejo's combined ``state`` goes ``failure`` the
            # moment one gate fails even mid-run (and can read
            # ``success`` before a late gate reports). classify_ci_run
            # resolves a still-pending run as genuinely running (wait)
            # vs. a zombie (a dead run / a gate whose terminal status
            # was never posted — ``status-check`` does this routinely).
            run_class = classify_ci_run(
                ci.get("statuses") or [],
                now=now,
                get_action_tasks=get_action_tasks,
                owner=owner,
                repo=repo,
                head_sha=head_sha,
            )
            if run_class == "running":
                report.workflows_waiting += 1
                continue
            if run_class == "stale":
                # Zombie pending gate(s): the run has stopped. Take the
                # verdict from the gates that finished — "categorize it
                # and move on" — and map it STRAIGHT to the event. Do
                # NOT route through _decide_failure_event: it re-reads
                # the zombie pending gate and bounces a real, terminal
                # red verdict to ci_infra_recheck. The implementer in
                # IMPLEMENTING does the infra-vs-real call itself (it
                # has the ci-infra-failure outcome + the CI re-trigger).
                # A fully-dead run (no terminal gate at all) → red.
                verdict = terminal_verdict(ci.get("statuses") or [])
                event = (
                    "ci_green" if verdict == "success" else "ci_red_retry_same_tier"
                )
                # Downstream logging + the controller_events payload read
                # ci_state; for a stale run it IS the terminal verdict.
                ci_state = verdict
                logger.info(
                    "ci_status_poll: workflow %s CI run is STALE/zombie "
                    "(head=%s) — terminal verdict %s -> %s",
                    wf_id,
                    head_sha[:12],
                    verdict,
                    event,
                )
            else:  # "complete"
                ci_state = ci.get("state")
                # Warn on unknown Forgejo states so a new state string
                # surfaces in operator logs instead of being silently
                # treated as "pending" (TE9: previously indistinguishable
                # from real pending — operators got no signal).
                if ci_state not in _STATE_TO_EVENT:
                    logger.warning(
                        "ci_status_poll: workflow %s reported unknown CI "
                        "state %r (head=%s); treating as wait — add to "
                        "_STATE_TO_EVENT if recurring",
                        wf_id,
                        ci_state,
                        (head_sha or "")[:12],
                    )
                event = _STATE_TO_EVENT.get(ci_state)
                if event is None:
                    # Pending / queued / in_progress / unknown → wait.
                    report.workflows_waiting += 1
                    continue
                # A failure result needs an infra-vs-real classification:
                # an infra-broken failure routes back to DISCOVERED (the
                # CI-freshness gate re-handles it under its rerun budget)
                # rather than sending an implementer to flail on infra.
                if isinstance(ci_state, str) and ci_state in _CLASSIFY_STATES:
                    event = _decide_failure_event(
                        owner=owner,
                        repo=repo,
                        head_sha=head_sha,
                        wf_id=wf_id,
                        ci=ci,
                        fallback_event=event,
                        get_failure_logs=get_failure_logs,
                    )

            # A green CI result passes through the pre-review mergeable
            # gate: a PR whose base advanced while CI ran no longer
            # merges cleanly, so route it to CONFLICT_RESOLVING now and
            # skip the doomed reviewer pass. Cheap detection (one API
            # call) gating expensive resolution.
            if event == "ci_green":
                event = _decide_green_event(
                    owner=owner,
                    repo=repo,
                    pr_number=row.entity_number,
                    wf_id=wf_id,
                    get_pr_details=get_pr_details,
                )

            try:
                new_state = apply_event("AWAITING_CI", event)
            except (IllegalTransitionError, ValueError) as exc:
                logger.warning(
                    "ci_status_poll: apply_event(AWAITING_CI, %s) "
                    "raised %s for workflow %s; skipping",
                    event,
                    exc,
                    wf_id,
                )
                continue

            # Apply via direct UPDATE filtered on current_state so a
            # concurrent reconciliation can't double-transition (TOCTOU
            # defense, same pattern as promote.py).
            result = session.execute(
                text(
                    "UPDATE workflows SET "
                    "  current_state = :to_state, "
                    "  last_transition_at = :now, "
                    "  entered_state_at = :now "
                    "WHERE workflow_id = :wf_id "
                    "  AND current_state = 'AWAITING_CI'"
                ),
                {
                    "to_state": new_state,
                    "now": now,
                    "wf_id": wf_id,
                },
            )
            if (result.rowcount or 0) == 0:
                logger.info(
                    "ci_status_poll: workflow %s no longer in AWAITING_CI "
                    "(race lost); skipping event row",
                    wf_id,
                )
                continue
            event_payload: dict[str, object] = {
                "reason": event,
                "ci_state": ci_state,
                "head_sha": head_sha,
                "source": "ci_status_poll",
            }
            if event == "pre_review_base_conflict":
                # Record the bit the pre-review mergeable gate acted on
                # so the gate's (rare) false-positive rate is observable
                # straight from controller_events — no log-scraping.
                event_payload["mergeable"] = False
            session.execute(
                text(
                    "INSERT INTO controller_events "
                    "(workflow_id, ts, event_type, from_state, to_state, "
                    " payload, forgejo_write_pending, replay_attempts) "
                    "VALUES (:wf_id, :ts, :event_type, 'AWAITING_CI', "
                    "        :to_state, :payload, 0, 0)"
                ),
                {
                    "wf_id": wf_id,
                    "ts": now,
                    # PD8: use the state-machine event name (ci_green /
                    # ci_red_retry_same_tier) directly so consumers
                    # joining on event_type don't see schema drift
                    # between sources.
                    "event_type": event,
                    "to_state": new_state,
                    "payload": json.dumps(event_payload),
                },
            )

            if event == "ci_green":
                report.workflows_advanced_green += 1
            elif event == "ci_infra_recheck":
                report.workflows_infra_recheck += 1
            elif event == "pre_review_base_conflict":
                report.workflows_pre_review_conflict += 1
            else:
                report.workflows_advanced_red += 1
            report.transitions.append((wf_id, ci_state, event))

    if (
        report.workflows_advanced_green
        or report.workflows_advanced_red
        or report.workflows_infra_recheck
        or report.workflows_pre_review_conflict
    ):
        logger.info(
            "ci_status_poll: %d scanned (green=%d, red=%d, "
            "infra_recheck=%d, pre_review_conflict=%d, waiting=%d, "
            "fetch_failed=%d)",
            report.workflows_scanned,
            report.workflows_advanced_green,
            report.workflows_advanced_red,
            report.workflows_infra_recheck,
            report.workflows_pre_review_conflict,
            report.workflows_waiting,
            report.workflows_fetch_failed,
        )
    return report


__all__ = [
    "CIStatusPollReport",
    "GetCIStatusCallback",
    "GetPRDetailsCallback",
    "run_ci_status_poll_tick",
]