cleveragents-core/tools/controller/master/ci_status_poll.py

"""AWAITING_CI status poll — autonomous CI exit (Phase 1k++++ trial).

Closes the last operator-intervention gap for the Phase 2 trial.
Previously, workflows that transitioned to AWAITING_CI sat there
forever (only ``ci_polling_exhausted`` fired, leading to STUCK after
the 2h timeout). The RUNBOOK had a manual SQL workaround.

This module ships an autonomous tick:

1. SELECT workflows in AWAITING_CI + their latest implementer
   attempt's ``head_sha_after`` (the SHA the implementer pushed).
2. For each: call ``get_ci_status(owner, repo, head_sha)`` — wraps
   Forgejo's ``/commits/{sha}/status`` combined-status endpoint.
3. Decide the next event based on the combined ``state`` field:
   - ``success`` → ``ci_green`` → REVIEWING
   - ``failure`` / ``error`` → ``ci_red_retry_same_tier`` →
     IMPLEMENTING (the scheduler will then enqueue a fresh
     implementer attempt at the same tier; pickup_guard +
     ci_poll_exhaustion catch infinite loops)
   - ``pending`` / ``queued`` / ``in_progress`` → no-op (wait for
     the next tick)
   - None / fetch failure → no-op (transient; reconciliation also
     re-checks PR state independently)

What this module DOES NOT do (yet):
- Per-gate CI summarization. ``ci_summarize.py`` exists but plugging
  it in would require fetching every job log per gate — heavy. For
  the trial we only need the overall state to advance the workflow.
  The next implementer attempt's prefetch picks up the failing
  gates via the existing ci_summary path.
- Escalation to ``ci_red_escalate``. Always uses retry_same_tier;
  ESCALATING is reached via the regular attempts-per-tier exhaustion
  path (pickup_guard / outcome mapper).
- Flake detection. Future ``ci_flake_retry`` integration ships when
  the flake classifier lands.
"""
from __future__ import annotations

import json
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timezone

from sqlalchemy import text
from sqlalchemy.engine import Engine

from ..db.session import session_scope
from ..state_machine import IllegalTransitionError, apply_event

logger = logging.getLogger(__name__)


# get_ci_status(owner, repo, head_sha) → Forgejo combined-status
# dict ({"state": "...", "statuses": [...]}) or None on transient
# fetch failure.
GetCIStatusCallback = Callable[[str, str, str], dict | None]


# Map Forgejo combined-status state → state-machine event.
# None = no-op (wait for next tick).
#
# CA8: ``cancelled`` and ``stale`` are NOT genuine CI failures —
# ``cancelled`` is typically an operator hitting "cancel job" or CI
# system shutting down; ``stale`` is when a new push superseded the
# run on a different branch (common during force-pushes). Treating
# these as ci_red burns a ``pickup_count`` slot on a healthy PR,
# which combined with MAX_PICKUPS pushes the workflow to STUCK after
# a few cancels. Map them to None (wait for the next tick) so the
# poller picks up the eventual real status (or operator can re-run).
# ``timed_out`` IS a real failure — keep that as red.
_STATE_TO_EVENT: dict[str | None, str | None] = {
    "success": "ci_green",
    "failure": "ci_red_retry_same_tier",
    "error": "ci_red_retry_same_tier",
    "pending": None,
    "queued": None,
    "in_progress": None,
    "warning": "ci_green",  # advisory; treat as passed
    "neutral": "ci_green",
    "skipped": "ci_green",
    "cancelled": None,  # CA8 — operator/CI-system action, not a failure
    "timed_out": "ci_red_retry_same_tier",
    "action_required": None,  # human intervention needed; wait
    "stale": None,  # CA8 — superseded run, wait for current to land
    None: None,
}


@dataclass
class CIStatusPollReport:
    """Per-sweep summary."""

    workflows_scanned: int = 0
    workflows_advanced_green: int = 0
    workflows_advanced_red: int = 0
    workflows_waiting: int = 0
    workflows_fetch_failed: int = 0
    transitions: list[tuple[int, str, str]] = field(default_factory=list)
    # Each entry: (workflow_id, ci_state, event_fired)


def run_ci_status_poll_tick(
    engine: Engine, *,
    owner: str, repo: str,
    get_ci_status: GetCIStatusCallback,
) -> CIStatusPollReport:
    """One sweep: poll Forgejo CI status for every AWAITING_CI
    workflow in this (owner, repo) + apply state-machine transitions.
    """
    report = CIStatusPollReport()
    now = datetime.now(timezone.utc)

    with session_scope(engine) as session:
        # Find AWAITING_CI workflows + the SHA whose CI to poll.
        #
        # CA6: both implementer AND conflict_resolver push commits.
        # The previous query filtered ``a.role = 'implementer'`` so
        # workflows that re-entered AWAITING_CI after CONFLICT_RESOLVING
        # → IMPLEMENTING used a stale pre-conflict SHA. Union both roles
        # and pick whichever attempt has the highest attempt_number.
        #
        # PD9: filter on ``a.outcome = 'resolved'`` (implementer) or
        # ``a.outcome = 'resolved'`` (conflict_resolver) — a blocked
        # attempt has ``head_sha_after`` equal to ``head_sha_before``
        # (no push happened), so the previous query would treat the
        # stale pre-blocked SHA as the "latest push" and false-green
        # advance to REVIEWING for code that was never re-pushed.
        rows = session.execute(
            text(
                "SELECT w.workflow_id, w.entity_number, "
                "       (SELECT a.head_sha_after FROM workflow_attempts a "
                "         WHERE a.workflow_id = w.workflow_id "
                "           AND a.role IN ('implementer', 'conflict_resolver') "
                "           AND a.status = 'complete' "
                "           AND a.outcome = 'resolved' "
                "           AND a.head_sha_after IS NOT NULL "
                "         ORDER BY a.attempt_number DESC LIMIT 1) AS head_sha "
                "  FROM workflows w "
                " WHERE w.current_state = 'AWAITING_CI' "
                "   AND w.owner = :owner AND w.repo = :repo "
                "   AND w.kind = 'pr'"
            ),
            {"owner": owner, "repo": repo},
        ).all()
        report.workflows_scanned = len(rows)

        for row in rows:
            wf_id = row.workflow_id
            head_sha = row.head_sha
            if not head_sha:
                logger.warning(
                    "ci_status_poll: workflow %s in AWAITING_CI without "
                    "a head_sha (no completed implementer attempt found); "
                    "skipping", wf_id,
                )
                report.workflows_waiting += 1
                continue

            try:
                ci = get_ci_status(owner, repo, head_sha)
            except Exception as exc:  # noqa: BLE001 — transient
                logger.warning(
                    "ci_status_poll: fetch failed for workflow %s "
                    "(head=%s): %s", wf_id, head_sha[:12], exc,
                )
                report.workflows_fetch_failed += 1
                continue

            if ci is None:
                report.workflows_fetch_failed += 1
                continue

            ci_state = ci.get("state")
            # Warn on unknown Forgejo states so a new state string
            # surfaces in operator logs instead of being silently
            # treated as "pending" (TE9: previously indistinguishable
            # from real pending — operators got no signal).
            if ci_state not in _STATE_TO_EVENT:
                logger.warning(
                    "ci_status_poll: workflow %s reported unknown CI state "
                    "%r (head=%s); treating as wait — add to _STATE_TO_EVENT "
                    "if recurring",
                    wf_id, ci_state, (head_sha or "")[:12],
                )
            event = _STATE_TO_EVENT.get(ci_state)
            if event is None:
                # Pending / queued / in_progress / unknown → wait.
                report.workflows_waiting += 1
                continue

            try:
                new_state = apply_event("AWAITING_CI", event)
            except (IllegalTransitionError, ValueError) as exc:
                logger.warning(
                    "ci_status_poll: apply_event(AWAITING_CI, %s) "
                    "raised %s for workflow %s; skipping",
                    event, exc, wf_id,
                )
                continue

            # Apply via direct UPDATE filtered on current_state so a
            # concurrent reconciliation can't double-transition (TOCTOU
            # defense, same pattern as promote.py).
            result = session.execute(
                text(
                    "UPDATE workflows SET "
                    "  current_state = :to_state, "
                    "  last_transition_at = :now, "
                    "  entered_state_at = :now "
                    "WHERE workflow_id = :wf_id "
                    "  AND current_state = 'AWAITING_CI'"
                ),
                {
                    "to_state": new_state, "now": now, "wf_id": wf_id,
                },
            )
            if (result.rowcount or 0) == 0:
                logger.info(
                    "ci_status_poll: workflow %s no longer in AWAITING_CI "
                    "(race lost); skipping event row",
                    wf_id,
                )
                continue
            session.execute(
                text(
                    "INSERT INTO controller_events "
                    "(workflow_id, ts, event_type, from_state, to_state, "
                    " payload, forgejo_write_pending, replay_attempts) "
                    "VALUES (:wf_id, :ts, :event_type, 'AWAITING_CI', "
                    "        :to_state, :payload, 0, 0)"
                ),
                {
                    "wf_id": wf_id, "ts": now,
                    # PD8: use the state-machine event name (ci_green /
                    # ci_red_retry_same_tier) directly so consumers
                    # joining on event_type don't see schema drift
                    # between sources.
                    "event_type": event,
                    "to_state": new_state,
                    "payload": json.dumps({
                        "reason": event,
                        "ci_state": ci_state,
                        "head_sha": head_sha,
                        "source": "ci_status_poll",
                    }),
                },
            )

            if event == "ci_green":
                report.workflows_advanced_green += 1
            else:
                report.workflows_advanced_red += 1
            report.transitions.append((wf_id, ci_state, event))

    if report.workflows_advanced_green or report.workflows_advanced_red:
        logger.info(
            "ci_status_poll: %d scanned (green=%d, red=%d, waiting=%d, "
            "fetch_failed=%d)",
            report.workflows_scanned, report.workflows_advanced_green,
            report.workflows_advanced_red, report.workflows_waiting,
            report.workflows_fetch_failed,
        )
    return report


__all__ = [
    "CIStatusPollReport",
    "GetCIStatusCallback",
    "run_ci_status_poll_tick",
]