cleveragents-core/tools/controller/master/ci_poll.py

"""AWAITING_CI poll-exhaustion handler.

Per plan v9 + the round-2 review (item N2): without this, workflows
that enter ``AWAITING_CI`` and never receive ``ci_green`` / ``ci_red_*``
events (CI runner outage, broken integration, etc.) hang indefinitely
— only ``operator_unstick`` could rescue them.

This module ships the minimum-viable escape: a periodic scan that
finds workflows whose ``entered_state_at`` for AWAITING_CI exceeds
the configured poll-exhaustion threshold + fires the
``ci_polling_exhausted`` event → STUCK.

What this module DOES NOT do (yet):
- Actual CI status polling against Forgejo. The reconciliation tick
  + the prefetch-driven CI summarizer cover that path; this handler
  is the EXIT for workflows that have been polling-without-progress
  for too long.
- Differentiating "CI never reported" from "CI reported but the
  controller missed it". Both fall under the same timeout.

Default threshold: 2 hours (``CONTROLLER_AWAITING_CI_TIMEOUT_S``).
Operators can tune per repo via env.
"""

from __future__ import annotations

import json
import logging
import os
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timezone

from sqlalchemy import text
from sqlalchemy.engine import Engine

from ..db.session import session_scope
from ..state_machine import IllegalTransitionError, apply_event

logger = logging.getLogger(__name__)


DEFAULT_AWAITING_CI_TIMEOUT_S = int(
    os.environ.get("CONTROLLER_AWAITING_CI_TIMEOUT_S", "7200")
)


@dataclass
class CIPollExhaustionReport:
    """Per-sweep summary."""

    workflows_scanned: int = 0
    workflows_exhausted: int = 0
    exhausted_workflow_ids: list[int] = field(default_factory=list)


def run_ci_poll_exhaustion_tick(
    engine: Engine,
    *,
    timeout_s: int | None = None,
    local_ci_in_flight: Callable[[], bool] | None = None,
) -> CIPollExhaustionReport:
    """One sweep: STUCK any AWAITING_CI workflow whose entered_state_at
    is older than ``timeout_s``.

    Composes with the master's tick layers (the loop calls this on
    the same cadence as reconciliation by default — see loop.py).

    ``local_ci_in_flight`` (wired only under ``RUN_CI_LOCAL``): when it
    reports a local CI run is executing, the whole sweep is skipped.
    Local CI is on-demand and serial — a verdict lands minutes after
    the implementer finishes, and other AWAITING_CI workflows queue
    behind it. While that pipeline is busy every AWAITING_CI wait is
    legitimate progress, so the poll-exhaustion timer (sized for remote
    CI) must not STUCK a workflow whose verdict is simply not done yet.
    """
    threshold = timeout_s or DEFAULT_AWAITING_CI_TIMEOUT_S
    report = CIPollExhaustionReport()
    now = datetime.now(timezone.utc)

    if local_ci_in_flight is not None:
        try:
            if local_ci_in_flight():
                logger.info(
                    "ci_poll_exhaustion: local CI run in flight — skipping "
                    "this sweep; AWAITING_CI workflows keep waiting for the "
                    "on-demand verdict"
                )
                return report
        except Exception:  # noqa: BLE001 — a probe failure must not abort the sweep
            logger.exception(
                "ci_poll_exhaustion: local-CI in-flight probe raised; "
                "proceeding with the sweep"
            )

    with session_scope(engine) as session:
        dialect = session.bind.dialect.name if session.bind else "sqlite"
        if dialect == "postgresql":
            select_sql = text(
                "SELECT workflow_id, current_state, entered_state_at "
                "  FROM workflows "
                " WHERE current_state = 'AWAITING_CI' "
                "   AND entered_state_at IS NOT NULL "
                "   AND entered_state_at + (:threshold || ' seconds')::interval < :now"
            )
        else:
            # SQLite: TIMESTAMP arithmetic via julianday.
            select_sql = text(
                "SELECT workflow_id, current_state, entered_state_at "
                "  FROM workflows "
                " WHERE current_state = 'AWAITING_CI' "
                "   AND entered_state_at IS NOT NULL "
                "   AND (julianday(:now) - julianday(entered_state_at)) "
                "       * 86400 > :threshold"
            )
        rows = session.execute(
            select_sql,
            {"threshold": threshold, "now": now},
        ).all()
        report.workflows_scanned = len(rows)
        for row in rows:
            try:
                new_state = apply_event(
                    row.current_state,
                    "ci_polling_exhausted",
                )
            except (IllegalTransitionError, ValueError) as exc:
                # IllegalTransitionError: state doesn't accept the event.
                # ValueError: state name isn't in KNOWN_STATES (DB row
                # corruption or an unknown-state guard miss). Both
                # should skip this row rather than aborting the tick.
                logger.warning(
                    "ci_polling_exhausted: workflow_id=%s state=%r "
                    "rejected the event (%s); skipping",
                    row.workflow_id,
                    row.current_state,
                    exc,
                )
                continue
            session.execute(
                text(
                    "UPDATE workflows SET "
                    "  current_state = :to_state, "
                    "  last_transition_at = :now, "
                    "  entered_state_at = :now "
                    "WHERE workflow_id = :wf_id"
                ),
                {
                    "to_state": new_state,
                    "now": now,
                    "wf_id": row.workflow_id,
                },
            )
            session.execute(
                text(
                    "INSERT INTO controller_events "
                    "(workflow_id, ts, event_type, from_state, to_state, "
                    " payload, forgejo_write_pending, replay_attempts) "
                    "VALUES (:wf_id, :ts, 'ci_poll_exhausted', "
                    "        :from_state, :to_state, :payload, 0, 0)"
                ),
                {
                    "wf_id": row.workflow_id,
                    "ts": now,
                    "from_state": row.current_state,
                    "to_state": new_state,
                    "payload": json.dumps(
                        {
                            "reason": "awaiting_ci_timeout",
                            "threshold_seconds": threshold,
                            # text() SELECTs return TIMESTAMP as a string on
                            # SQLite (and a datetime on Postgres). Normalize.
                            "entered_state_at": (
                                row.entered_state_at.isoformat()
                                if hasattr(row.entered_state_at, "isoformat")
                                else (
                                    str(row.entered_state_at)
                                    if row.entered_state_at
                                    else None
                                )
                            ),
                            "source": "ci_poll_exhaustion",
                        }
                    ),
                },
            )
            report.workflows_exhausted += 1
            report.exhausted_workflow_ids.append(row.workflow_id)

    if report.workflows_exhausted:
        logger.warning(
            "ci_poll_exhaustion: %d workflow(s) STUCK after %ds in "
            "AWAITING_CI (ids: %s)",
            report.workflows_exhausted,
            threshold,
            report.exhausted_workflow_ids,
        )
    return report


__all__ = [
    "CIPollExhaustionReport",
    "DEFAULT_AWAITING_CI_TIMEOUT_S",
    "run_ci_poll_exhaustion_tick",
]