cleveragents-core/tools/controller/master/ci_run_status.py

"""Zombie / stale CI-run detection.

A CI gate stuck ``pending`` is ambiguous — the job may genuinely be
running, or the run may be dead (a crashed runner, an Actions job whose
terminal commit-status was never posted). ``CI / status-check`` — the
aggregation gate — zombies routinely on this deployment. The controller
must not wait forever on the dead case.

``classify_ci_run`` resolves a CI run to one of:

- ``complete`` — every gate reached a terminal status.
- ``running``  — at least one gate is pending AND the run is live.
- ``stale``    — at least one gate is pending but the run is NOT live;
                 the pending gates are zombies. The verdict is taken
                 from the gates that did finish (see ``terminal_verdict``).

"Live" is decided by two checks:

1. ACTIVE-RUN — when an Actions-task fetcher is supplied, ask Forgejo
   whether a task for the commit is still running. A running task is
   authoritative proof the run is live. The ABSENCE of one is NOT proof
   it is dead — a just-pushed run has no task until a runner picks it
   up — so "no active task" only counts as stale once the run has ALSO
   had no gate update for > ``ZOMBIE_GRACE``.
2. AGE — when the Actions API is unavailable, a run with no gate update
   in > ``STALE_AFTER`` has stopped progressing. The fallback.
"""

from __future__ import annotations

import logging
import os
from collections.abc import Callable
from datetime import UTC, datetime, timedelta

logger = logging.getLogger(__name__)

# A run whose newest gate has not changed in this long has stopped
# progressing — any still-pending gate is a zombie. 90 min by default:
# longer than any plausible single CI job here, short enough that a
# dead run is caught well before it wastes a workflow's wallclock.
STALE_AFTER_MIN = int(os.environ.get("CONTROLLER_CI_STALE_AFTER_MIN", "90"))
STALE_AFTER = timedelta(minutes=STALE_AFTER_MIN)

# Grace period before "no active Actions task" is trusted as proof a run
# is dead. A freshly pushed run has no task simply because no runner has
# picked it up yet, and there are brief gaps between jobs — in both
# cases the Actions API momentarily reports no running task. So "no
# active task" only classifies a run as stale once it has ALSO had no
# gate update for this long. Much shorter than STALE_AFTER: the
# active-task signal lets the verdict be aggressive once the grace has
# elapsed. The PR-46 incident: a 6-second-old run, polled before any
# runner picked it up, was flagged a zombie and bounced before CI could
# even start.
ZOMBIE_GRACE_MIN = int(os.environ.get("CONTROLLER_CI_ZOMBIE_GRACE_MIN", "3"))
ZOMBIE_GRACE = timedelta(minutes=ZOMBIE_GRACE_MIN)

# Per-gate commit-status values that mean "not terminal."
_PENDING_GATE_STATES = {"pending", "queued", "running", "in_progress", ""}
# Failure-ish terminal gate states.
_FAILED_GATE_STATES = {"failure", "error", "timed_out"}
# Forgejo Actions *task* statuses that mean "still going."
_ACTIVE_TASK_STATES = {"running", "waiting", "blocked"}

# get_action_tasks(owner, repo, head_sha) -> list[dict] | None
GetActionTasksCallback = Callable[[str, str, str], "list[dict] | None"]


def _gate_state(s: dict) -> str:
    """Per-gate state; tolerates the ``status``/``state`` key split."""
    return str(s.get("state") or s.get("status") or "").strip().lower()


def _parse_ts(raw: object) -> datetime | None:
    """Parse an ISO-8601 timestamp (Forgejo emits a trailing ``Z``)."""
    if not isinstance(raw, str) or not raw.strip():
        return None
    try:
        dt = datetime.fromisoformat(raw.strip().replace("Z", "+00:00"))
    except ValueError:
        return None
    return dt if dt.tzinfo else dt.replace(tzinfo=UTC)


def newest_gate_update(statuses: list, *, now: datetime) -> datetime | None:
    """Most recent ``updated_at`` / ``created_at`` across all gates.

    A future timestamp (clock skew) is clamped to ``now`` so it cannot
    make a genuinely-old run look fresh.
    """
    newest: datetime | None = None
    for s in statuses:
        if not isinstance(s, dict):
            continue
        for key in ("updated_at", "created_at"):
            ts = _parse_ts(s.get(key))
            if ts is None:
                continue
            if ts > now:
                ts = now
            if newest is None or ts > newest:
                newest = ts
    return newest


def has_pending_gate(statuses: list) -> bool:
    return any(
        isinstance(s, dict) and _gate_state(s) in _PENDING_GATE_STATES
        for s in statuses
    )


def _run_is_active(
    get_action_tasks: GetActionTasksCallback,
    owner: str,
    repo: str,
    head_sha: str,
) -> bool | None:
    """Authoritative liveness check via Forgejo's Actions API.

    Returns True if any Actions task for the commit is still running,
    False if every task is terminal (or there are none), None if the
    lookup is unavailable / errored (caller falls back to the age check).
    """
    try:
        tasks = get_action_tasks(owner, repo, head_sha)
    except Exception as exc:
        logger.warning(
            "get_action_tasks raised for %s/%s@%s: %s",
            owner,
            repo,
            (head_sha or "")[:12],
            exc,
        )
        return None
    if tasks is None:
        return None
    for t in tasks:
        if (
            isinstance(t, dict)
            and str(t.get("status") or "").strip().lower() in _ACTIVE_TASK_STATES
        ):
            return True
    return False


def classify_ci_run(
    statuses: list,
    *,
    now: datetime,
    get_action_tasks: GetActionTasksCallback | None = None,
    owner: str = "",
    repo: str = "",
    head_sha: str = "",
) -> str:
    """Classify a CI run as ``complete`` / ``running`` / ``stale``.

    ``statuses`` is the per-gate list from Forgejo's combined
    commit-status. An empty list yields ``complete`` — the caller's
    combined-state logic then governs (no per-gate detail to reason
    over).
    """
    if not isinstance(statuses, list) or not statuses:
        return "complete"
    if not has_pending_gate(statuses):
        return "complete"
    # Pending gate(s) present — genuinely running, or a zombie?
    # 1. Active-run check. An executing Actions task is authoritative
    #    proof the run is alive. The ABSENCE of one is NOT proof it is
    #    dead: a just-pushed run has no task until a runner picks it up,
    #    and there are brief gaps between jobs. So "no active task" only
    #    classifies as stale once the run has ALSO gone quiet past
    #    ZOMBIE_GRACE — otherwise the controller races its own push
    #    (the PR-46 incident: a 6-second-old run flagged a zombie).
    if get_action_tasks is not None and owner and repo and head_sha:
        active = _run_is_active(get_action_tasks, owner, repo, head_sha)
        if active is True:
            return "running"
        if active is False:
            newest = newest_gate_update(statuses, now=now)
            if newest is not None and (now - newest) > ZOMBIE_GRACE:
                return "stale"
            return "running"
        # active is None → API unavailable; fall through to the age check.
    # 2. Age check — the fallback when the Actions API is unavailable.
    newest = newest_gate_update(statuses, now=now)
    if newest is not None and (now - newest) > STALE_AFTER:
        return "stale"
    return "running"


def terminal_verdict(statuses: list) -> str:
    """Verdict from the gates that reached a terminal status, ignoring
    any zombie pending ones.

    ``failure`` if any terminal gate failed; ``success`` if there is
    ≥1 terminal gate and none failed; ``pending`` only when every gate
    is still pending (a fully-dead run with no verdict at all — the
    caller decides how to route that).
    """
    seen_terminal = False
    for s in statuses:
        if not isinstance(s, dict):
            continue
        st = _gate_state(s)
        if st in _PENDING_GATE_STATES:
            continue
        seen_terminal = True
        if st in _FAILED_GATE_STATES:
            return "failure"
    return "success" if seen_terminal else "pending"


__all__ = [
    "STALE_AFTER",
    "STALE_AFTER_MIN",
    "GetActionTasksCallback",
    "classify_ci_run",
    "has_pending_gate",
    "newest_gate_update",
    "terminal_verdict",
]