"""CI-freshness gate — the early DISCOVERED tick. THE INCIDENT this addresses --------------------------- A PR was discovered with 2-day-stale CI where every job had failed at the *checkout* step (a ``git fetch`` connection-reset — pure infra, no code ever ran). The pipeline had no notion of CI freshness: the workflow went DISCOVERED → ANALYZING → IMPLEMENTING → STUCK, burning an estimator + a tier-2 implementer attempt before the implementer correctly said "nothing to fix, rerun CI" and dead-ended. The controller never *triggers* CI — it only polls. This tick is the fix. It runs BEFORE the normal DISCOVERED → ANALYZING promotion (``promote.py``). For each ``kind='pr'`` workflow in ``DISCOVERED``: 1. Fetch the PR's head SHA + the CI combined status. 2. When the combined status shows a failure, fetch the failing jobs' CI **log content** via the injected ``get_failure_logs`` callback (backed by ``_ci_logs``). The infra-vs-real decision needs the actual log text — Forgejo status descriptions are generic duration strings and never carry the error. 3. Classify via ``ci_freshness.classify_ci_result``: - ``fresh_real`` / ``pending`` → leave it alone; ``promote.py`` handles the normal path. - ``infra_broken`` / ``stale`` / ``no_ci`` / ``indeterminate`` → trigger a CI rerun (empty-commit push). ``indeterminate`` is a failing run whose full log carries no verdict marker — a hard-kill / OOM that never produced a result. On success: stash the new head SHA, fire ``discovery_ci_rerun_triggered`` (DISCOVERED → AWAITING_CI) via a TOCTOU-guarded UPDATE, and write a ``ci-rerun-triggered`` event. 3. RERUN BUDGET = 3: before triggering, count prior ``ci-rerun-triggered`` events for the workflow. At >= 3, do NOT rerun again — route the workflow to STUCK (the CI runner itself is broken; an operator must investigate). Both this tick and ``promote.py`` filter ``current_state='DISCOVERED'`` with guarded UPDATEs, so they're race-safe even out of order — but the loop orders this one FIRST so a stale-CI workflow is diverted before promotion ever fires. """ from __future__ import annotations import json import logging from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime, timezone from sqlalchemy import text from sqlalchemy.engine import Engine from ..db.session import session_scope from ..state_machine import IllegalTransitionError, apply_event from .ci_freshness import ( GetFailureLogsCallback, classify_ci_result, combined_status_is_failure, ) from .ci_rerun import CIRerunCallback, CIRerunResult logger = logging.getLogger(__name__) # Maximum number of automatic CI reruns per workflow before the gate # gives up and routes the workflow to STUCK for operator attention. RERUN_BUDGET = 3 # controller_events.event_type written when a rerun is triggered. The # rerun-budget counter scans for this exact value, and ci_status_poll # reads the latest such row's payload to recover the gate-pushed SHA. CI_RERUN_EVENT_TYPE = "ci-rerun-triggered" CI_RERUN_BUDGET_EXHAUSTED_EVENT_TYPE = "ci-rerun-budget-exhausted" # get_ci_status(owner, repo, head_sha) -> Forgejo combined-status dict # or None on fetch failure. GetCIStatusCallback = Callable[[str, str, str], dict | None] # get_pr_details(owner, repo, pr_number) -> Forgejo PR dict or None. GetPRDetailsCallback = Callable[[str, str, int], dict | None] @dataclass class CIGateReport: """Per-sweep summary of the CI-freshness gate.""" workflows_scanned: int = 0 # fresh / pending — left for the normal promote path. workflows_left_for_promote: int = 0 # rerun triggered + routed DISCOVERED → AWAITING_CI. workflows_reran: int = 0 # rerun budget hit — routed DISCOVERED → STUCK. workflows_stuck_budget: int = 0 # rerun attempt failed (clone/push error) — left in DISCOVERED. workflows_rerun_failed: int = 0 # could not fetch PR details / head SHA — left in DISCOVERED. workflows_fetch_failed: int = 0 # (workflow_id, verdict, action) for observability/tests. actions: list[tuple[int, str, str]] = field(default_factory=list) def _pr_head(pr: dict | None) -> tuple[str | None, str | None]: """Extract (head_sha, head_ref) from a Forgejo PR dict.""" if not isinstance(pr, dict): return None, None head = pr.get("head") if not isinstance(head, dict): return None, None sha = head.get("sha") ref = head.get("ref") return ( sha if isinstance(sha, str) and sha else None, ref if isinstance(ref, str) and ref else None, ) def _count_prior_reruns(session, workflow_id: int) -> int: """Count the ``ci-rerun-triggered`` events already recorded for a workflow — the rerun-budget counter.""" row = session.execute( text( "SELECT COUNT(*) AS n FROM controller_events " "WHERE workflow_id = :wf_id AND event_type = :etype" ), {"wf_id": workflow_id, "etype": CI_RERUN_EVENT_TYPE}, ).first() return int(row.n) if row is not None else 0 def _guarded_transition( session, *, workflow_id: int, event: str, now: datetime, ) -> str | None: """Apply ``event`` from DISCOVERED via a TOCTOU-guarded UPDATE. Returns the new state on success, or None when the row is no longer DISCOVERED (race lost) / the event is illegal. """ try: new_state = apply_event("DISCOVERED", event) except (IllegalTransitionError, ValueError) as exc: logger.warning( "ci_gate: apply_event(DISCOVERED, %s) rejected for " "workflow %s (%s); skipping", event, workflow_id, exc, ) return None result = session.execute( text( "UPDATE workflows SET " " current_state = :to_state, " " last_transition_at = :now, " " entered_state_at = :now " "WHERE workflow_id = :wf_id " " AND current_state = 'DISCOVERED'" ), {"to_state": new_state, "now": now, "wf_id": workflow_id}, ) if (result.rowcount or 0) == 0: logger.info( "ci_gate: workflow %s no longer DISCOVERED (race lost); skipping %s", workflow_id, event, ) return None return new_state def _write_event( session, *, workflow_id: int, event_type: str, from_state: str, to_state: str, now: datetime, payload: dict, ) -> None: session.execute( text( "INSERT INTO controller_events " "(workflow_id, ts, event_type, from_state, to_state, " " payload, forgejo_write_pending, replay_attempts) " "VALUES (:wf_id, :ts, :etype, :from_state, :to_state, " " :payload, 0, 0)" ), { "wf_id": workflow_id, "ts": now, "etype": event_type, "from_state": from_state, "to_state": to_state, "payload": json.dumps(payload), }, ) def _fetch_failure_logs( get_failure_logs: GetFailureLogsCallback | None, *, owner: str, repo: str, head_sha: str, wf_id: int, ) -> str: """Best-effort fetch of the failing jobs' CI log content. Returns the concatenated log text, or an empty string when no fetcher is wired / the fetch fails. Never raises — a log-fetch failure must not abort the gate; the classifier degrades an un-loggable failure to ``fresh_real`` (conservative). """ if get_failure_logs is None: return "" try: text_out = get_failure_logs(owner, repo, head_sha) except Exception as exc: # noqa: BLE001 — never abort the tick logger.warning( "ci_gate: get_failure_logs raised for workflow %s (head=%s): %s", wf_id, head_sha[:12], exc, ) return "" return text_out if isinstance(text_out, str) else "" def run_ci_gate_tick( engine: Engine, *, owner: str, repo: str, get_ci_status: GetCIStatusCallback, get_pr_details: GetPRDetailsCallback, trigger_ci_rerun: CIRerunCallback, get_failure_logs: GetFailureLogsCallback | None = None, rerun_budget: int = RERUN_BUDGET, ) -> CIGateReport: """One sweep: divert stale/infra-broken DISCOVERED PRs. For each ``kind='pr'`` workflow in ``DISCOVERED`` for this (owner, repo): classify CI freshness and either leave it for the normal promote path, trigger a CI rerun (→ AWAITING_CI), or — once the rerun budget is exhausted — route it to STUCK. ``get_failure_logs`` is the CI job-log fetcher (backed by ``_ci_logs`` in production). The gate calls it only when the combined status shows a failure, and feeds the result to ``classify_ci_result`` so the infra-vs-real decision runs against real log content rather than the generic status descriptions. When it is not wired (older callers / a transient failure), the classifier falls back to ``fresh_real`` for a failure it cannot confirm is infra — conservative by design. All callbacks are injected so tests run without a live Forgejo / git. Callback exceptions are caught and treated as "fetch failed" (the workflow is left in DISCOVERED for the next tick / retry). """ report = CIGateReport() now = datetime.now(timezone.utc) with session_scope(engine) as session: rows = session.execute( text( "SELECT workflow_id, current_state, entity_number " " FROM workflows " " WHERE current_state = 'DISCOVERED' " " AND kind = 'pr' " " AND owner = :owner AND repo = :repo" ), {"owner": owner, "repo": repo}, ).all() report.workflows_scanned = len(rows) for row in rows: wf_id = row.workflow_id pr_number = row.entity_number # 1. Fetch PR head SHA + branch. try: pr = get_pr_details(owner, repo, pr_number) except Exception as exc: # noqa: BLE001 — transient logger.warning( "ci_gate: get_pr_details failed for workflow %s (PR #%s): %s", wf_id, pr_number, exc, ) report.workflows_fetch_failed += 1 report.actions.append((wf_id, "unknown", "fetch-failed")) continue head_sha, head_ref = _pr_head(pr) if not head_sha: logger.warning( "ci_gate: workflow %s (PR #%s) has no head SHA; " "leaving in DISCOVERED", wf_id, pr_number, ) report.workflows_fetch_failed += 1 report.actions.append((wf_id, "unknown", "fetch-failed")) continue # 2. Fetch the CI combined status for the head SHA. try: ci_status = get_ci_status(owner, repo, head_sha) except Exception as exc: # noqa: BLE001 — transient logger.warning( "ci_gate: get_ci_status failed for workflow %s (head=%s): %s", wf_id, head_sha[:12], exc, ) # A fetch failure is itself "no CI verdict available". # Be conservative: treat as fetch-failed (retry next # tick) rather than forcing a rerun on transient noise. report.workflows_fetch_failed += 1 report.actions.append((wf_id, "unknown", "fetch-failed")) continue # Fetch the failing jobs' CI log content — only when the # combined status actually shows a failure (a green/pending # PR has nothing to scan). The classifier scans this text # for infra signatures; without it an infra failure would # be misclassified as fresh_real (the incident's bug). failure_logs = "" if combined_status_is_failure(ci_status): failure_logs = _fetch_failure_logs( get_failure_logs, owner=owner, repo=repo, head_sha=head_sha, wf_id=wf_id, ) classification = classify_ci_result( head_sha=head_sha, forgejo_status=ci_status, failure_logs=failure_logs, ) verdict = classification.verdict # fresh_real / pending → leave for promote.py. if verdict in {"fresh_real", "pending"}: report.workflows_left_for_promote += 1 report.actions.append((wf_id, verdict, "left-for-promote")) continue # infra_broken / stale / no_ci / indeterminate → CI needs # re-triggering. prior_reruns = _count_prior_reruns(session, wf_id) # 3. Rerun budget gate. DISCOVERED has no state-machine # event mapping straight to STUCK (a broken CI runner is a # runner-health condition, not a workflow-logic # transition), so route via a guarded raw UPDATE. if prior_reruns >= rerun_budget: new_state = _force_stuck( session, workflow_id=wf_id, now=now, ) if new_state is None: # Race lost — another tick moved the row. report.actions.append((wf_id, verdict, "budget-race-lost")) continue _write_event( session, workflow_id=wf_id, event_type=CI_RERUN_BUDGET_EXHAUSTED_EVENT_TYPE, from_state="DISCOVERED", to_state=new_state, now=now, payload={ "reason": ( "CI infra still failing after " f"{prior_reruns} reruns — operator must " "investigate the runner" ), "verdict": verdict, "classification_reason": classification.reason, "prior_reruns": prior_reruns, "rerun_budget": rerun_budget, "head_sha": head_sha, "matched_signatures": classification.matched_signatures, "source": "ci_gate", }, ) report.workflows_stuck_budget += 1 report.actions.append((wf_id, verdict, "stuck-budget")) logger.warning( "ci_gate: workflow %s (PR #%s) STUCK — CI infra " "still failing after %d reruns", wf_id, pr_number, prior_reruns, ) continue # 4. Trigger a CI rerun (empty-commit push). if not head_ref: logger.warning( "ci_gate: workflow %s (PR #%s) has no head branch " "ref; cannot push an empty commit", wf_id, pr_number, ) report.workflows_rerun_failed += 1 report.actions.append((wf_id, verdict, "rerun-failed")) continue try: rerun: CIRerunResult = trigger_ci_rerun( owner, repo, head_ref, ) except Exception as exc: # noqa: BLE001 — never abort tick logger.warning( "ci_gate: trigger_ci_rerun raised for workflow %s (PR #%s): %s", wf_id, pr_number, exc, ) report.workflows_rerun_failed += 1 report.actions.append((wf_id, verdict, "rerun-failed")) continue if not rerun.ok or not rerun.new_head_sha: logger.warning( "ci_gate: CI rerun failed for workflow %s (PR #%s): %s", wf_id, pr_number, rerun.error, ) report.workflows_rerun_failed += 1 report.actions.append((wf_id, verdict, "rerun-failed")) continue # Rerun succeeded — route DISCOVERED → AWAITING_CI. new_state = _guarded_transition( session, workflow_id=wf_id, event="discovery_ci_rerun_triggered", now=now, ) if new_state is None: report.actions.append((wf_id, verdict, "rerun-race-lost")) continue _write_event( session, workflow_id=wf_id, event_type=CI_RERUN_EVENT_TYPE, from_state="DISCOVERED", to_state=new_state, now=now, payload={ "reason": "discovery_ci_rerun_triggered", "verdict": verdict, "classification_reason": classification.reason, "matched_signatures": classification.matched_signatures, "failing_contexts": classification.failing_contexts, # The OLD head SHA the classifier saw + the NEW one # the empty-commit push created. ci_status_poll # reads ``new_head_sha`` to know which commit's CI # to poll (the workflow has no implementer attempt). "stale_head_sha": head_sha, "new_head_sha": rerun.new_head_sha, "head_ref": head_ref, "rerun_number": prior_reruns + 1, "source": "ci_gate", }, ) report.workflows_reran += 1 report.actions.append((wf_id, verdict, "rerun-triggered")) logger.info( "ci_gate: workflow %s (PR #%s) — %s CI; triggered " "rerun #%d (new HEAD %s), routed to AWAITING_CI", wf_id, pr_number, verdict, prior_reruns + 1, rerun.new_head_sha[:12], ) if report.workflows_reran or report.workflows_stuck_budget: logger.info( "ci_gate: %d scanned (reran=%d, stuck-budget=%d, " "left-for-promote=%d, rerun-failed=%d, fetch-failed=%d)", report.workflows_scanned, report.workflows_reran, report.workflows_stuck_budget, report.workflows_left_for_promote, report.workflows_rerun_failed, report.workflows_fetch_failed, ) return report def _force_stuck( session, *, workflow_id: int, now: datetime, ) -> str | None: """Route a DISCOVERED workflow directly to STUCK via a guarded UPDATE. DISCOVERED has no state-machine event mapping straight to STUCK (the budget-exhausted case is an operator-attention condition the transition table doesn't model — it's a runner-health problem, not a workflow-logic transition). We therefore do a guarded raw UPDATE, exactly the TOCTOU pattern ``promote.py`` uses, and pair it with an audit event so the trail stays intact. """ result = session.execute( text( "UPDATE workflows SET " " current_state = 'STUCK', " " last_transition_at = :now, " " entered_state_at = :now " "WHERE workflow_id = :wf_id " " AND current_state = 'DISCOVERED'" ), {"now": now, "wf_id": workflow_id}, ) if (result.rowcount or 0) == 0: return None return "STUCK" __all__ = [ "CI_RERUN_BUDGET_EXHAUSTED_EVENT_TYPE", "CI_RERUN_EVENT_TYPE", "CIGateReport", "GetCIStatusCallback", "GetPRDetailsCallback", "RERUN_BUDGET", "run_ci_gate_tick", ]