drew
6ab6df319c
Diagnosed live on PR 39 (STUCK 2026-05-20): both CI gates ran 17 minutes, the captured log was <3 minutes and ended mid-execution (`still running` / a just-launched runner) with ZERO error markers. A hard process kill (OOM-killer / pod eviction) cannot flush a buffer, print a traceback or emit an exit code — it leaves a *hole*, not a phrase. The classifier called this `fresh_real`, the implementer was sent to "fix" a failure with nothing to fix, and the PR dead-ended at blocked → STUCK. New `indeterminate` verdict: a failing gate whose FULL log carries no terminal verdict marker (`##[error]`, test summary, Traceback, exit code) AND ends mid-execution. Both conditions required — "no marker" alone over-fires on a real failure whose tool output isn't in the marker set (e.g. `ruff format`). Routed to a bounded rerun, same as `infra_broken`/`stale`, via `ci_status_poll` and `ci_gate`. Also fixes an infra-signature regression exposed by the full-log change: the bare step-name signatures (`git fetch`, `Set up job`, `actions/checkout`) matched the checkout/setup preamble of EVERY job log — against a full untruncated log they turned every failing run into `infra_broken`. Replaced with genuine error-text signatures (`could not read from remote`, `download action repository failed`, `unable to access`, `failed to connect`). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
547 lines
20 KiB
Python
547 lines
20 KiB
Python
"""CI-freshness gate — the early DISCOVERED tick.
|
|
|
|
THE INCIDENT this addresses
|
|
---------------------------
|
|
A PR was discovered with 2-day-stale CI where every job had failed at
|
|
the *checkout* step (a ``git fetch`` connection-reset — pure infra, no
|
|
code ever ran). The pipeline had no notion of CI freshness: the
|
|
workflow went DISCOVERED → ANALYZING → IMPLEMENTING → STUCK, burning
|
|
an estimator + a tier-2 implementer attempt before the implementer
|
|
correctly said "nothing to fix, rerun CI" and dead-ended. The
|
|
controller never *triggers* CI — it only polls.
|
|
|
|
This tick is the fix. It runs BEFORE the normal DISCOVERED → ANALYZING
|
|
promotion (``promote.py``). For each ``kind='pr'`` workflow in
|
|
``DISCOVERED``:
|
|
|
|
1. Fetch the PR's head SHA + the CI combined status.
|
|
2. When the combined status shows a failure, fetch the failing jobs'
|
|
CI **log content** via the injected ``get_failure_logs`` callback
|
|
(backed by ``_ci_logs``). The infra-vs-real decision needs the
|
|
actual log text — Forgejo status descriptions are generic duration
|
|
strings and never carry the error.
|
|
3. Classify via ``ci_freshness.classify_ci_result``:
|
|
- ``fresh_real`` / ``pending`` → leave it alone; ``promote.py``
|
|
handles the normal path.
|
|
- ``infra_broken`` / ``stale`` / ``no_ci`` / ``indeterminate`` →
|
|
trigger a CI rerun (empty-commit push). ``indeterminate`` is a
|
|
failing run whose full log carries no verdict marker — a
|
|
hard-kill / OOM that never produced a result. On success: stash
|
|
the new head SHA, fire
|
|
``discovery_ci_rerun_triggered`` (DISCOVERED → AWAITING_CI) via a
|
|
TOCTOU-guarded UPDATE, and write a ``ci-rerun-triggered`` event.
|
|
3. RERUN BUDGET = 3: before triggering, count prior
|
|
``ci-rerun-triggered`` events for the workflow. At >= 3, do NOT
|
|
rerun again — route the workflow to STUCK (the CI runner itself is
|
|
broken; an operator must investigate).
|
|
|
|
Both this tick and ``promote.py`` filter ``current_state='DISCOVERED'``
|
|
with guarded UPDATEs, so they're race-safe even out of order — but the
|
|
loop orders this one FIRST so a stale-CI workflow is diverted before
|
|
promotion ever fires.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.engine import Engine
|
|
|
|
from ..db.session import session_scope
|
|
from ..state_machine import IllegalTransitionError, apply_event
|
|
from .ci_freshness import (
|
|
GetFailureLogsCallback,
|
|
classify_ci_result,
|
|
combined_status_is_failure,
|
|
)
|
|
from .ci_rerun import CIRerunCallback, CIRerunResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Maximum number of automatic CI reruns per workflow before the gate
|
|
# gives up and routes the workflow to STUCK for operator attention.
|
|
RERUN_BUDGET = 3
|
|
|
|
# controller_events.event_type written when a rerun is triggered. The
|
|
# rerun-budget counter scans for this exact value, and ci_status_poll
|
|
# reads the latest such row's payload to recover the gate-pushed SHA.
|
|
CI_RERUN_EVENT_TYPE = "ci-rerun-triggered"
|
|
CI_RERUN_BUDGET_EXHAUSTED_EVENT_TYPE = "ci-rerun-budget-exhausted"
|
|
|
|
|
|
# get_ci_status(owner, repo, head_sha) -> Forgejo combined-status dict
|
|
# or None on fetch failure.
|
|
GetCIStatusCallback = Callable[[str, str, str], dict | None]
|
|
# get_pr_details(owner, repo, pr_number) -> Forgejo PR dict or None.
|
|
GetPRDetailsCallback = Callable[[str, str, int], dict | None]
|
|
|
|
|
|
@dataclass
|
|
class CIGateReport:
|
|
"""Per-sweep summary of the CI-freshness gate."""
|
|
|
|
workflows_scanned: int = 0
|
|
# fresh / pending — left for the normal promote path.
|
|
workflows_left_for_promote: int = 0
|
|
# rerun triggered + routed DISCOVERED → AWAITING_CI.
|
|
workflows_reran: int = 0
|
|
# rerun budget hit — routed DISCOVERED → STUCK.
|
|
workflows_stuck_budget: int = 0
|
|
# rerun attempt failed (clone/push error) — left in DISCOVERED.
|
|
workflows_rerun_failed: int = 0
|
|
# could not fetch PR details / head SHA — left in DISCOVERED.
|
|
workflows_fetch_failed: int = 0
|
|
# (workflow_id, verdict, action) for observability/tests.
|
|
actions: list[tuple[int, str, str]] = field(default_factory=list)
|
|
|
|
|
|
def _pr_head(pr: dict | None) -> tuple[str | None, str | None]:
|
|
"""Extract (head_sha, head_ref) from a Forgejo PR dict."""
|
|
if not isinstance(pr, dict):
|
|
return None, None
|
|
head = pr.get("head")
|
|
if not isinstance(head, dict):
|
|
return None, None
|
|
sha = head.get("sha")
|
|
ref = head.get("ref")
|
|
return (
|
|
sha if isinstance(sha, str) and sha else None,
|
|
ref if isinstance(ref, str) and ref else None,
|
|
)
|
|
|
|
|
|
def _count_prior_reruns(session, workflow_id: int) -> int:
|
|
"""Count the ``ci-rerun-triggered`` events already recorded for a
|
|
workflow — the rerun-budget counter."""
|
|
row = session.execute(
|
|
text(
|
|
"SELECT COUNT(*) AS n FROM controller_events "
|
|
"WHERE workflow_id = :wf_id AND event_type = :etype"
|
|
),
|
|
{"wf_id": workflow_id, "etype": CI_RERUN_EVENT_TYPE},
|
|
).first()
|
|
return int(row.n) if row is not None else 0
|
|
|
|
|
|
def _guarded_transition(
|
|
session,
|
|
*,
|
|
workflow_id: int,
|
|
event: str,
|
|
now: datetime,
|
|
) -> str | None:
|
|
"""Apply ``event`` from DISCOVERED via a TOCTOU-guarded UPDATE.
|
|
|
|
Returns the new state on success, or None when the row is no
|
|
longer DISCOVERED (race lost) / the event is illegal.
|
|
"""
|
|
try:
|
|
new_state = apply_event("DISCOVERED", event)
|
|
except (IllegalTransitionError, ValueError) as exc:
|
|
logger.warning(
|
|
"ci_gate: apply_event(DISCOVERED, %s) rejected for "
|
|
"workflow %s (%s); skipping",
|
|
event,
|
|
workflow_id,
|
|
exc,
|
|
)
|
|
return None
|
|
result = session.execute(
|
|
text(
|
|
"UPDATE workflows SET "
|
|
" current_state = :to_state, "
|
|
" last_transition_at = :now, "
|
|
" entered_state_at = :now "
|
|
"WHERE workflow_id = :wf_id "
|
|
" AND current_state = 'DISCOVERED'"
|
|
),
|
|
{"to_state": new_state, "now": now, "wf_id": workflow_id},
|
|
)
|
|
if (result.rowcount or 0) == 0:
|
|
logger.info(
|
|
"ci_gate: workflow %s no longer DISCOVERED (race lost); skipping %s",
|
|
workflow_id,
|
|
event,
|
|
)
|
|
return None
|
|
return new_state
|
|
|
|
|
|
def _write_event(
|
|
session,
|
|
*,
|
|
workflow_id: int,
|
|
event_type: str,
|
|
from_state: str,
|
|
to_state: str,
|
|
now: datetime,
|
|
payload: dict,
|
|
) -> None:
|
|
session.execute(
|
|
text(
|
|
"INSERT INTO controller_events "
|
|
"(workflow_id, ts, event_type, from_state, to_state, "
|
|
" payload, forgejo_write_pending, replay_attempts) "
|
|
"VALUES (:wf_id, :ts, :etype, :from_state, :to_state, "
|
|
" :payload, 0, 0)"
|
|
),
|
|
{
|
|
"wf_id": workflow_id,
|
|
"ts": now,
|
|
"etype": event_type,
|
|
"from_state": from_state,
|
|
"to_state": to_state,
|
|
"payload": json.dumps(payload),
|
|
},
|
|
)
|
|
|
|
|
|
def _fetch_failure_logs(
|
|
get_failure_logs: GetFailureLogsCallback | None,
|
|
*,
|
|
owner: str,
|
|
repo: str,
|
|
head_sha: str,
|
|
wf_id: int,
|
|
) -> str:
|
|
"""Best-effort fetch of the failing jobs' CI log content.
|
|
|
|
Returns the concatenated log text, or an empty string when no
|
|
fetcher is wired / the fetch fails. Never raises — a log-fetch
|
|
failure must not abort the gate; the classifier degrades an
|
|
un-loggable failure to ``fresh_real`` (conservative).
|
|
"""
|
|
if get_failure_logs is None:
|
|
return ""
|
|
try:
|
|
text_out = get_failure_logs(owner, repo, head_sha)
|
|
except Exception as exc: # noqa: BLE001 — never abort the tick
|
|
logger.warning(
|
|
"ci_gate: get_failure_logs raised for workflow %s (head=%s): %s",
|
|
wf_id,
|
|
head_sha[:12],
|
|
exc,
|
|
)
|
|
return ""
|
|
return text_out if isinstance(text_out, str) else ""
|
|
|
|
|
|
def run_ci_gate_tick(
|
|
engine: Engine,
|
|
*,
|
|
owner: str,
|
|
repo: str,
|
|
get_ci_status: GetCIStatusCallback,
|
|
get_pr_details: GetPRDetailsCallback,
|
|
trigger_ci_rerun: CIRerunCallback,
|
|
get_failure_logs: GetFailureLogsCallback | None = None,
|
|
rerun_budget: int = RERUN_BUDGET,
|
|
) -> CIGateReport:
|
|
"""One sweep: divert stale/infra-broken DISCOVERED PRs.
|
|
|
|
For each ``kind='pr'`` workflow in ``DISCOVERED`` for this
|
|
(owner, repo): classify CI freshness and either leave it for the
|
|
normal promote path, trigger a CI rerun (→ AWAITING_CI), or — once
|
|
the rerun budget is exhausted — route it to STUCK.
|
|
|
|
``get_failure_logs`` is the CI job-log fetcher (backed by
|
|
``_ci_logs`` in production). The gate calls it only when the
|
|
combined status shows a failure, and feeds the result to
|
|
``classify_ci_result`` so the infra-vs-real decision runs against
|
|
real log content rather than the generic status descriptions.
|
|
When it is not wired (older callers / a transient failure), the
|
|
classifier falls back to ``fresh_real`` for a failure it cannot
|
|
confirm is infra — conservative by design.
|
|
|
|
All callbacks are injected so tests run without a live Forgejo /
|
|
git. Callback exceptions are caught and treated as "fetch failed"
|
|
(the workflow is left in DISCOVERED for the next tick / retry).
|
|
"""
|
|
report = CIGateReport()
|
|
now = datetime.now(timezone.utc)
|
|
|
|
with session_scope(engine) as session:
|
|
rows = session.execute(
|
|
text(
|
|
"SELECT workflow_id, current_state, entity_number "
|
|
" FROM workflows "
|
|
" WHERE current_state = 'DISCOVERED' "
|
|
" AND kind = 'pr' "
|
|
" AND owner = :owner AND repo = :repo"
|
|
),
|
|
{"owner": owner, "repo": repo},
|
|
).all()
|
|
report.workflows_scanned = len(rows)
|
|
|
|
for row in rows:
|
|
wf_id = row.workflow_id
|
|
pr_number = row.entity_number
|
|
|
|
# 1. Fetch PR head SHA + branch.
|
|
try:
|
|
pr = get_pr_details(owner, repo, pr_number)
|
|
except Exception as exc: # noqa: BLE001 — transient
|
|
logger.warning(
|
|
"ci_gate: get_pr_details failed for workflow %s (PR #%s): %s",
|
|
wf_id,
|
|
pr_number,
|
|
exc,
|
|
)
|
|
report.workflows_fetch_failed += 1
|
|
report.actions.append((wf_id, "unknown", "fetch-failed"))
|
|
continue
|
|
head_sha, head_ref = _pr_head(pr)
|
|
if not head_sha:
|
|
logger.warning(
|
|
"ci_gate: workflow %s (PR #%s) has no head SHA; "
|
|
"leaving in DISCOVERED",
|
|
wf_id,
|
|
pr_number,
|
|
)
|
|
report.workflows_fetch_failed += 1
|
|
report.actions.append((wf_id, "unknown", "fetch-failed"))
|
|
continue
|
|
|
|
# 2. Fetch the CI combined status for the head SHA.
|
|
try:
|
|
ci_status = get_ci_status(owner, repo, head_sha)
|
|
except Exception as exc: # noqa: BLE001 — transient
|
|
logger.warning(
|
|
"ci_gate: get_ci_status failed for workflow %s (head=%s): %s",
|
|
wf_id,
|
|
head_sha[:12],
|
|
exc,
|
|
)
|
|
# A fetch failure is itself "no CI verdict available".
|
|
# Be conservative: treat as fetch-failed (retry next
|
|
# tick) rather than forcing a rerun on transient noise.
|
|
report.workflows_fetch_failed += 1
|
|
report.actions.append((wf_id, "unknown", "fetch-failed"))
|
|
continue
|
|
|
|
# Fetch the failing jobs' CI log content — only when the
|
|
# combined status actually shows a failure (a green/pending
|
|
# PR has nothing to scan). The classifier scans this text
|
|
# for infra signatures; without it an infra failure would
|
|
# be misclassified as fresh_real (the incident's bug).
|
|
failure_logs = ""
|
|
if combined_status_is_failure(ci_status):
|
|
failure_logs = _fetch_failure_logs(
|
|
get_failure_logs,
|
|
owner=owner,
|
|
repo=repo,
|
|
head_sha=head_sha,
|
|
wf_id=wf_id,
|
|
)
|
|
|
|
classification = classify_ci_result(
|
|
head_sha=head_sha,
|
|
forgejo_status=ci_status,
|
|
failure_logs=failure_logs,
|
|
)
|
|
verdict = classification.verdict
|
|
|
|
# fresh_real / pending → leave for promote.py.
|
|
if verdict in {"fresh_real", "pending"}:
|
|
report.workflows_left_for_promote += 1
|
|
report.actions.append((wf_id, verdict, "left-for-promote"))
|
|
continue
|
|
|
|
# infra_broken / stale / no_ci / indeterminate → CI needs
|
|
# re-triggering.
|
|
prior_reruns = _count_prior_reruns(session, wf_id)
|
|
|
|
# 3. Rerun budget gate. DISCOVERED has no state-machine
|
|
# event mapping straight to STUCK (a broken CI runner is a
|
|
# runner-health condition, not a workflow-logic
|
|
# transition), so route via a guarded raw UPDATE.
|
|
if prior_reruns >= rerun_budget:
|
|
new_state = _force_stuck(
|
|
session,
|
|
workflow_id=wf_id,
|
|
now=now,
|
|
)
|
|
if new_state is None:
|
|
# Race lost — another tick moved the row.
|
|
report.actions.append((wf_id, verdict, "budget-race-lost"))
|
|
continue
|
|
_write_event(
|
|
session,
|
|
workflow_id=wf_id,
|
|
event_type=CI_RERUN_BUDGET_EXHAUSTED_EVENT_TYPE,
|
|
from_state="DISCOVERED",
|
|
to_state=new_state,
|
|
now=now,
|
|
payload={
|
|
"reason": (
|
|
"CI infra still failing after "
|
|
f"{prior_reruns} reruns — operator must "
|
|
"investigate the runner"
|
|
),
|
|
"verdict": verdict,
|
|
"classification_reason": classification.reason,
|
|
"prior_reruns": prior_reruns,
|
|
"rerun_budget": rerun_budget,
|
|
"head_sha": head_sha,
|
|
"matched_signatures": classification.matched_signatures,
|
|
"source": "ci_gate",
|
|
},
|
|
)
|
|
report.workflows_stuck_budget += 1
|
|
report.actions.append((wf_id, verdict, "stuck-budget"))
|
|
logger.warning(
|
|
"ci_gate: workflow %s (PR #%s) STUCK — CI infra "
|
|
"still failing after %d reruns",
|
|
wf_id,
|
|
pr_number,
|
|
prior_reruns,
|
|
)
|
|
continue
|
|
|
|
# 4. Trigger a CI rerun (empty-commit push).
|
|
if not head_ref:
|
|
logger.warning(
|
|
"ci_gate: workflow %s (PR #%s) has no head branch "
|
|
"ref; cannot push an empty commit",
|
|
wf_id,
|
|
pr_number,
|
|
)
|
|
report.workflows_rerun_failed += 1
|
|
report.actions.append((wf_id, verdict, "rerun-failed"))
|
|
continue
|
|
try:
|
|
rerun: CIRerunResult = trigger_ci_rerun(
|
|
owner,
|
|
repo,
|
|
head_ref,
|
|
)
|
|
except Exception as exc: # noqa: BLE001 — never abort tick
|
|
logger.warning(
|
|
"ci_gate: trigger_ci_rerun raised for workflow %s (PR #%s): %s",
|
|
wf_id,
|
|
pr_number,
|
|
exc,
|
|
)
|
|
report.workflows_rerun_failed += 1
|
|
report.actions.append((wf_id, verdict, "rerun-failed"))
|
|
continue
|
|
|
|
if not rerun.ok or not rerun.new_head_sha:
|
|
logger.warning(
|
|
"ci_gate: CI rerun failed for workflow %s (PR #%s): %s",
|
|
wf_id,
|
|
pr_number,
|
|
rerun.error,
|
|
)
|
|
report.workflows_rerun_failed += 1
|
|
report.actions.append((wf_id, verdict, "rerun-failed"))
|
|
continue
|
|
|
|
# Rerun succeeded — route DISCOVERED → AWAITING_CI.
|
|
new_state = _guarded_transition(
|
|
session,
|
|
workflow_id=wf_id,
|
|
event="discovery_ci_rerun_triggered",
|
|
now=now,
|
|
)
|
|
if new_state is None:
|
|
report.actions.append((wf_id, verdict, "rerun-race-lost"))
|
|
continue
|
|
_write_event(
|
|
session,
|
|
workflow_id=wf_id,
|
|
event_type=CI_RERUN_EVENT_TYPE,
|
|
from_state="DISCOVERED",
|
|
to_state=new_state,
|
|
now=now,
|
|
payload={
|
|
"reason": "discovery_ci_rerun_triggered",
|
|
"verdict": verdict,
|
|
"classification_reason": classification.reason,
|
|
"matched_signatures": classification.matched_signatures,
|
|
"failing_contexts": classification.failing_contexts,
|
|
# The OLD head SHA the classifier saw + the NEW one
|
|
# the empty-commit push created. ci_status_poll
|
|
# reads ``new_head_sha`` to know which commit's CI
|
|
# to poll (the workflow has no implementer attempt).
|
|
"stale_head_sha": head_sha,
|
|
"new_head_sha": rerun.new_head_sha,
|
|
"head_ref": head_ref,
|
|
"rerun_number": prior_reruns + 1,
|
|
"source": "ci_gate",
|
|
},
|
|
)
|
|
report.workflows_reran += 1
|
|
report.actions.append((wf_id, verdict, "rerun-triggered"))
|
|
logger.info(
|
|
"ci_gate: workflow %s (PR #%s) — %s CI; triggered "
|
|
"rerun #%d (new HEAD %s), routed to AWAITING_CI",
|
|
wf_id,
|
|
pr_number,
|
|
verdict,
|
|
prior_reruns + 1,
|
|
rerun.new_head_sha[:12],
|
|
)
|
|
|
|
if report.workflows_reran or report.workflows_stuck_budget:
|
|
logger.info(
|
|
"ci_gate: %d scanned (reran=%d, stuck-budget=%d, "
|
|
"left-for-promote=%d, rerun-failed=%d, fetch-failed=%d)",
|
|
report.workflows_scanned,
|
|
report.workflows_reran,
|
|
report.workflows_stuck_budget,
|
|
report.workflows_left_for_promote,
|
|
report.workflows_rerun_failed,
|
|
report.workflows_fetch_failed,
|
|
)
|
|
return report
|
|
|
|
|
|
def _force_stuck(
|
|
session,
|
|
*,
|
|
workflow_id: int,
|
|
now: datetime,
|
|
) -> str | None:
|
|
"""Route a DISCOVERED workflow directly to STUCK via a guarded
|
|
UPDATE.
|
|
|
|
DISCOVERED has no state-machine event mapping straight to STUCK
|
|
(the budget-exhausted case is an operator-attention condition the
|
|
transition table doesn't model — it's a runner-health problem,
|
|
not a workflow-logic transition). We therefore do a guarded raw
|
|
UPDATE, exactly the TOCTOU pattern ``promote.py`` uses, and pair
|
|
it with an audit event so the trail stays intact.
|
|
"""
|
|
result = session.execute(
|
|
text(
|
|
"UPDATE workflows SET "
|
|
" current_state = 'STUCK', "
|
|
" last_transition_at = :now, "
|
|
" entered_state_at = :now "
|
|
"WHERE workflow_id = :wf_id "
|
|
" AND current_state = 'DISCOVERED'"
|
|
),
|
|
{"now": now, "wf_id": workflow_id},
|
|
)
|
|
if (result.rowcount or 0) == 0:
|
|
return None
|
|
return "STUCK"
|
|
|
|
|
|
__all__ = [
|
|
"CI_RERUN_BUDGET_EXHAUSTED_EVENT_TYPE",
|
|
"CI_RERUN_EVENT_TYPE",
|
|
"CIGateReport",
|
|
"GetCIStatusCallback",
|
|
"GetPRDetailsCallback",
|
|
"RERUN_BUDGET",
|
|
"run_ci_gate_tick",
|
|
]
|