Files
cleveragents-core/tools/controller/master/tick.py
T
drew 8c9a0c50bb fix(controller): green-CI noop routes forward instead of ABANDONED
An implementer dispatched against a fully-green CI that emits `noop`
("nothing to fix") was mapped to competence-failure -> ESCALATING ->
ABANDONED at MAX_TIER. PR-39 and PR-40 dead-ended exactly this way:
12/12 CI gates green, workflow ABANDONED, solely because the
implementer said `noop` instead of the synonymous `verified-clean`.

The tick now reads the attempt's input_payload ci_summary; a `noop`
whose attempt saw an unambiguously green CI (overall success, zero
failed, zero pending, >=1 passed) routes via `implementer_verified`
-> AWAITING_CI (-> ci_green -> REVIEWING) -- the same forward path
`verified-clean` already takes. A non-green `noop` (red / pending /
unknown / no CI) keeps the competence-failure -> escalate behavior.

- outcomes.py: `attempt_saw_green_ci` param gates the noop branch.
- tick.py: `_input_ci_summary` + `_ci_summary_is_green` helpers.
- tests: green->AWAITING_CI, red/none->ESCALATING, helper unit tests.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-21 02:44:57 -04:00

818 lines
29 KiB
Python

"""Master tick handler — advances state machine for completed attempts.
One tick:
1. Query workflow_attempts where status='complete' AND workflow has
NOT yet processed this attempt (we track via a "consumed_at"
column? No — for v1 we use a simpler heuristic: an attempt is
"unprocessed" if its workflow_attempts.finished_at is more recent
than the workflow's last_transition_at).
2. For each, look up the workflow + use ``map_outcome_to_event`` to
pick a state machine event.
3. Apply via ``apply_event``; update workflow.current_state;
insert controller_events row.
What's deliberately deferred to Phase 1d-3:
- Discovery (Forgejo poll for new PRs/issues)
- Forgejo writes (status comments, labels, merges)
- Per-workflow "what to enqueue next" logic (after a state
transition, the master schedules the next attempt; for v1
this commit only handles the transition itself)
- Reconciliation tick (DB↔Forgejo sync)
- MERGING state's Forgejo merge call
This tick is composable: the master's main loop will call this tick,
then the reaper, then the pickup guard, then sleep + repeat.
"""
from __future__ import annotations
import json
import logging
import re
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
from sqlalchemy import text
from sqlalchemy.engine import Engine
from ..db.session import session_scope
from ..state_machine import (
IllegalTransitionError,
KNOWN_STATES,
TERMINAL_STATES,
apply_event,
)
from .outcomes import EventMapResult, map_outcome_to_event
logger = logging.getLogger(__name__)
@dataclass
class TickReport:
"""Per-tick summary."""
attempts_processed: int = 0
transitions_applied: int = 0
unmapped_attempts: list[tuple[int, str]] = field(default_factory=list)
# (attempt_id, reason) — operator-visible via tail-events
transitions_log: list[dict] = field(default_factory=list)
# Each entry: {workflow_id, from_state, to_state, event, attempt_id}
def run_tick(engine: Engine) -> TickReport:
"""One master tick: advance state for any complete attempts not
yet processed."""
report = TickReport()
now = datetime.now(timezone.utc)
with session_scope(engine) as session:
rows = session.execute(
text(
"SELECT a.attempt_id, a.workflow_id, a.role, a.tier, "
" a.output_payload, a.input_payload, a.status, "
" a.head_sha_before, a.head_sha_after, "
" a.pickup_count, "
" w.current_state, w.current_tier, w.last_transition_at "
" FROM workflow_attempts a "
" JOIN workflows w ON w.workflow_id = a.workflow_id "
" WHERE a.status IN ('complete', 'failed') "
" AND a.finished_at IS NOT NULL "
" AND a.finished_at > w.last_transition_at "
" AND w.current_state NOT IN ('MERGED', 'ABANDONED', 'STUCK', 'CREATED_PR') "
" ORDER BY a.finished_at ASC"
)
).all()
for r in rows:
report.attempts_processed += 1
transitioned = _process_attempt(session, r, now)
if transitioned is None:
# No mappable event; log + skip.
continue
report.transitions_applied += 1
report.transitions_log.append(transitioned)
return report
# ─── per-row processing ───────────────────────────────────────────────
def _process_attempt(session, row, now: datetime) -> dict | None:
"""Apply one attempt's outcome to its workflow's state machine.
Returns the transition record on success, or None if no
transition was applied (unmapped outcome, illegal event, etc.).
"""
# 1. Validate the workflow's current state.
if row.current_state not in KNOWN_STATES:
logger.error(
"workflow %s has unknown current_state %r; transitioning to STUCK",
row.workflow_id,
row.current_state,
)
_transition_to_stuck(
session,
row.workflow_id,
row.current_state,
now,
reason="unknown-state",
attempt_id=row.attempt_id,
)
return {
"workflow_id": row.workflow_id,
"from_state": row.current_state,
"to_state": "STUCK",
"event": "(synthetic) unknown-state",
"attempt_id": row.attempt_id,
}
# 2. Decode output_payload + compute auxiliary context.
try:
output_payload = _decode_output_payload(row.output_payload)
except CorruptedOutputPayload as exc:
logger.error(
"tick: workflow %s attempt_id=%s has corrupted output_payload "
"(%s); routing to STUCK",
row.workflow_id,
row.attempt_id,
exc,
)
_transition_to_stuck(
session,
row.workflow_id,
row.current_state,
now,
reason="corrupted-output",
attempt_id=row.attempt_id,
)
return {
"workflow_id": row.workflow_id,
"from_state": row.current_state,
"to_state": "STUCK",
"event": "(synthetic) corrupted-output",
"attempt_id": row.attempt_id,
}
head_sha_advanced = (
bool(row.head_sha_after)
and bool(row.head_sha_before)
and row.head_sha_after != row.head_sha_before
)
# 3. Map outcome → event.
# attempts_remaining_at_tier + conflict_count_at_current_tier:
# for v1 we use a simple computation (count of attempts at this
# role+tier vs max_attempts on the workflow). The full policy
# lives in Phase 1d-3; v1 ships with a "≥1 remaining always"
# default which keeps the state machine in a simple loop until
# max_attempts kicks in via the pickup guard.
attempts_remaining_at_tier = 1
conflict_count_at_current_tier = _count_conflict_resolver_attempts(
session,
row.workflow_id,
row.current_tier,
)
# E-1: count PRIOR (not including this attempt) contract-violation
# outcomes for this workflow+role. _map_failed_outcome uses it to
# gate the retry-once policy (spec promise; pre-fix the 1st
# violation went straight to STUCK).
prior_contract_violations = _count_prior_contract_violations(
session,
row.workflow_id,
row.role,
row.attempt_id,
)
# T5-9: per-tier dispute cap (1 dispute per tier; subsequent
# dispute outcomes at the same tier downgrade to competence-failure).
prior_disputes_at_current_tier = _count_prior_disputes_at_tier(
session,
row.workflow_id,
row.tier,
row.attempt_id,
)
# A: worker-internal-error count at this tier — escalate-on-repeated-
# timeout rule (map_outcome_to_event) reads this so a too-big-for-
# tier PR escalates instead of retrying the same tier to STUCK.
prior_worker_errors_at_tier = _count_prior_worker_errors_at_tier(
session,
row.workflow_id,
row.tier,
row.attempt_id,
)
# ci-not-ready backstop count (per-workflow): map_outcome_to_event
# caps ci-not-ready so it cannot ping-pong with ci_red_retry forever.
prior_ci_not_ready = _count_prior_ci_not_ready(
session,
row.workflow_id,
row.attempt_id,
)
# ci-infra-failure backstop count (per-workflow): caps the
# IMPLEMENTING → DISCOVERED → CI-freshness-gate loop in case the
# implementer keeps mis-classifying a real failure as infra.
prior_ci_infra_failure = _count_prior_ci_infra_failure(
session,
row.workflow_id,
row.attempt_id,
)
# Estimator worker-internal-error cap — bounds the unbounded
# estimator re-enqueue loop (run-2 saw 174x on one workflow).
prior_estimator_worker_errors = _count_prior_estimator_worker_errors(
session,
row.workflow_id,
row.attempt_id,
)
# Green-CI noop guard: was THIS attempt dispatched against a
# fully-green CI? An implementer ``noop`` against a green CI is
# "nothing to fix" reported correctly — it must route the workflow
# forward (AWAITING_CI), not escalate it to ABANDONED.
attempt_saw_green_ci = _ci_summary_is_green(_input_ci_summary(row.input_payload))
mapped: EventMapResult = map_outcome_to_event(
role=row.role,
current_state=row.current_state,
output_payload=output_payload,
status=row.status,
head_sha_advanced=head_sha_advanced,
attempts_remaining_at_tier=attempts_remaining_at_tier,
conflict_count_at_current_tier=conflict_count_at_current_tier,
prior_contract_violations=prior_contract_violations,
attempt_tier=row.tier,
workflow_current_tier=row.current_tier,
prior_disputes_at_current_tier=prior_disputes_at_current_tier,
prior_worker_errors_at_tier=prior_worker_errors_at_tier,
prior_ci_not_ready=prior_ci_not_ready,
prior_ci_infra_failure=prior_ci_infra_failure,
prior_estimator_worker_errors=prior_estimator_worker_errors,
attempt_saw_green_ci=attempt_saw_green_ci,
)
if mapped.event_name is None:
# Bump the workflow's last_transition_at so we don't re-process
# the same attempt every tick.
session.execute(
text(
"UPDATE workflows SET last_transition_at = :now "
"WHERE workflow_id = :wf_id"
),
{"now": now, "wf_id": row.workflow_id},
)
logger.debug(
"attempt_id=%s no mappable event: %s",
row.attempt_id,
mapped.reason,
)
return None
# 4. Apply the event.
try:
to_state = apply_event(row.current_state, mapped.event_name)
except IllegalTransitionError as exc:
# E-5 fix (2026-05-19): not all illegal transitions are
# corruption — many are "stale outcome": the workflow advanced
# past this role's state (e.g., via ci_status_poll or
# reconciliation) between when this attempt was scheduled and
# when its outcome is being processed. In that case the
# outcome is OK-but-late; just consume it without STUCK.
#
# A stale outcome is detected by checking: is the event one
# this role would fire from a DIFFERENT state? (e.g.,
# estimator_done fires from ANALYZING; if workflow is now in
# IMPLEMENTING/REVIEWING/MERGING/MERGED, the estimator's
# outcome arrived after the workflow already advanced.)
if _is_stale_role_outcome(row.role, row.current_state):
logger.info(
"tick: workflow %s outcome from role=%r is stale "
"(workflow is now in %s, past this role's state); "
"consuming attempt without transition. event=%s",
row.workflow_id,
row.role,
row.current_state,
mapped.event_name,
)
session.execute(
text(
"UPDATE workflows SET last_transition_at = :now "
"WHERE workflow_id = :wf_id"
),
{"now": now, "wf_id": row.workflow_id},
)
return None
logger.warning(
"illegal transition for workflow %s: %s (event=%s); transitioning to STUCK",
row.workflow_id,
exc,
mapped.event_name,
)
_transition_to_stuck(
session,
row.workflow_id,
row.current_state,
now,
reason=f"illegal-event: {mapped.event_name}",
attempt_id=row.attempt_id,
)
return {
"workflow_id": row.workflow_id,
"from_state": row.current_state,
"to_state": "STUCK",
"event": mapped.event_name,
"attempt_id": row.attempt_id,
}
# 5. Compute per-event side-effects on workflow columns beyond
# ``current_state``. Three v1 fields ride along with specific
# transitions:
#
# - ``current_tier`` ← ``recommended_tier`` from the estimator's
# output, on either ``estimator_done`` or
# ``estimator_metadata_only``. Without this, downstream
# IMPLEMENTING attempts always run at the workflow's creation-
# time tier (typically 0) regardless of what the estimator
# recommended — making the estimator role purely informational.
# The metadata-only branch ALSO sets tier so that a subsequent
# reviewer→request-changes path lands the implementer at the
# right tier.
#
# - ``tier_last_succeeded`` ← ``current_tier`` on
# ``implementer_pushed`` (the success path). Read by
# ``merging.py`` on post-approval 409 conflicts to route the
# workflow back to IMPLEMENTING at the last-known-good tier.
# Pre-2026-05-19 this column had ZERO writers; the 409 path
# transitioned to ``IMPLEMENTING(tier=NULL)`` which the
# scheduler silently coerced to tier 0.
extra_set: list[str] = []
extra_params: dict[str, Any] = {}
if mapped.event_name in ("estimator_done", "estimator_metadata_only"):
rec_tier = (output_payload or {}).get("recommended_tier")
if isinstance(rec_tier, int) and rec_tier in (0, 1, 2):
# Deterministic diff-size floor. The estimator's structured
# recommended_tier can contradict its own reasoning — run-25
# observed a 387-file PR whose reasoning said "firmly Tier 2"
# emitted as tier 0, so a huge cross-subsystem change ran on
# the weakest model and dead-ended. The controller already
# knows the diff size, so floor the tier deterministically:
# a genuinely large PR can never start below the floor. The
# floor only ever RAISES the tier.
floor = _diff_size_tier_floor(_input_diff_summary(row.input_payload))
effective_tier = max(rec_tier, floor)
if effective_tier != rec_tier:
logger.warning(
"estimator attempt_id=%s recommended_tier=%d but diff "
"size floors to tier %d; starting at tier %d",
row.attempt_id,
rec_tier,
floor,
effective_tier,
)
extra_set.append("current_tier = :rec_tier")
extra_params["rec_tier"] = effective_tier
else:
logger.warning(
"estimator attempt_id=%s emitted invalid recommended_tier=%r; "
"leaving current_tier unchanged",
row.attempt_id,
rec_tier,
)
elif mapped.event_name == "implementer_pushed":
# Latch the successful tier so the merging-409 conflict path
# can recover to the last-known-good tier.
if row.current_tier is not None:
extra_set.append("tier_last_succeeded = :tls")
extra_params["tls"] = row.current_tier
extra_set_clause = ("," + ", ".join(extra_set)) if extra_set else ""
# 5. Commit the transition.
session.execute(
text(
"UPDATE workflows SET "
" current_state = :to_state, "
" last_transition_at = :now, "
" entered_state_at = :now"
f"{extra_set_clause} "
"WHERE workflow_id = :wf_id"
),
{
"to_state": to_state,
"now": now,
"wf_id": row.workflow_id,
**extra_params,
},
)
session.execute(
text(
"INSERT INTO controller_events "
"(workflow_id, ts, event_type, from_state, to_state, "
" attempt_id, payload, forgejo_write_pending, replay_attempts) "
"VALUES (:wf_id, :ts, 'transition', :from_state, :to_state, "
" :aid, :payload, 0, 0)"
),
{
"wf_id": row.workflow_id,
"ts": now,
"from_state": row.current_state,
"to_state": to_state,
"aid": row.attempt_id,
"payload": json.dumps(
{
"event": mapped.event_name,
"reason": mapped.reason,
"role": row.role,
}
),
},
)
return {
"workflow_id": row.workflow_id,
"from_state": row.current_state,
"to_state": to_state,
"event": mapped.event_name,
"attempt_id": row.attempt_id,
}
# ─── helpers ──────────────────────────────────────────────────────────
def _is_stale_role_outcome(role: str, current_state: str) -> bool:
"""True if this role's outcome could only have come from a state
the workflow has already moved past. Used to distinguish
"stale-late outcome that's safe to skip" from "genuine state
corruption that warrants STUCK".
Stale-pattern: estimator only fires from ANALYZING. If the
workflow is in IMPLEMENTING/REVIEWING/MERGING/MERGED/STUCK/
AWAITING_CI/ABANDONED, the estimator outcome arrived too late
(it was scheduled, ran, finished — but in the meantime the
workflow advanced via a parallel path).
"""
role_to_origin_state = {
"estimator": "ANALYZING",
"implementer": "IMPLEMENTING",
"reviewer": "REVIEWING",
"conflict_resolver": "CONFLICT_RESOLVING",
"summarizer": None, # summarizer drives no transition; never stale-illegal
}
origin = role_to_origin_state.get(role)
if origin is None:
return False
return current_state != origin
class CorruptedOutputPayload(Exception):
"""Raised when the raw output_payload column is non-NULL but does
not decode as JSON. Distinguishes "no output" (None → mapper
returns no-op event) from "broken bytes" (workflow should STUCK
with reason='corrupted-output').
"""
def _decode_output_payload(raw) -> dict | None:
"""SQLite returns JSON columns as strings (when bound via text());
Postgres returns dicts. Normalize.
Raises CorruptedOutputPayload on decode failure for str inputs so
the caller can route the workflow to STUCK with a clear reason.
Pre-2026-05-19 this silently swallowed JSONDecodeError → mapper
returned None → tick bumped last_transition_at but never moved
the workflow (E-3: silent stuck-forever with no operator signal).
"""
if raw is None:
return None
if isinstance(raw, str):
try:
return json.loads(raw)
except json.JSONDecodeError as exc:
raise CorruptedOutputPayload(
f"output_payload is not valid JSON: {exc}"
) from exc
return raw
# ─── diff-size tier floor (estimator self-contradiction guard) ────────
# A PR this large cannot be safely implemented at a weak tier
# regardless of the estimator's structured recommended_tier (run-25: a
# 387-file PR's reasoning said "Tier 2" but the emitted field was 0).
# Tuned conservatively — only genuinely big PRs trip a floor, and the
# floor never lowers a tier.
_TIER1_FLOOR_FILES = 40
_TIER1_FLOOR_LINES = 1_500
_TIER2_FLOOR_FILES = 120
_TIER2_FLOOR_LINES = 6_000
def _input_diff_summary(raw) -> str | None:
"""Pull ``diff_summary`` from an attempt's input_payload column.
Tolerates the SQLite-string vs Postgres-dict split + missing keys;
returns None on anything unparseable (→ no floor applied).
"""
if raw is None:
return None
payload: Any = raw
if isinstance(raw, str):
try:
payload = json.loads(raw)
except json.JSONDecodeError:
return None
if not isinstance(payload, dict):
return None
ds = payload.get("diff_summary")
return ds if isinstance(ds, str) else None
def _diff_size_tier_floor(diff_summary: str | None) -> int:
"""Minimum starting tier implied by the PR's diff size.
``diff_summary`` is the controller-built one-liner from
``prefetch._diff_summary_from_pr`` — e.g. ``"387 files, +32579,
-12900"``. Returns 0 (no floor) when the summary is absent or
unparseable, so a parse miss degrades to the estimator's own
recommendation rather than over-tiering.
"""
if not diff_summary:
return 0
files = 0
lines = 0
m = re.search(r"(\d+)\s+files?\b", diff_summary)
if m:
files = int(m.group(1))
for sign in (r"\+", "-"):
sm = re.search(rf"{sign}(\d+)", diff_summary)
if sm:
lines += int(sm.group(1))
if files >= _TIER2_FLOOR_FILES or lines >= _TIER2_FLOOR_LINES:
return 2
if files >= _TIER1_FLOOR_FILES or lines >= _TIER1_FLOOR_LINES:
return 1
return 0
# ─── green-CI noop guard ──────────────────────────────────────────────
def _input_ci_summary(raw) -> dict | None:
"""Pull the ``ci_summary`` dict from an attempt's input_payload.
Mirrors ``_input_diff_summary``: tolerates the SQLite-string vs
Postgres-dict split and missing keys; returns None on anything
unparseable so a parse miss degrades to "no CI seen".
"""
if raw is None:
return None
payload: Any = raw
if isinstance(raw, str):
try:
payload = json.loads(raw)
except json.JSONDecodeError:
return None
if not isinstance(payload, dict):
return None
ci = payload.get("ci_summary")
return ci if isinstance(ci, dict) else None
def _ci_summary_is_green(ci: dict | None) -> bool:
"""True iff a CISummary dict is an unambiguous all-gates-passed run.
Strict on purpose: requires overall success, zero failed gates,
zero pending gates, AND at least one passed gate — an empty or
unknown summary is NOT "green". Feeds the green-CI ``noop`` guard
in ``map_outcome_to_event``: an implementer that emits ``noop``
against a green CI is reporting "nothing to fix" correctly, so the
workflow should move forward rather than escalate to ABANDONED.
"""
if not isinstance(ci, dict):
return False
overall = str(ci.get("overall_state") or "").lower()
failed = ci.get("gates_failed")
pending = ci.get("gates_pending")
passed = ci.get("gates_passed")
return (
overall == "success"
and isinstance(failed, int)
and failed == 0
and isinstance(pending, int)
and pending == 0
and isinstance(passed, int)
and passed > 0
)
def _count_prior_contract_violations(
session,
workflow_id: int,
role: str,
this_attempt_id: int,
) -> int:
"""Count contract-violation outcomes for this workflow+role from
attempts OTHER than this one. Used by E-1 retry-budget gating —
a workflow gets ``_CONTRACT_VIOLATION_RETRY_LIMIT + 1`` total
contract-violation tolerance before STUCKing."""
row = session.execute(
text(
"SELECT COUNT(*) AS n FROM workflow_attempts "
"WHERE workflow_id = :wf_id AND role = :role "
" AND outcome = 'contract-violation' "
" AND attempt_id != :this_id"
),
{"wf_id": workflow_id, "role": role, "this_id": this_attempt_id},
).first()
return int(row.n) if row else 0
def _count_prior_disputes_at_tier(
session,
workflow_id: int,
tier: int | None,
this_attempt_id: int,
) -> int:
"""Count completed implementer attempts at this tier with
outcome=dispute-reviewer, EXCLUDING the attempt being processed.
Used by T5-9 dispute cap: a workflow gets exactly
``_MAX_DISPUTES_PER_TIER`` disputes at each tier before further
dispute outcomes are downgraded to competence-failure (which
escalates the workflow).
"""
if tier is None:
return 0
row = session.execute(
text(
"SELECT COUNT(*) AS n FROM workflow_attempts "
"WHERE workflow_id = :wf_id "
" AND role = 'implementer' "
" AND tier = :tier "
" AND outcome = 'dispute-reviewer' "
" AND attempt_id != :this_id"
),
{"wf_id": workflow_id, "tier": tier, "this_id": this_attempt_id},
).first()
return int(row.n) if row else 0
def _count_prior_worker_errors_at_tier(
session,
workflow_id: int,
tier: int | None,
this_attempt_id: int,
) -> int:
"""Count implementer attempts at this tier that failed with
outcome=worker-internal-error (dominated by OpenCode session
timeouts), EXCLUDING the attempt being processed.
Feeds the A-escalation rule in ``map_outcome_to_event``: repeated
worker-internal-error at a tier escalates the workflow instead of
retrying that tier until pickup-exhaustion → STUCK.
"""
if tier is None:
return 0
row = session.execute(
text(
"SELECT COUNT(*) AS n FROM workflow_attempts "
"WHERE workflow_id = :wf_id "
" AND role = 'implementer' "
" AND tier = :tier "
" AND outcome = 'worker-internal-error' "
" AND attempt_id != :this_id"
),
{"wf_id": workflow_id, "tier": tier, "this_id": this_attempt_id},
).first()
return int(row.n) if row else 0
def _count_prior_ci_not_ready(
session,
workflow_id: int,
this_attempt_id: int,
) -> int:
"""Count implementer attempts on this workflow with
outcome=ci-not-ready, EXCLUDING the attempt being processed.
Feeds the ci-not-ready backstop cap in ``map_outcome_to_event`` —
counted per-workflow (not per-tier) so the cap bounds the whole
ci-not-ready <-> ci_red_retry loop regardless of tier.
"""
row = session.execute(
text(
"SELECT COUNT(*) AS n FROM workflow_attempts "
"WHERE workflow_id = :wf_id "
" AND role = 'implementer' "
" AND outcome = 'ci-not-ready' "
" AND attempt_id != :this_id"
),
{"wf_id": workflow_id, "this_id": this_attempt_id},
).first()
return int(row.n) if row else 0
def _count_prior_ci_infra_failure(
session,
workflow_id: int,
this_attempt_id: int,
) -> int:
"""Count implementer attempts on this workflow with
outcome=ci-infra-failure, EXCLUDING the attempt being processed.
Feeds the ci-infra-failure backstop cap in ``map_outcome_to_event``
— counted per-workflow so the cap bounds the whole IMPLEMENTING →
DISCOVERED → gate loop regardless of tier.
"""
row = session.execute(
text(
"SELECT COUNT(*) AS n FROM workflow_attempts "
"WHERE workflow_id = :wf_id "
" AND role = 'implementer' "
" AND outcome = 'ci-infra-failure' "
" AND attempt_id != :this_id"
),
{"wf_id": workflow_id, "this_id": this_attempt_id},
).first()
return int(row.n) if row else 0
def _count_prior_estimator_worker_errors(
session,
workflow_id: int,
this_attempt_id: int,
) -> int:
"""Count estimator attempts on this workflow that failed with
outcome=worker-internal-error, EXCLUDING the attempt being processed.
Feeds the estimator worker-error cap in ``map_outcome_to_event``: an
estimator that never emits canonical output has no escalation path
and no salvage, so without a cap it re-enqueues forever (run-2
observed 174x on one workflow). Counted per-workflow — the estimator
runs pre-tier, so tier is not a meaningful axis here.
"""
row = session.execute(
text(
"SELECT COUNT(*) AS n FROM workflow_attempts "
"WHERE workflow_id = :wf_id "
" AND role = 'estimator' "
" AND outcome = 'worker-internal-error' "
" AND attempt_id != :this_id"
),
{"wf_id": workflow_id, "this_id": this_attempt_id},
).first()
return int(row.n) if row else 0
def _count_conflict_resolver_attempts(
session,
workflow_id: int,
current_tier: int | None,
) -> int:
"""Count CONFLICT_RESOLVING attempts at the current tier."""
if current_tier is None:
return 0
row = session.execute(
text(
"SELECT COUNT(*) AS n FROM workflow_attempts "
"WHERE workflow_id = :wf_id AND role = 'conflict_resolver' "
" AND tier = :tier"
),
{"wf_id": workflow_id, "tier": current_tier},
).first()
return row.n if row else 0
def _transition_to_stuck(
session,
workflow_id: int,
from_state: str,
now: datetime,
*,
reason: str,
attempt_id: int | None,
) -> None:
session.execute(
text(
"UPDATE workflows SET "
" current_state = 'STUCK', "
" last_transition_at = :now, "
" entered_state_at = :now "
"WHERE workflow_id = :wf_id"
),
{"now": now, "wf_id": workflow_id},
)
session.execute(
text(
"INSERT INTO controller_events "
"(workflow_id, ts, event_type, from_state, to_state, "
" attempt_id, payload, forgejo_write_pending, replay_attempts) "
"VALUES (:wf_id, :ts, 'transition', :from_state, 'STUCK', "
" :aid, :payload, 0, 0)"
),
{
"wf_id": workflow_id,
"ts": now,
"from_state": from_state,
"aid": attempt_id,
"payload": json.dumps({"reason": reason}),
},
)
__all__ = ["TickReport", "run_tick"]