cleveragents-core/tools/controller/master/tick.py

"""Master tick handler — advances state machine for completed attempts.

One tick:
1. Query workflow_attempts where status='complete' AND workflow has
   NOT yet processed this attempt (we track via a "consumed_at"
   column? No — for v1 we use a simpler heuristic: an attempt is
   "unprocessed" if its workflow_attempts.finished_at is more recent
   than the workflow's last_transition_at).
2. For each, look up the workflow + use ``map_outcome_to_event`` to
   pick a state machine event.
3. Apply via ``apply_event``; update workflow.current_state;
   insert controller_events row.

What's deliberately deferred to Phase 1d-3:
- Discovery (Forgejo poll for new PRs/issues)
- Forgejo writes (status comments, labels, merges)
- Per-workflow "what to enqueue next" logic (after a state
  transition, the master schedules the next attempt; for v1
  this commit only handles the transition itself)
- Reconciliation tick (DB↔Forgejo sync)
- MERGING state's Forgejo merge call

This tick is composable: the master's main loop will call this tick,
then the reaper, then the pickup guard, then sleep + repeat.
"""
from __future__ import annotations

import json
import logging
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any

from sqlalchemy import text
from sqlalchemy.engine import Engine

from ..db.session import session_scope
from ..state_machine import (
    IllegalTransitionError,
    KNOWN_STATES,
    TERMINAL_STATES,
    apply_event,
)
from .outcomes import EventMapResult, map_outcome_to_event

logger = logging.getLogger(__name__)


@dataclass
class TickReport:
    """Per-tick summary."""

    attempts_processed: int = 0
    transitions_applied: int = 0
    unmapped_attempts: list[tuple[int, str]] = field(default_factory=list)
    # (attempt_id, reason) — operator-visible via tail-events

    transitions_log: list[dict] = field(default_factory=list)
    # Each entry: {workflow_id, from_state, to_state, event, attempt_id}


def run_tick(engine: Engine) -> TickReport:
    """One master tick: advance state for any complete attempts not
    yet processed."""
    report = TickReport()
    now = datetime.now(timezone.utc)
    with session_scope(engine) as session:
        rows = session.execute(
            text(
                "SELECT a.attempt_id, a.workflow_id, a.role, a.tier, "
                "       a.output_payload, a.status, "
                "       a.head_sha_before, a.head_sha_after, "
                "       a.pickup_count, "
                "       w.current_state, w.current_tier, w.last_transition_at "
                "  FROM workflow_attempts a "
                "  JOIN workflows w ON w.workflow_id = a.workflow_id "
                " WHERE a.status IN ('complete', 'failed') "
                "   AND a.finished_at IS NOT NULL "
                "   AND a.finished_at > w.last_transition_at "
                "   AND w.current_state NOT IN ('MERGED', 'ABANDONED', 'STUCK', 'CREATED_PR') "
                " ORDER BY a.finished_at ASC"
            )
        ).all()

        for r in rows:
            report.attempts_processed += 1
            transitioned = _process_attempt(session, r, now)
            if transitioned is None:
                # No mappable event; log + skip.
                continue
            report.transitions_applied += 1
            report.transitions_log.append(transitioned)

    return report


# ─── per-row processing ───────────────────────────────────────────────


def _process_attempt(session, row, now: datetime) -> dict | None:
    """Apply one attempt's outcome to its workflow's state machine.

    Returns the transition record on success, or None if no
    transition was applied (unmapped outcome, illegal event, etc.).
    """
    # 1. Validate the workflow's current state.
    if row.current_state not in KNOWN_STATES:
        logger.error(
            "workflow %s has unknown current_state %r; transitioning to STUCK",
            row.workflow_id, row.current_state,
        )
        _transition_to_stuck(
            session, row.workflow_id, row.current_state, now,
            reason="unknown-state",
            attempt_id=row.attempt_id,
        )
        return {
            "workflow_id": row.workflow_id,
            "from_state": row.current_state,
            "to_state": "STUCK",
            "event": "(synthetic) unknown-state",
            "attempt_id": row.attempt_id,
        }

    # 2. Decode output_payload + compute auxiliary context.
    try:
        output_payload = _decode_output_payload(row.output_payload)
    except CorruptedOutputPayload as exc:
        logger.error(
            "tick: workflow %s attempt_id=%s has corrupted output_payload "
            "(%s); routing to STUCK",
            row.workflow_id, row.attempt_id, exc,
        )
        _transition_to_stuck(
            session, row.workflow_id, row.current_state, now,
            reason="corrupted-output",
            attempt_id=row.attempt_id,
        )
        return {
            "workflow_id": row.workflow_id,
            "from_state": row.current_state,
            "to_state": "STUCK",
            "event": "(synthetic) corrupted-output",
            "attempt_id": row.attempt_id,
        }
    head_sha_advanced = (
        bool(row.head_sha_after)
        and bool(row.head_sha_before)
        and row.head_sha_after != row.head_sha_before
    )

    # 3. Map outcome → event.
    # attempts_remaining_at_tier + conflict_count_at_current_tier:
    # for v1 we use a simple computation (count of attempts at this
    # role+tier vs max_attempts on the workflow). The full policy
    # lives in Phase 1d-3; v1 ships with a "≥1 remaining always"
    # default which keeps the state machine in a simple loop until
    # max_attempts kicks in via the pickup guard.
    attempts_remaining_at_tier = 1
    conflict_count_at_current_tier = (
        _count_conflict_resolver_attempts(
            session, row.workflow_id, row.current_tier,
        )
    )
    # E-1: count PRIOR (not including this attempt) contract-violation
    # outcomes for this workflow+role. _map_failed_outcome uses it to
    # gate the retry-once policy (spec promise; pre-fix the 1st
    # violation went straight to STUCK).
    prior_contract_violations = _count_prior_contract_violations(
        session, row.workflow_id, row.role, row.attempt_id,
    )
    # T5-9: per-tier dispute cap (1 dispute per tier; subsequent
    # dispute outcomes at the same tier downgrade to competence-failure).
    prior_disputes_at_current_tier = _count_prior_disputes_at_tier(
        session, row.workflow_id, row.tier, row.attempt_id,
    )

    mapped: EventMapResult = map_outcome_to_event(
        role=row.role,
        current_state=row.current_state,
        output_payload=output_payload,
        status=row.status,
        head_sha_advanced=head_sha_advanced,
        attempts_remaining_at_tier=attempts_remaining_at_tier,
        conflict_count_at_current_tier=conflict_count_at_current_tier,
        prior_contract_violations=prior_contract_violations,
        attempt_tier=row.tier,
        workflow_current_tier=row.current_tier,
        prior_disputes_at_current_tier=prior_disputes_at_current_tier,
    )
    if mapped.event_name is None:
        # Bump the workflow's last_transition_at so we don't re-process
        # the same attempt every tick.
        session.execute(
            text(
                "UPDATE workflows SET last_transition_at = :now "
                "WHERE workflow_id = :wf_id"
            ),
            {"now": now, "wf_id": row.workflow_id},
        )
        logger.debug(
            "attempt_id=%s no mappable event: %s",
            row.attempt_id, mapped.reason,
        )
        return None

    # 4. Apply the event.
    try:
        to_state = apply_event(row.current_state, mapped.event_name)
    except IllegalTransitionError as exc:
        # E-5 fix (2026-05-19): not all illegal transitions are
        # corruption — many are "stale outcome": the workflow advanced
        # past this role's state (e.g., via ci_status_poll or
        # reconciliation) between when this attempt was scheduled and
        # when its outcome is being processed. In that case the
        # outcome is OK-but-late; just consume it without STUCK.
        #
        # A stale outcome is detected by checking: is the event one
        # this role would fire from a DIFFERENT state? (e.g.,
        # estimator_done fires from ANALYZING; if workflow is now in
        # IMPLEMENTING/REVIEWING/MERGING/MERGED, the estimator's
        # outcome arrived after the workflow already advanced.)
        if _is_stale_role_outcome(row.role, row.current_state):
            logger.info(
                "tick: workflow %s outcome from role=%r is stale "
                "(workflow is now in %s, past this role's state); "
                "consuming attempt without transition. event=%s",
                row.workflow_id, row.role, row.current_state, mapped.event_name,
            )
            session.execute(
                text(
                    "UPDATE workflows SET last_transition_at = :now "
                    "WHERE workflow_id = :wf_id"
                ),
                {"now": now, "wf_id": row.workflow_id},
            )
            return None
        logger.warning(
            "illegal transition for workflow %s: %s (event=%s); "
            "transitioning to STUCK",
            row.workflow_id, exc, mapped.event_name,
        )
        _transition_to_stuck(
            session, row.workflow_id, row.current_state, now,
            reason=f"illegal-event: {mapped.event_name}",
            attempt_id=row.attempt_id,
        )
        return {
            "workflow_id": row.workflow_id,
            "from_state": row.current_state,
            "to_state": "STUCK",
            "event": mapped.event_name,
            "attempt_id": row.attempt_id,
        }

    # 5. Compute per-event side-effects on workflow columns beyond
    # ``current_state``. Three v1 fields ride along with specific
    # transitions:
    #
    #   - ``current_tier`` ← ``recommended_tier`` from the estimator's
    #     output, on either ``estimator_done`` or
    #     ``estimator_metadata_only``. Without this, downstream
    #     IMPLEMENTING attempts always run at the workflow's creation-
    #     time tier (typically 0) regardless of what the estimator
    #     recommended — making the estimator role purely informational.
    #     The metadata-only branch ALSO sets tier so that a subsequent
    #     reviewer→request-changes path lands the implementer at the
    #     right tier.
    #
    #   - ``tier_last_succeeded`` ← ``current_tier`` on
    #     ``implementer_pushed`` (the success path). Read by
    #     ``merging.py`` on post-approval 409 conflicts to route the
    #     workflow back to IMPLEMENTING at the last-known-good tier.
    #     Pre-2026-05-19 this column had ZERO writers; the 409 path
    #     transitioned to ``IMPLEMENTING(tier=NULL)`` which the
    #     scheduler silently coerced to tier 0.
    extra_set: list[str] = []
    extra_params: dict[str, Any] = {}
    if mapped.event_name in ("estimator_done", "estimator_metadata_only"):
        rec_tier = (output_payload or {}).get("recommended_tier")
        if isinstance(rec_tier, int) and rec_tier in (0, 1, 2):
            extra_set.append("current_tier = :rec_tier")
            extra_params["rec_tier"] = rec_tier
        else:
            logger.warning(
                "estimator attempt_id=%s emitted invalid recommended_tier=%r; "
                "leaving current_tier unchanged",
                row.attempt_id, rec_tier,
            )
    elif mapped.event_name == "implementer_pushed":
        # Latch the successful tier so the merging-409 conflict path
        # can recover to the last-known-good tier.
        if row.current_tier is not None:
            extra_set.append("tier_last_succeeded = :tls")
            extra_params["tls"] = row.current_tier

    extra_set_clause = ("," + ", ".join(extra_set)) if extra_set else ""
    # 5. Commit the transition.
    session.execute(
        text(
            "UPDATE workflows SET "
            "  current_state = :to_state, "
            "  last_transition_at = :now, "
            "  entered_state_at = :now"
            f"{extra_set_clause} "
            "WHERE workflow_id = :wf_id"
        ),
        {
            "to_state": to_state, "now": now, "wf_id": row.workflow_id,
            **extra_params,
        },
    )
    session.execute(
        text(
            "INSERT INTO controller_events "
            "(workflow_id, ts, event_type, from_state, to_state, "
            " attempt_id, payload, forgejo_write_pending, replay_attempts) "
            "VALUES (:wf_id, :ts, 'transition', :from_state, :to_state, "
            "        :aid, :payload, 0, 0)"
        ),
        {
            "wf_id": row.workflow_id,
            "ts": now,
            "from_state": row.current_state,
            "to_state": to_state,
            "aid": row.attempt_id,
            "payload": json.dumps({
                "event": mapped.event_name,
                "reason": mapped.reason,
                "role": row.role,
            }),
        },
    )
    return {
        "workflow_id": row.workflow_id,
        "from_state": row.current_state,
        "to_state": to_state,
        "event": mapped.event_name,
        "attempt_id": row.attempt_id,
    }


# ─── helpers ──────────────────────────────────────────────────────────


def _is_stale_role_outcome(role: str, current_state: str) -> bool:
    """True if this role's outcome could only have come from a state
    the workflow has already moved past. Used to distinguish
    "stale-late outcome that's safe to skip" from "genuine state
    corruption that warrants STUCK".

    Stale-pattern: estimator only fires from ANALYZING. If the
    workflow is in IMPLEMENTING/REVIEWING/MERGING/MERGED/STUCK/
    AWAITING_CI/ABANDONED, the estimator outcome arrived too late
    (it was scheduled, ran, finished — but in the meantime the
    workflow advanced via a parallel path).
    """
    role_to_origin_state = {
        "estimator": "ANALYZING",
        "implementer": "IMPLEMENTING",
        "reviewer": "REVIEWING",
        "conflict_resolver": "CONFLICT_RESOLVING",
        "summarizer": None,  # summarizer drives no transition; never stale-illegal
    }
    origin = role_to_origin_state.get(role)
    if origin is None:
        return False
    return current_state != origin


class CorruptedOutputPayload(Exception):
    """Raised when the raw output_payload column is non-NULL but does
    not decode as JSON. Distinguishes "no output" (None → mapper
    returns no-op event) from "broken bytes" (workflow should STUCK
    with reason='corrupted-output').
    """


def _decode_output_payload(raw) -> dict | None:
    """SQLite returns JSON columns as strings (when bound via text());
    Postgres returns dicts. Normalize.

    Raises CorruptedOutputPayload on decode failure for str inputs so
    the caller can route the workflow to STUCK with a clear reason.
    Pre-2026-05-19 this silently swallowed JSONDecodeError → mapper
    returned None → tick bumped last_transition_at but never moved
    the workflow (E-3: silent stuck-forever with no operator signal).
    """
    if raw is None:
        return None
    if isinstance(raw, str):
        try:
            return json.loads(raw)
        except json.JSONDecodeError as exc:
            raise CorruptedOutputPayload(
                f"output_payload is not valid JSON: {exc}"
            ) from exc
    return raw


def _count_prior_contract_violations(
    session, workflow_id: int, role: str, this_attempt_id: int,
) -> int:
    """Count contract-violation outcomes for this workflow+role from
    attempts OTHER than this one. Used by E-1 retry-budget gating —
    a workflow gets ``_CONTRACT_VIOLATION_RETRY_LIMIT + 1`` total
    contract-violation tolerance before STUCKing."""
    row = session.execute(
        text(
            "SELECT COUNT(*) AS n FROM workflow_attempts "
            "WHERE workflow_id = :wf_id AND role = :role "
            "  AND outcome = 'contract-violation' "
            "  AND attempt_id != :this_id"
        ),
        {"wf_id": workflow_id, "role": role, "this_id": this_attempt_id},
    ).first()
    return int(row.n) if row else 0


def _count_prior_disputes_at_tier(
    session, workflow_id: int, tier: int | None, this_attempt_id: int,
) -> int:
    """Count completed implementer attempts at this tier with
    outcome=dispute-reviewer, EXCLUDING the attempt being processed.

    Used by T5-9 dispute cap: a workflow gets exactly
    ``_MAX_DISPUTES_PER_TIER`` disputes at each tier before further
    dispute outcomes are downgraded to competence-failure (which
    escalates the workflow).
    """
    if tier is None:
        return 0
    row = session.execute(
        text(
            "SELECT COUNT(*) AS n FROM workflow_attempts "
            "WHERE workflow_id = :wf_id "
            "  AND role = 'implementer' "
            "  AND tier = :tier "
            "  AND outcome = 'dispute-reviewer' "
            "  AND attempt_id != :this_id"
        ),
        {"wf_id": workflow_id, "tier": tier, "this_id": this_attempt_id},
    ).first()
    return int(row.n) if row else 0


def _count_conflict_resolver_attempts(
    session, workflow_id: int, current_tier: int | None,
) -> int:
    """Count CONFLICT_RESOLVING attempts at the current tier."""
    if current_tier is None:
        return 0
    row = session.execute(
        text(
            "SELECT COUNT(*) AS n FROM workflow_attempts "
            "WHERE workflow_id = :wf_id AND role = 'conflict_resolver' "
            "  AND tier = :tier"
        ),
        {"wf_id": workflow_id, "tier": current_tier},
    ).first()
    return row.n if row else 0


def _transition_to_stuck(
    session, workflow_id: int, from_state: str, now: datetime, *,
    reason: str, attempt_id: int | None,
) -> None:
    session.execute(
        text(
            "UPDATE workflows SET "
            "  current_state = 'STUCK', "
            "  last_transition_at = :now, "
            "  entered_state_at = :now "
            "WHERE workflow_id = :wf_id"
        ),
        {"now": now, "wf_id": workflow_id},
    )
    session.execute(
        text(
            "INSERT INTO controller_events "
            "(workflow_id, ts, event_type, from_state, to_state, "
            " attempt_id, payload, forgejo_write_pending, replay_attempts) "
            "VALUES (:wf_id, :ts, 'transition', :from_state, 'STUCK', "
            "        :aid, :payload, 0, 0)"
        ),
        {
            "wf_id": workflow_id,
            "ts": now,
            "from_state": from_state,
            "aid": attempt_id,
            "payload": json.dumps({"reason": reason}),
        },
    )


__all__ = ["TickReport", "run_tick"]