cleveragents-core/tools/controller/worker/runner.py

"""Per-attempt runner: dequeue → run → write output → release lock.

Plan v9 worker mainline:
1. The dequeue (in loop.py) gives us an attempt_id + role + input_payload
2. Start a heartbeat thread keeping the lock alive
3. Invoke the agent (Phase 1c-2: dispatch via OpenCode using the role's
   builder MCP; this skeleton emits a stand-in for the test harness)
4. On success: write status='complete' + output_payload; release lock
5. On lost lock (heartbeat detected reap): abort silently — reaper has
   already re-pended; another worker will retry
6. On worker-internal error: write status='failed' + outcome classification

This module's ``run_one_attempt`` is the pure function the test
harness exercises. The loop module wraps it with the dequeue +
ThreadPoolExecutor + retry semantics.

For Phase 1c skeleton: the actual OpenCode invocation is parameterised
as ``agent_runner``. Tests inject a FakeAgentRunner; the production
adapter (Phase 1c-2) will spawn the MCP subprocess + the OpenCode
session.
"""

from __future__ import annotations

import logging
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any

from sqlalchemy import text
from sqlalchemy.engine import Engine

from ..db.session import session_scope
from .heartbeat import Heartbeat, HeartbeatStopped

logger = logging.getLogger(__name__)


# ─── result types ─────────────────────────────────────────────────────


@dataclass
class AttemptOutcome:
    """What ``run_one_attempt`` returns.

    Mirrors the status='complete' vs 'failed' write the runner
    performs. Tests can introspect this without re-querying the DB.
    """

    attempt_id: int
    status: str  # 'complete' | 'failed' | 'aborted'
    outcome: str | None  # role-output outcome OR worker-side classification
    output_payload: dict[str, Any] | None
    wallclock_seconds: float


class WorkerError(Exception):
    """Internal worker-side error (transport, MCP crash, etc.).

    The runner catches this and writes ``status='failed'`` with
    ``outcome`` set from this exception; the master re-enqueues per the
    status=failed policy.

    ``output_payload`` lets a caller attach structured detail that must
    survive to the next attempt's prompt (e.g. a worker-run gate's
    failure report) — the runner merges it into the recorded payload.
    """

    def __init__(
        self,
        message: str,
        outcome: str = "worker-internal-error",
        output_payload: dict | None = None,
    ):
        self.outcome = outcome
        self.output_payload = output_payload
        super().__init__(message)


class WorkerLostLock(Exception):
    """Raised when the heartbeat detected the lock was reaped.

    The runner catches this and silently aborts — does NOT write
    output. The reaper has already re-pended the attempt; another
    worker will pick it up.
    """


# ─── agent_runner protocol ────────────────────────────────────────────


# Type alias for the callable that drives one attempt's role-MCP +
# OpenCode session. The production adapter wraps OpenCode + spawns
# the matching builder MCP subprocess. Tests inject a fake.
#
# Inputs:
#   attempt_id, role, tier, input_payload, instance_id, lost_lock_event_check
#
# The agent_runner reads ``input_payload`` (already validated against
# {Role}InputV1 contract by the master), invokes the role-builder MCP,
# and returns the canonical output dict (already round-tripped through
# Pydantic by the controller's strict-parse layer).
#
# ``lost_lock_event_check()`` returns True if the heartbeat thread has
# signaled lost lock; the agent_runner SHOULD check this periodically
# and raise ``WorkerLostLock`` to bail out early. The runner also
# catches the raised event for defense-in-depth.
AgentRunner = Callable[..., dict[str, Any]]


# ─── runner ────────────────────────────────────────────────────────────


def run_one_attempt(
    engine: Engine,
    *,
    attempt_id: int,
    role: str,
    tier: int | None,
    input_payload: dict[str, Any],
    instance_id: str,
    agent_runner: AgentRunner,
    heartbeat_interval_s: int = 30,
) -> AttemptOutcome:
    """Run one attempt end-to-end.

    Caller has already dequeued the attempt (status='in_progress',
    lock columns set). This function:
    - Starts the heartbeat
    - Invokes ``agent_runner`` to do the actual work
    - On success: writes status='complete' + output payload
    - On lost lock: aborts silently (no DB write — reaper handled it)
    - On WorkerError: writes status='failed' with the outcome label
    - On other Exception: writes status='failed' outcome='worker-internal-error'
    - Stops the heartbeat in finally

    R-round4 P2: defensive normalization of input_payload — if the
    DB returned NULL (legacy/seeded data, schema bug, etc.) we
    substitute an empty dict so the agent_runner doesn't crash on
    ``dict(None)`` and infinite-loop pickup_count → STUCK.
    """
    if not isinstance(input_payload, dict):
        logger.warning(
            "attempt_id=%s received non-dict input_payload (%r); "
            "substituting empty dict so the agent_runner doesn't crash",
            attempt_id,
            type(input_payload).__name__,
        )
        input_payload = {}
    started_at = datetime.now(timezone.utc)
    heartbeat = Heartbeat(
        engine, attempt_id, instance_id, interval_s=heartbeat_interval_s
    )
    heartbeat.start()
    output_payload: dict[str, Any] | None = None
    outcome: str | None = None
    final_status = "failed"
    try:
        # Wrap the agent_runner so it can check lost_lock without
        # depending on the heartbeat module directly.
        def lost_lock_check() -> bool:
            return heartbeat.lost_lock_event.is_set()

        try:
            output_payload = agent_runner(
                attempt_id=attempt_id,
                role=role,
                tier=tier,
                input_payload=input_payload,
                instance_id=instance_id,
                lost_lock_check=lost_lock_check,
            )
        except WorkerLostLock:
            # Agent observed lock loss + bailed. Honor the contract:
            # silent abort.
            logger.info(
                "attempt_id=%s aborted: lost lock during agent run",
                attempt_id,
            )
            return AttemptOutcome(
                attempt_id=attempt_id,
                status="aborted",
                outcome="lost-lock",
                output_payload=None,
                wallclock_seconds=_elapsed(started_at),
            )

        # Defense in depth: even if the agent didn't raise, check the
        # event before writing.
        if heartbeat.lost_lock_event.is_set():
            logger.info(
                "attempt_id=%s lock lost during agent run; not writing output",
                attempt_id,
            )
            return AttemptOutcome(
                attempt_id=attempt_id,
                status="aborted",
                outcome="lost-lock",
                output_payload=None,
                wallclock_seconds=_elapsed(started_at),
            )

        # Success path: write status='complete'.
        #
        # The ``outcome`` column on workflow_attempts is the operator-
        # facing summary of what the attempt did. Pre-2026-05-19 we
        # extracted ``output_payload.get("outcome")`` blindly — fine
        # for implementer/conflict_resolver, but estimator (has
        # ``recommended_tier``, no ``outcome``), reviewer (has
        # ``verdict``), and summarizer (neither) all wrote
        # ``outcome=NULL``. Operator queries like
        # ``SELECT … WHERE outcome IS NOT NULL`` silently missed every
        # estimator/reviewer/summarizer attempt. Synthesize a
        # meaningful per-role value so the audit column is useful.
        outcome = _derive_outcome_for_audit(role, output_payload)
        final_status = "complete"
    except WorkerError as exc:
        logger.warning(
            "attempt_id=%s worker error: %s (outcome=%s)",
            attempt_id,
            exc,
            exc.outcome,
        )
        outcome = exc.outcome
        output_payload = {"error": str(exc), "worker_outcome": exc.outcome}
        if isinstance(exc.output_payload, dict):
            # Structured detail (e.g. a gate-failure report) the next
            # attempt's prompt needs — keep it alongside the error
            # markers (the markers win on any key collision).
            output_payload = {**exc.output_payload, **output_payload}
        final_status = "failed"
    except Exception as exc:  # noqa: BLE001 — last-resort catch
        logger.exception("attempt_id=%s unexpected exception", attempt_id)
        outcome = "worker-internal-error"
        output_payload = {"error": str(exc), "exception_type": type(exc).__name__}
        final_status = "failed"
    finally:
        heartbeat.stop()

    # Phase 1k++++ (run-fix): head_sha bookkeeping. tick.py reads
    # head_sha_before/after to compute head_sha_advanced; without
    # writing them, implementer 'resolved' outcomes would map to a
    # no-op event and workflows would stall after the agent ran.
    #
    # head_sha_before comes from the input_payload (what the
    # prefetch saw + the agent was told to start from).
    # head_sha_after comes from the agent's output_payload — the
    # last commit_sha if any commits were produced; falls back to
    # head_sha_before when the agent didn't commit (no advance).
    hs_before = (
        input_payload.get("head_sha") if isinstance(input_payload, dict) else None
    )
    hs_after = hs_before
    if isinstance(output_payload, dict):
        # ConflictResolverOutputV1 carries the canonical post-rebase
        # HEAD in ``new_head_sha`` (contract: "Required iff
        # outcome='resolved'"). Prefer it over ``commit_shas[-1]``
        # because a force-pushed rebase's last commit SHA may differ
        # from the actual branch tip (e.g., if the resolver did a
        # merge commit after fixing conflicts). The CI status poll
        # queries this SHA — getting it wrong polls the wrong commit.
        if role == "conflict_resolver":
            new_head = output_payload.get("new_head_sha")
            if isinstance(new_head, str) and new_head:
                hs_after = new_head
        if hs_after == hs_before:
            commits = output_payload.get("commit_shas") or []
            if isinstance(commits, list) and commits:
                last = commits[-1]
                if isinstance(last, str) and last:
                    hs_after = last

    # Write the result. If the lock was lost between agent return and
    # this write, the UPDATE WHERE locked_by_instance=us returns
    # rowcount=0 and we treat as aborted.
    wrote = _write_outcome(
        engine,
        attempt_id=attempt_id,
        instance_id=instance_id,
        status=final_status,
        outcome=outcome,
        output_payload=output_payload,
        wallclock_seconds=_elapsed(started_at),
        head_sha_before=hs_before,
        head_sha_after=hs_after,
    )
    if not wrote:
        logger.info(
            "attempt_id=%s output write found no matching locked row "
            "(lost lock); treating as aborted",
            attempt_id,
        )
        return AttemptOutcome(
            attempt_id=attempt_id,
            status="aborted",
            outcome="lost-lock-at-write",
            output_payload=None,
            wallclock_seconds=_elapsed(started_at),
        )
    return AttemptOutcome(
        attempt_id=attempt_id,
        status=final_status,
        outcome=outcome,
        output_payload=output_payload,
        wallclock_seconds=_elapsed(started_at),
    )


def _elapsed(started_at: datetime) -> float:
    return (datetime.now(timezone.utc) - started_at).total_seconds()


def _derive_outcome_for_audit(role: str, output_payload) -> str | None:
    """Extract the operator-facing ``outcome`` summary from a V1 payload.

    Different V1 contracts carry the "what happened" signal in
    different fields:

      - ``ImplementerOutputV1`` / ``ConflictResolverOutputV1``: ``outcome``
      - ``ReviewerOutputV1``: ``verdict``
      - ``EstimatorOutputV1``: ``recommended_tier`` (+ ``is_metadata_only``)
      - ``SummarizerOutputV1``: neither — synthesized constant

    Pre-2026-05-19 only the ``outcome`` field was extracted, so the
    audit column was NULL for the three roles that don't have it.
    This synthesizes a meaningful value per role so operator queries
    like ``SELECT ... WHERE outcome IS NOT NULL`` don't silently miss
    every successful estimator / reviewer / summarizer attempt.
    """
    if not isinstance(output_payload, dict):
        return None
    if role in ("implementer", "conflict_resolver"):
        return output_payload.get("outcome")
    if role == "reviewer":
        return output_payload.get("verdict")
    if role == "estimator":
        if output_payload.get("is_metadata_only") is True:
            return "metadata-only"
        rt = output_payload.get("recommended_tier")
        if isinstance(rt, int) and rt in (0, 1, 2):
            return f"tier-{rt}"
        return None
    if role == "summarizer":
        # Summarizer always produces a summary on success; the
        # operator-facing audit value is just "summarized" — the
        # actual summary text lives in output_payload.
        return "summarized"
    return output_payload.get("outcome")


def _write_outcome(
    engine: Engine,
    *,
    attempt_id: int,
    instance_id: str,
    status: str,
    outcome: str | None,
    output_payload: dict[str, Any] | None,
    wallclock_seconds: float,
    head_sha_before: str | None = None,
    head_sha_after: str | None = None,
) -> bool:
    """UPDATE the attempt row with terminal state + release lock.

    Returns True if the row was updated (we still held the lock).
    Returns False if rowcount=0 (lost lock; another worker / reaper
    took over).

    ``head_sha_before`` / ``head_sha_after`` (Phase 1k++++ run-fix):
    tick.py reads these to compute ``head_sha_advanced``, which the
    outcome mapper requires to distinguish ``implementer_pushed``
    (true push happened) from ``implementer_blocked`` (worker said
    resolved but git didn't move). Without writing them, the tick
    sees head_sha_advanced=False and the workflow doesn't progress.
    """
    now = datetime.now(timezone.utc)
    from .._json_safe import safe_json_dumps as _safe_json_dumps

    with session_scope(engine) as session:
        result = session.execute(
            text(
                "UPDATE workflow_attempts SET "
                "  status = :status, "
                "  outcome = :outcome, "
                "  output_payload = :output_payload, "
                "  output_version = :output_version, "
                "  finished_at = :now, "
                "  wallclock_seconds = :wallclock, "
                "  head_sha_before = :head_sha_before, "
                "  head_sha_after = :head_sha_after, "
                "  locked_by_instance = NULL, "
                "  locked_at = NULL, "
                "  lock_heartbeat_at = NULL "
                "WHERE attempt_id = :attempt_id "
                "  AND locked_by_instance = :instance"
            ),
            {
                "status": status,
                "outcome": outcome,
                "output_payload": (
                    # Restricted encoder (datetime / Decimal / UUID /
                    # Path / set) — anything else raises so worker
                    # output regressions surface loudly instead of
                    # silently stringifying to "<MyObj at 0x...>".
                    _safe_json_dumps(output_payload)
                    if output_payload is not None
                    else None
                ),
                "output_version": (
                    output_payload.get("output_version")
                    if isinstance(output_payload, dict)
                    else None
                ),
                "now": now,
                "wallclock": wallclock_seconds,
                "head_sha_before": head_sha_before,
                "head_sha_after": head_sha_after,
                "attempt_id": attempt_id,
                "instance": instance_id,
            },
        )
        return result.rowcount > 0