"""Mechanical hang-pattern detectors for OpenCode worker sessions. Used by :func:`tools._opencode_worker.run_session_blocking` to abort a stuck worker BEFORE its wallclock budget expires, with a named cause (``doom-loop``, ``retry-cascade``, etc.) flowing into ``SessionResult.error_kind`` and the cycle archive — instead of the opaque ``watchdog-timeout`` that a hard-timeout abort produces. Pure functions; no I/O, no LLM. The wider worker module owns the fetch of ``/session/{id}/message`` and the abort POST; this module only inspects the message-stream shape. Design constraints (carried over from ``agents/final-working``'s ``session-health-full-util.md`` — A3 harvest, 2026-05-15): * A ``bash`` invocation that runs ``sleep …`` does NOT count as meaningful tool activity (it is by definition waiting). The detectors ignore it when counting repeats / consecutive errors. * ``step_finish.reason == "tool-calls"`` means the turn is "waiting for the tool result," NOT "stuck." Detectors operate on the presence of tool *parts* in a turn, so turns that emitted a tool call and are awaiting its result are counted as activity, not as empty-reasoning. * Healthy is the default when signals are mixed. All thresholds are set conservatively so a healthy long session is not aborted; the calling code's absolute wallclock timeout remains the ultimate net. Out of scope here: "permanent block" was listed in the original harvest finding but has no clean mechanical definition that does not duplicate ``permission-deadlock`` or ``retry-cascade``. Left as a TODO; revisit when a sample session demonstrates a pattern the four detectors below miss. """ from __future__ import annotations from dataclasses import dataclass from typing import Any # Public, named-by-cause failure kinds — flow into # ``SessionResult.error_kind`` and the cycle-archive payload so an # operator can grep for them across runs. ERROR_KIND_DOOM_LOOP = "doom-loop" ERROR_KIND_RETRY_CASCADE = "retry-cascade" ERROR_KIND_PERMISSION_DEADLOCK = "permission-deadlock" ERROR_KIND_EMPTY_REASONING = "empty-reasoning-loop" # Detection thresholds — picked conservatively so a healthy long # session is not aborted. Tuning these DOWN is more aggressive (more # false positives); tuning UP is more conservative (more wallclock # burned before a hang is diagnosed). _DOOM_LOOP_REPEAT_THRESHOLD = 4 # same tool+args 4+ consecutive times _RETRY_CASCADE_ERROR_THRESHOLD = 5 # 5+ consecutive tool errors _PERMISSION_DEADLOCK_THRESHOLD = 3 # 3+ consecutive errors on the same tool _EMPTY_REASONING_THRESHOLD = 6 # 6+ consecutive assistant turns with NO tool parts @dataclass(frozen=True) class PatternFinding: """A pattern detector hit. ``error_kind`` is one of the ``ERROR_KIND_*`` module constants — the named failure cause the worker module reports to its caller. ``reason`` is a short operator-facing string with the specific evidence (e.g. ``"same tool action repeated 4 times: bash::nox -e lint"``) — flows into telemetry and the session-archive payload so a future operator does not have to re-derive what tripped. """ error_kind: str reason: str def check_message_stream( messages: list[dict[str, Any]] | None, ) -> PatternFinding | None: """Return a :class:`PatternFinding` if the assistant's recent message stream matches any hang pattern; ``None`` if the stream looks healthy or has too little signal to judge. First-match-wins in the order the detectors run below — the most specific / actionable diagnosis wins (e.g. a permission deadlock is reported BEFORE its retry-cascade cousin even though both might match). The order is fixed and documented; tests pin it. ``messages`` is the raw JSON list returned by ``GET /session/{id}/message`` — a list of message objects in chronological order. Tolerant of OpenCode shape variations: every accessor is defensive, and an unexpected payload yields ``None`` (healthy-by-default) rather than raising. """ if not isinstance(messages, list) or not messages: return None actions = _extract_tool_actions(messages) finding = _detect_doom_loop(actions) if finding is not None: return finding # Permission deadlock is a strict subset of retry-cascade (same-tool # consecutive errors) — check it first so an operator gets the # specific diagnosis when both would match. finding = _detect_permission_deadlock(actions) if finding is not None: return finding finding = _detect_retry_cascade(actions) if finding is not None: return finding finding = _detect_empty_reasoning(messages) if finding is not None: return finding return None # ─── Extractors ────────────────────────────────────────────────────────── def _extract_tool_actions( messages: list[dict[str, Any]], ) -> list[dict[str, Any]]: """Flatten the OpenCode message stream into a chronological list of tool actions taken by the assistant. Each entry:: { "tool": str, # tool name (e.g. "bash", "skill") "args_key": str, # stable, args-derived digest for # repeat detection "is_error": bool, # tool result indicates an error "is_sleep": bool, # bash invocation whose command is # ``sleep`` (A3: not real work) } Messages that are not assistant turns, parts that are not tool parts, and shapes that don't match expectations are silently skipped — the detectors then operate on whatever signal IS available. """ actions: list[dict[str, Any]] = [] for m in messages: if not isinstance(m, dict): continue info = m.get("info") if isinstance(m.get("info"), dict) else m if not isinstance(info, dict) or info.get("role") != "assistant": continue parts = m.get("parts") if isinstance(m.get("parts"), list) else [] for p in parts: if not isinstance(p, dict) or p.get("type") != "tool": continue tool = p.get("tool") or "?" args = _part_args(p) actions.append( { "tool": str(tool), "args_key": _args_signature(str(tool), args), "is_error": _part_is_error(p), "is_sleep": _part_is_sleep(str(tool), args), } ) return actions def _part_args(part: dict[str, Any]) -> dict[str, Any]: """Best-effort extraction of a tool part's input args. OpenCode's tool part shape has carried tool input under several keys across versions. Probes ``input``, ``params``, and ``state.input`` in turn; returns the first dict found, or an empty dict (the detectors treat absent args as "no signal" rather than raising). """ for key in ("input", "params"): v = part.get(key) if isinstance(v, dict): return v state = part.get("state") if isinstance(state, dict): v = state.get("input") if isinstance(v, dict): return v return {} def _part_is_error(part: dict[str, Any]) -> bool: """True iff the tool part's state indicates the tool returned an error. Defensive: shape variations are tolerated, a part with no ``state`` is treated as "no error signal" rather than as failed. """ state = part.get("state") if not isinstance(state, dict): return False if state.get("status") == "error": return True if state.get("error"): return True return False def _part_is_sleep(tool: str, args: dict[str, Any]) -> bool: """A3 design constraint: a ``bash`` invocation whose command is ``sleep …`` is NOT meaningful tool activity. Identifying it here lets the activity-based detectors (doom-loop, retry-cascade, permission-deadlock) skip past it without treating an in-script cooldown as evidence of progress or as a repeat. """ if tool != "bash": return False cmd = args.get("command") if not isinstance(cmd, str): return False return cmd.strip().startswith("sleep") def _args_signature(tool: str, args: dict[str, Any]) -> str: """Stable, args-derived digest of a tool action for repeat detection. Two tool calls with identical signatures are treated as repeats; different signatures are treated as different actions. For ``bash``, the digest is the (whitespace-trimmed, capped) command string — the disambiguator that matters. For other tools, a sorted, length-capped stringification of the args dict; lossy by design (we only care about *equality* of consecutive actions, not reconstructibility). """ if tool == "bash": cmd = args.get("command", "") if isinstance(cmd, str): return f"bash::{cmd.strip()[:120]}" pairs = sorted((k, str(v)[:60]) for k, v in args.items() if isinstance(k, str)) return f"{tool}::" + "|".join(f"{k}={v}" for k, v in pairs)[:200] # ─── Detectors ─────────────────────────────────────────────────────────── def _detect_doom_loop( actions: list[dict[str, Any]], ) -> PatternFinding | None: """Doom loop = the same (tool, args) action called :data:`_DOOM_LOOP_REPEAT_THRESHOLD` times in a row. Sleep invocations break the run rather than counting toward it — they would otherwise mask a doom loop by inserting "activity" between the real repeats. """ if len(actions) < _DOOM_LOOP_REPEAT_THRESHOLD: return None run_len = 0 prev_key: str | None = None for a in actions: if a.get("is_sleep"): run_len = 0 prev_key = None continue key = a.get("args_key", "") if key == prev_key: run_len += 1 else: prev_key = key run_len = 1 if run_len >= _DOOM_LOOP_REPEAT_THRESHOLD: return PatternFinding( error_kind=ERROR_KIND_DOOM_LOOP, reason=(f"same tool action repeated {run_len} times: {key[:120]}"), ) return None def _detect_retry_cascade( actions: list[dict[str, Any]], ) -> PatternFinding | None: """Retry cascade = :data:`_RETRY_CASCADE_ERROR_THRESHOLD` consecutive tool calls that returned an error. Sleep invocations don't count for or against — they neither error nor make progress, so they pass through the cascade detector silently. """ if len(actions) < _RETRY_CASCADE_ERROR_THRESHOLD: return None err_run = 0 last_action: dict[str, Any] | None = None for a in actions: if a.get("is_sleep"): continue if a.get("is_error"): err_run += 1 last_action = a if err_run >= _RETRY_CASCADE_ERROR_THRESHOLD: tail_key = (last_action or {}).get("args_key", "") return PatternFinding( error_kind=ERROR_KIND_RETRY_CASCADE, reason=( f"{err_run} consecutive tool errors; last: {tail_key[:120]}" ), ) else: err_run = 0 last_action = None return None def _detect_permission_deadlock( actions: list[dict[str, Any]], ) -> PatternFinding | None: """Permission deadlock = :data:`_PERMISSION_DEADLOCK_THRESHOLD` consecutive errors on the SAME tool. The worker is asking for something it cannot have; without intervention the pattern will continue until the wallclock timeout. Specifically detects same-tool-error runs that are shorter than the broader retry-cascade threshold (3..4 by default) — a true permission deadlock is a narrower diagnosis (one tool consistently denied) than a cascade (any sequence of tool failures), and an operator benefits from the more specific diagnosis. Same-tool runs that ALSO satisfy the cascade threshold (5+) are reported as deadlocks here too — caller's first-match-wins ordering surfaces the specific one. """ if len(actions) < _PERMISSION_DEADLOCK_THRESHOLD: return None last_tool: str | None = None run = 0 for a in actions: if a.get("is_sleep"): continue if not a.get("is_error"): last_tool = None run = 0 continue tool = a.get("tool", "") if tool == last_tool: run += 1 else: last_tool = tool run = 1 if run >= _PERMISSION_DEADLOCK_THRESHOLD: return PatternFinding( error_kind=ERROR_KIND_PERMISSION_DEADLOCK, reason=(f"{run} consecutive errors on the same tool ({tool!r})"), ) return None def _detect_empty_reasoning( messages: list[dict[str, Any]], ) -> PatternFinding | None: """Empty-reasoning loop = :data:`_EMPTY_REASONING_THRESHOLD` consecutive assistant turns that contained NO tool parts. A3 design constraint: a turn whose ``step_finish.reason == "tool-calls"`` is *waiting for a tool result*, NOT empty — but that turn DID emit a tool part (the call), so it counts as activity here and won't trip this detector. A turn with only text and no tool call IS evidence of stuck-in-thinking when it repeats N consecutive times. """ empty_run = 0 for m in messages: if not isinstance(m, dict): continue info = m.get("info") if isinstance(m.get("info"), dict) else m if not isinstance(info, dict) or info.get("role") != "assistant": continue parts = m.get("parts") if isinstance(m.get("parts"), list) else [] has_tool = any(isinstance(p, dict) and p.get("type") == "tool" for p in parts) if has_tool: empty_run = 0 else: empty_run += 1 if empty_run >= _EMPTY_REASONING_THRESHOLD: return PatternFinding( error_kind=ERROR_KIND_EMPTY_REASONING, reason=( f"{empty_run} consecutive assistant turns with no tool call" ), ) return None