drew
0bc734c020
Applies `ruff format` to the accumulated formatting debt on this branch. Formatting-only — no behavioral changes. Required for CI/lint's format gate (`nox -s format -- --check`), which the branch was failing on 288 tracked files that drifted from ruff's canonical style. In-progress WIP files are intentionally excluded so this commit stays a clean formatting-only diff. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
386 lines
14 KiB
Python
386 lines
14 KiB
Python
"""Mechanical hang-pattern detectors for OpenCode worker sessions.
|
|
|
|
Used by :func:`tools._opencode_worker.run_session_blocking` to abort
|
|
a stuck worker BEFORE its wallclock budget expires, with a named
|
|
cause (``doom-loop``, ``retry-cascade``, etc.) flowing into
|
|
``SessionResult.error_kind`` and the cycle archive — instead of the
|
|
opaque ``watchdog-timeout`` that a hard-timeout abort produces.
|
|
|
|
Pure functions; no I/O, no LLM. The wider worker module owns the
|
|
fetch of ``/session/{id}/message`` and the abort POST; this module
|
|
only inspects the message-stream shape.
|
|
|
|
Design constraints (carried over from ``agents/final-working``'s
|
|
``session-health-full-util.md`` — A3 harvest, 2026-05-15):
|
|
|
|
* A ``bash`` invocation that runs ``sleep …`` does NOT count as
|
|
meaningful tool activity (it is by definition waiting). The
|
|
detectors ignore it when counting repeats / consecutive errors.
|
|
* ``step_finish.reason == "tool-calls"`` means the turn is "waiting
|
|
for the tool result," NOT "stuck." Detectors operate on the
|
|
presence of tool *parts* in a turn, so turns that emitted a tool
|
|
call and are awaiting its result are counted as activity, not as
|
|
empty-reasoning.
|
|
* Healthy is the default when signals are mixed. All thresholds are
|
|
set conservatively so a healthy long session is not aborted; the
|
|
calling code's absolute wallclock timeout remains the ultimate net.
|
|
|
|
Out of scope here: "permanent block" was listed in the original
|
|
harvest finding but has no clean mechanical definition that does
|
|
not duplicate ``permission-deadlock`` or ``retry-cascade``. Left as
|
|
a TODO; revisit when a sample session demonstrates a pattern the
|
|
four detectors below miss.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
|
|
# Public, named-by-cause failure kinds — flow into
|
|
# ``SessionResult.error_kind`` and the cycle-archive payload so an
|
|
# operator can grep for them across runs.
|
|
ERROR_KIND_DOOM_LOOP = "doom-loop"
|
|
ERROR_KIND_RETRY_CASCADE = "retry-cascade"
|
|
ERROR_KIND_PERMISSION_DEADLOCK = "permission-deadlock"
|
|
ERROR_KIND_EMPTY_REASONING = "empty-reasoning-loop"
|
|
|
|
|
|
# Detection thresholds — picked conservatively so a healthy long
|
|
# session is not aborted. Tuning these DOWN is more aggressive (more
|
|
# false positives); tuning UP is more conservative (more wallclock
|
|
# burned before a hang is diagnosed).
|
|
_DOOM_LOOP_REPEAT_THRESHOLD = 4 # same tool+args 4+ consecutive times
|
|
_RETRY_CASCADE_ERROR_THRESHOLD = 5 # 5+ consecutive tool errors
|
|
_PERMISSION_DEADLOCK_THRESHOLD = 3 # 3+ consecutive errors on the same tool
|
|
_EMPTY_REASONING_THRESHOLD = 6 # 6+ consecutive assistant turns with NO tool parts
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PatternFinding:
|
|
"""A pattern detector hit.
|
|
|
|
``error_kind`` is one of the ``ERROR_KIND_*`` module constants —
|
|
the named failure cause the worker module reports to its caller.
|
|
``reason`` is a short operator-facing string with the specific
|
|
evidence (e.g. ``"same tool action repeated 4 times: bash::nox -e
|
|
lint"``) — flows into telemetry and the session-archive payload
|
|
so a future operator does not have to re-derive what tripped.
|
|
"""
|
|
|
|
error_kind: str
|
|
reason: str
|
|
|
|
|
|
def check_message_stream(
|
|
messages: list[dict[str, Any]] | None,
|
|
) -> PatternFinding | None:
|
|
"""Return a :class:`PatternFinding` if the assistant's recent
|
|
message stream matches any hang pattern; ``None`` if the stream
|
|
looks healthy or has too little signal to judge.
|
|
|
|
First-match-wins in the order the detectors run below — the most
|
|
specific / actionable diagnosis wins (e.g. a permission deadlock
|
|
is reported BEFORE its retry-cascade cousin even though both
|
|
might match). The order is fixed and documented; tests pin it.
|
|
|
|
``messages`` is the raw JSON list returned by
|
|
``GET /session/{id}/message`` — a list of message objects in
|
|
chronological order. Tolerant of OpenCode shape variations: every
|
|
accessor is defensive, and an unexpected payload yields ``None``
|
|
(healthy-by-default) rather than raising.
|
|
"""
|
|
if not isinstance(messages, list) or not messages:
|
|
return None
|
|
|
|
actions = _extract_tool_actions(messages)
|
|
|
|
finding = _detect_doom_loop(actions)
|
|
if finding is not None:
|
|
return finding
|
|
|
|
# Permission deadlock is a strict subset of retry-cascade (same-tool
|
|
# consecutive errors) — check it first so an operator gets the
|
|
# specific diagnosis when both would match.
|
|
finding = _detect_permission_deadlock(actions)
|
|
if finding is not None:
|
|
return finding
|
|
|
|
finding = _detect_retry_cascade(actions)
|
|
if finding is not None:
|
|
return finding
|
|
|
|
finding = _detect_empty_reasoning(messages)
|
|
if finding is not None:
|
|
return finding
|
|
|
|
return None
|
|
|
|
|
|
# ─── Extractors ──────────────────────────────────────────────────────────
|
|
|
|
|
|
def _extract_tool_actions(
|
|
messages: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
"""Flatten the OpenCode message stream into a chronological list
|
|
of tool actions taken by the assistant.
|
|
|
|
Each entry::
|
|
|
|
{
|
|
"tool": str, # tool name (e.g. "bash", "skill")
|
|
"args_key": str, # stable, args-derived digest for
|
|
# repeat detection
|
|
"is_error": bool, # tool result indicates an error
|
|
"is_sleep": bool, # bash invocation whose command is
|
|
# ``sleep`` (A3: not real work)
|
|
}
|
|
|
|
Messages that are not assistant turns, parts that are not tool
|
|
parts, and shapes that don't match expectations are silently
|
|
skipped — the detectors then operate on whatever signal IS
|
|
available.
|
|
"""
|
|
actions: list[dict[str, Any]] = []
|
|
for m in messages:
|
|
if not isinstance(m, dict):
|
|
continue
|
|
info = m.get("info") if isinstance(m.get("info"), dict) else m
|
|
if not isinstance(info, dict) or info.get("role") != "assistant":
|
|
continue
|
|
parts = m.get("parts") if isinstance(m.get("parts"), list) else []
|
|
for p in parts:
|
|
if not isinstance(p, dict) or p.get("type") != "tool":
|
|
continue
|
|
tool = p.get("tool") or "?"
|
|
args = _part_args(p)
|
|
actions.append(
|
|
{
|
|
"tool": str(tool),
|
|
"args_key": _args_signature(str(tool), args),
|
|
"is_error": _part_is_error(p),
|
|
"is_sleep": _part_is_sleep(str(tool), args),
|
|
}
|
|
)
|
|
return actions
|
|
|
|
|
|
def _part_args(part: dict[str, Any]) -> dict[str, Any]:
|
|
"""Best-effort extraction of a tool part's input args.
|
|
|
|
OpenCode's tool part shape has carried tool input under
|
|
several keys across versions. Probes ``input``, ``params``,
|
|
and ``state.input`` in turn; returns the first dict found, or
|
|
an empty dict (the detectors treat absent args as "no signal"
|
|
rather than raising).
|
|
"""
|
|
for key in ("input", "params"):
|
|
v = part.get(key)
|
|
if isinstance(v, dict):
|
|
return v
|
|
state = part.get("state")
|
|
if isinstance(state, dict):
|
|
v = state.get("input")
|
|
if isinstance(v, dict):
|
|
return v
|
|
return {}
|
|
|
|
|
|
def _part_is_error(part: dict[str, Any]) -> bool:
|
|
"""True iff the tool part's state indicates the tool returned an
|
|
error. Defensive: shape variations are tolerated, a part with no
|
|
``state`` is treated as "no error signal" rather than as failed.
|
|
"""
|
|
state = part.get("state")
|
|
if not isinstance(state, dict):
|
|
return False
|
|
if state.get("status") == "error":
|
|
return True
|
|
if state.get("error"):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _part_is_sleep(tool: str, args: dict[str, Any]) -> bool:
|
|
"""A3 design constraint: a ``bash`` invocation whose command is
|
|
``sleep …`` is NOT meaningful tool activity. Identifying it here
|
|
lets the activity-based detectors (doom-loop, retry-cascade,
|
|
permission-deadlock) skip past it without treating an in-script
|
|
cooldown as evidence of progress or as a repeat.
|
|
"""
|
|
if tool != "bash":
|
|
return False
|
|
cmd = args.get("command")
|
|
if not isinstance(cmd, str):
|
|
return False
|
|
return cmd.strip().startswith("sleep")
|
|
|
|
|
|
def _args_signature(tool: str, args: dict[str, Any]) -> str:
|
|
"""Stable, args-derived digest of a tool action for repeat
|
|
detection. Two tool calls with identical signatures are treated
|
|
as repeats; different signatures are treated as different actions.
|
|
|
|
For ``bash``, the digest is the (whitespace-trimmed, capped)
|
|
command string — the disambiguator that matters. For other tools,
|
|
a sorted, length-capped stringification of the args dict; lossy
|
|
by design (we only care about *equality* of consecutive actions,
|
|
not reconstructibility).
|
|
"""
|
|
if tool == "bash":
|
|
cmd = args.get("command", "")
|
|
if isinstance(cmd, str):
|
|
return f"bash::{cmd.strip()[:120]}"
|
|
pairs = sorted((k, str(v)[:60]) for k, v in args.items() if isinstance(k, str))
|
|
return f"{tool}::" + "|".join(f"{k}={v}" for k, v in pairs)[:200]
|
|
|
|
|
|
# ─── Detectors ───────────────────────────────────────────────────────────
|
|
|
|
|
|
def _detect_doom_loop(
|
|
actions: list[dict[str, Any]],
|
|
) -> PatternFinding | None:
|
|
"""Doom loop = the same (tool, args) action called
|
|
:data:`_DOOM_LOOP_REPEAT_THRESHOLD` times in a row.
|
|
|
|
Sleep invocations break the run rather than counting toward it —
|
|
they would otherwise mask a doom loop by inserting "activity"
|
|
between the real repeats.
|
|
"""
|
|
if len(actions) < _DOOM_LOOP_REPEAT_THRESHOLD:
|
|
return None
|
|
run_len = 0
|
|
prev_key: str | None = None
|
|
for a in actions:
|
|
if a.get("is_sleep"):
|
|
run_len = 0
|
|
prev_key = None
|
|
continue
|
|
key = a.get("args_key", "")
|
|
if key == prev_key:
|
|
run_len += 1
|
|
else:
|
|
prev_key = key
|
|
run_len = 1
|
|
if run_len >= _DOOM_LOOP_REPEAT_THRESHOLD:
|
|
return PatternFinding(
|
|
error_kind=ERROR_KIND_DOOM_LOOP,
|
|
reason=(f"same tool action repeated {run_len} times: {key[:120]}"),
|
|
)
|
|
return None
|
|
|
|
|
|
def _detect_retry_cascade(
|
|
actions: list[dict[str, Any]],
|
|
) -> PatternFinding | None:
|
|
"""Retry cascade = :data:`_RETRY_CASCADE_ERROR_THRESHOLD`
|
|
consecutive tool calls that returned an error.
|
|
|
|
Sleep invocations don't count for or against — they neither error
|
|
nor make progress, so they pass through the cascade detector
|
|
silently.
|
|
"""
|
|
if len(actions) < _RETRY_CASCADE_ERROR_THRESHOLD:
|
|
return None
|
|
err_run = 0
|
|
last_action: dict[str, Any] | None = None
|
|
for a in actions:
|
|
if a.get("is_sleep"):
|
|
continue
|
|
if a.get("is_error"):
|
|
err_run += 1
|
|
last_action = a
|
|
if err_run >= _RETRY_CASCADE_ERROR_THRESHOLD:
|
|
tail_key = (last_action or {}).get("args_key", "")
|
|
return PatternFinding(
|
|
error_kind=ERROR_KIND_RETRY_CASCADE,
|
|
reason=(
|
|
f"{err_run} consecutive tool errors; last: {tail_key[:120]}"
|
|
),
|
|
)
|
|
else:
|
|
err_run = 0
|
|
last_action = None
|
|
return None
|
|
|
|
|
|
def _detect_permission_deadlock(
|
|
actions: list[dict[str, Any]],
|
|
) -> PatternFinding | None:
|
|
"""Permission deadlock = :data:`_PERMISSION_DEADLOCK_THRESHOLD`
|
|
consecutive errors on the SAME tool. The worker is asking for
|
|
something it cannot have; without intervention the pattern will
|
|
continue until the wallclock timeout.
|
|
|
|
Specifically detects same-tool-error runs that are shorter than
|
|
the broader retry-cascade threshold (3..4 by default) — a true
|
|
permission deadlock is a narrower diagnosis (one tool consistently
|
|
denied) than a cascade (any sequence of tool failures), and an
|
|
operator benefits from the more specific diagnosis. Same-tool
|
|
runs that ALSO satisfy the cascade threshold (5+) are reported
|
|
as deadlocks here too — caller's first-match-wins ordering
|
|
surfaces the specific one.
|
|
"""
|
|
if len(actions) < _PERMISSION_DEADLOCK_THRESHOLD:
|
|
return None
|
|
last_tool: str | None = None
|
|
run = 0
|
|
for a in actions:
|
|
if a.get("is_sleep"):
|
|
continue
|
|
if not a.get("is_error"):
|
|
last_tool = None
|
|
run = 0
|
|
continue
|
|
tool = a.get("tool", "")
|
|
if tool == last_tool:
|
|
run += 1
|
|
else:
|
|
last_tool = tool
|
|
run = 1
|
|
if run >= _PERMISSION_DEADLOCK_THRESHOLD:
|
|
return PatternFinding(
|
|
error_kind=ERROR_KIND_PERMISSION_DEADLOCK,
|
|
reason=(f"{run} consecutive errors on the same tool ({tool!r})"),
|
|
)
|
|
return None
|
|
|
|
|
|
def _detect_empty_reasoning(
|
|
messages: list[dict[str, Any]],
|
|
) -> PatternFinding | None:
|
|
"""Empty-reasoning loop = :data:`_EMPTY_REASONING_THRESHOLD`
|
|
consecutive assistant turns that contained NO tool parts.
|
|
|
|
A3 design constraint: a turn whose ``step_finish.reason ==
|
|
"tool-calls"`` is *waiting for a tool result*, NOT empty — but
|
|
that turn DID emit a tool part (the call), so it counts as
|
|
activity here and won't trip this detector. A turn with only
|
|
text and no tool call IS evidence of stuck-in-thinking when it
|
|
repeats N consecutive times.
|
|
"""
|
|
empty_run = 0
|
|
for m in messages:
|
|
if not isinstance(m, dict):
|
|
continue
|
|
info = m.get("info") if isinstance(m.get("info"), dict) else m
|
|
if not isinstance(info, dict) or info.get("role") != "assistant":
|
|
continue
|
|
parts = m.get("parts") if isinstance(m.get("parts"), list) else []
|
|
has_tool = any(isinstance(p, dict) and p.get("type") == "tool" for p in parts)
|
|
if has_tool:
|
|
empty_run = 0
|
|
else:
|
|
empty_run += 1
|
|
if empty_run >= _EMPTY_REASONING_THRESHOLD:
|
|
return PatternFinding(
|
|
error_kind=ERROR_KIND_EMPTY_REASONING,
|
|
reason=(
|
|
f"{empty_run} consecutive assistant turns with no tool call"
|
|
),
|
|
)
|
|
return None
|