drew
8fe98cb5c0
Two reliability fixes from the proactive audit of untouched pipeline modules (AUDIT-1, AUDIT-2 in .drew/PENDING_FIXES.md). AUDIT-1 — a timed-out implementer's committed work was never salvaged. opencode_session maps an OpenCode timeout / transport-error to WorkerError, which propagated out of production_agent_runner BEFORE the canonical-output wait + _salvage_implementer_commits ran. A timeout is the case most likely to have a complete committed fix (agent ran out of wallclock, not correctness). The session-exception handler now runs the same salvage the canonical-missing path uses before discarding the attempt as worker-internal-error. AUDIT-2 — a persistently failing prefetch() retried every master tick forever (the T4-4 guard covered only the issue-kind sub-case; same unbounded-retry class as the run-2 174x estimator loop). The scheduler now journals each prefetch failure as a controller_events row, counts failures since the workflow's last transition, and routes the workflow to STUCK after _PREFETCH_FAILURE_LIMIT (10) failures. The STUCK transition is rowcount-guarded so the journal never records a transition the current_state guard rejected. 7 new tests; full controller suite (1169) green. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
800 lines
30 KiB
Python
800 lines
30 KiB
Python
"""Production agent_runner adapter — spawns the role-builder MCP +
|
|
the OpenCode session + returns the parsed worker output.
|
|
|
|
The injected ``agent_runner`` parameter to ``run_one_attempt``
|
|
(``runner.py``) is what production wires to this module. Tests use a
|
|
synthetic agent_runner.
|
|
|
|
Lifecycle per attempt (plan v9 per-attempt subprocess model):
|
|
1. Spawn the matching role-builder MCP subprocess (one of
|
|
``tools/controller/mcp/{role}_builder.py``) with stdout captured
|
|
so we can read the finalize'd JSON.
|
|
2. Tell the OpenCode session to run the role's agent, pointed at the
|
|
spawned MCP server (path passed via env / args).
|
|
3. Wait for the OpenCode session to complete + the MCP subprocess
|
|
to emit one JSON line on stdout.
|
|
4. Strict-parse against the V1 output contract for the role.
|
|
5. Cleanup: terminate the MCP subprocess (in case it didn't exit
|
|
cleanly after finalize).
|
|
|
|
The actual OpenCode session call is parameterized as
|
|
``run_opencode_session`` so this module is testable without a live
|
|
OpenCode. Production wires it to ``tools/_opencode_worker.run_session_blocking``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from ..contracts.parse import ContractValidationError, strict_parse
|
|
from ..contracts.v1 import (
|
|
ConflictResolverOutputV1,
|
|
EstimatorOutputV1,
|
|
ImplementerOutputV1,
|
|
ReviewerOutputV1,
|
|
SummarizerOutputV1,
|
|
)
|
|
from .legacy_adapter import adapt_to_v1
|
|
from .runner import WorkerError, WorkerLostLock
|
|
from .session_sidecar import (
|
|
WorkerSession,
|
|
now_iso,
|
|
read_proc_starttime,
|
|
write_sidecar,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Module path of each role's MCP server (Python -m runnable).
|
|
ROLE_TO_MCP_MODULE: dict[str, str] = {
|
|
"implementer": "tools.controller.mcp.implementer_builder",
|
|
"reviewer": "tools.controller.mcp.reviewer_builder",
|
|
"estimator": "tools.controller.mcp.estimator_builder",
|
|
"conflict_resolver": "tools.controller.mcp.conflict_resolver_builder",
|
|
"summarizer": "tools.controller.mcp.summarizer_builder",
|
|
}
|
|
|
|
# The Pydantic model each role's output is parsed against.
|
|
ROLE_TO_OUTPUT_MODEL: dict[str, type] = {
|
|
"implementer": ImplementerOutputV1,
|
|
"reviewer": ReviewerOutputV1,
|
|
"estimator": EstimatorOutputV1,
|
|
"conflict_resolver": ConflictResolverOutputV1,
|
|
"summarizer": SummarizerOutputV1,
|
|
}
|
|
|
|
|
|
# Callable signature for the injected OpenCode session runner.
|
|
# Inputs: role, mcp_subprocess (so the session can spawn-as-child or
|
|
# communicate via stdio), input_payload (already validated).
|
|
# Returns: nothing — the MCP subprocess is what emits the canonical
|
|
# JSON on its own stdout. The session is just the driver.
|
|
#
|
|
# Production wires this to a thin wrapper around
|
|
# ``_opencode_worker.run_session_blocking``.
|
|
OpenCodeSessionRunner = Callable[..., None]
|
|
|
|
|
|
@dataclass
|
|
class MCPSpawnResult:
|
|
"""Captured MCP subprocess + its stdout reader thread."""
|
|
|
|
process: subprocess.Popen
|
|
stdout_buffer: list[str] # mutable; appended by reader thread
|
|
reader_thread: threading.Thread
|
|
|
|
|
|
def production_agent_runner(
|
|
*,
|
|
attempt_id: int,
|
|
role: str,
|
|
tier: int | None,
|
|
input_payload: dict[str, Any],
|
|
instance_id: str,
|
|
lost_lock_check: Callable[[], bool],
|
|
run_opencode_session: OpenCodeSessionRunner,
|
|
python_executable: str = sys.executable,
|
|
mcp_startup_timeout_s: float = 5.0,
|
|
finalize_timeout_s: float = 30.0,
|
|
workspace_dir: Path | None = None,
|
|
opencode_server_url: str = "",
|
|
) -> dict[str, Any]:
|
|
"""The production agent_runner used by run_one_attempt.
|
|
|
|
Maps the role → matching MCP module + output contract; spawns
|
|
the MCP as a subprocess; calls the injected ``run_opencode_session``
|
|
to drive the LLM session; reads the MCP's emitted JSON; strict-
|
|
parses; returns the parsed dict.
|
|
|
|
Tests can call this directly with a synthetic ``run_opencode_session``
|
|
that pokes the MCP's stdin (or otherwise simulates a session). The
|
|
integration with ``_opencode_worker.run_session_blocking`` is the
|
|
production wiring.
|
|
|
|
Raises:
|
|
ValueError: unknown role.
|
|
WorkerError: MCP didn't start / didn't emit valid JSON /
|
|
strict-parse failed twice.
|
|
WorkerLostLock: ``lost_lock_check`` returned True during the run.
|
|
"""
|
|
if role not in ROLE_TO_MCP_MODULE:
|
|
raise ValueError(f"unknown role {role!r}; cannot select MCP")
|
|
mcp_module = ROLE_TO_MCP_MODULE[role]
|
|
model_class = ROLE_TO_OUTPUT_MODEL[role]
|
|
|
|
# Production: per-attempt canonical output path. MCP's
|
|
# finalize_and_emit writes the canonical JSON HERE rather than
|
|
# to stdout (which is shared with the FastMCP JSON-RPC transport).
|
|
import tempfile
|
|
|
|
out_fd, out_path = tempfile.mkstemp(
|
|
prefix=f"controller-canonical-{attempt_id}-",
|
|
suffix=".json",
|
|
)
|
|
os.close(out_fd)
|
|
|
|
spawn = _spawn_mcp_subprocess(
|
|
mcp_module,
|
|
python_executable=python_executable,
|
|
startup_timeout_s=mcp_startup_timeout_s,
|
|
canonical_output_path=out_path,
|
|
)
|
|
|
|
# Patch the input_payload so the prompt builder can interpolate
|
|
# real values (workspace_dir + attempt_id) into the agent's
|
|
# prompt. Without these, the prompt has literal "..." placeholders
|
|
# in the MCP tool-call signatures — the agent then guesses
|
|
# ``attempt_id=1`` every time, the cross-session reset can't tell
|
|
# sessions apart, stale MCP state persists, and every attempt
|
|
# after the first fails with "already finalized". (Real bug
|
|
# observed in trial run-2.) We mutate a shallow copy so the
|
|
# caller's dict isn't side-effected.
|
|
input_payload = dict(input_payload)
|
|
input_payload["attempt_id"] = attempt_id
|
|
if workspace_dir is not None:
|
|
input_payload["workspace_dir"] = str(workspace_dir)
|
|
|
|
# Phase 1k++ (N1): write a per-attempt sidecar so the startup
|
|
# janitor can detect orphaned MCP subprocesses + workspaces if
|
|
# this worker crashes. The starttime is read AFTER spawn so it
|
|
# reflects the actual /proc/{pid}/stat field 22 the janitor
|
|
# later compares against (defends against PID reuse).
|
|
sidecar_path: Path | None = None
|
|
if workspace_dir is not None:
|
|
try:
|
|
sidecar_path = Path(workspace_dir) / "worker.session"
|
|
write_sidecar(
|
|
sidecar_path,
|
|
WorkerSession(
|
|
opencode_server_url=opencode_server_url,
|
|
session_id="", # populated by the session adapter if/when known
|
|
subprocess_pid=spawn.process.pid,
|
|
spawned_by_controller_pid=os.getpid(),
|
|
instance_id=instance_id,
|
|
spawned_at=now_iso(),
|
|
subprocess_starttime=read_proc_starttime(spawn.process.pid),
|
|
),
|
|
)
|
|
except OSError as exc:
|
|
logger.warning(
|
|
"sidecar write failed for attempt_id=%s: %s — janitor "
|
|
"won't be able to detect orphans for this attempt",
|
|
attempt_id,
|
|
exc,
|
|
)
|
|
sidecar_path = None
|
|
|
|
# CA1 stale-file cleanup: the per-PR workspace dir is reused across
|
|
# attempts on the same PR, so a prior attempt's
|
|
# ``{workspace_dir}/{role}_output.json`` would be picked up by the
|
|
# poller as if it were this attempt's output. Wipe BOTH the MCP
|
|
# canonical path (defensive) and the fallback paths before the
|
|
# session starts.
|
|
fallback_paths: list[str] = []
|
|
if workspace_dir is not None:
|
|
fallback_paths.append(str(Path(workspace_dir) / f"{role}_output.json"))
|
|
for stale in [out_path] + fallback_paths:
|
|
try:
|
|
os.unlink(stale)
|
|
except FileNotFoundError:
|
|
pass
|
|
except OSError as exc:
|
|
# PD-R2-7: a stale file we can't remove is a worse failure
|
|
# mode than aborting — the poller would read it as if it
|
|
# were this attempt's output. Hard-fail so the attempt
|
|
# surfaces the underlying permissions/filesystem bug.
|
|
raise WorkerError(
|
|
f"could not unlink stale output file {stale} before session: {exc}",
|
|
outcome="worker-internal-error",
|
|
) from exc
|
|
|
|
try:
|
|
# Hand the OpenCode session the MCP we just spawned + the
|
|
# input payload. The session's job: drive the LLM through
|
|
# its tool-calling sequence; the MCP captures the assembled
|
|
# state + emits canonical JSON via its finalize() tool.
|
|
#
|
|
# Trial-path: legacy-style agents emit a single JSON object
|
|
# as their final response message; the OpenCode worker
|
|
# extracts it into ``SessionResult.parsed_json``. We hand
|
|
# the opencode_session adapter a callback that writes that
|
|
# JSON to the canonical-output file so the post-session poller
|
|
# picks it up uniformly with the MCP + file-write channels.
|
|
import json as _json_mod
|
|
|
|
import time as _time
|
|
|
|
_session_start = _time.monotonic()
|
|
|
|
def _capture_inline_json(parsed: dict) -> None:
|
|
# CA7: skip if we lost the lock — file written after lock-loss
|
|
# would poison a subsequent attempt on this same workspace.
|
|
if lost_lock_check():
|
|
logger.info(
|
|
"inline-output capture skipped (lost lock) "
|
|
"for attempt_id=%s role=%s",
|
|
attempt_id,
|
|
role,
|
|
)
|
|
return
|
|
# PD3: if the MCP already wrote canonical V1 to out_path,
|
|
# do NOT overwrite. The MCP path is authoritative; the
|
|
# inline-chat path is a fallback for agents that don't call
|
|
# the MCP. Inspect any existing file at out_path first.
|
|
for p in [out_path] + list(fallback_paths):
|
|
try:
|
|
with open(p, "r", encoding="utf-8") as f:
|
|
existing = f.read().strip()
|
|
except (FileNotFoundError, OSError):
|
|
continue
|
|
if not existing:
|
|
continue
|
|
try:
|
|
existing_obj = _json_mod.loads(existing)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if (
|
|
isinstance(existing_obj, dict)
|
|
and existing_obj.get("output_version") == "V1"
|
|
):
|
|
logger.debug(
|
|
"MCP already emitted V1 to %s; skipping inline "
|
|
"capture (would overwrite authoritative output)",
|
|
p,
|
|
)
|
|
return
|
|
# Adapt the legacy-shape JSON to the V1 contract before
|
|
# writing. Without this, strict_parse against the V1
|
|
# model rejects the legacy shape (missing output_version,
|
|
# different outcome enum, etc.).
|
|
wallclock = _time.monotonic() - _session_start
|
|
adapted = adapt_to_v1(
|
|
role,
|
|
parsed,
|
|
tier=tier,
|
|
wallclock_seconds=wallclock,
|
|
)
|
|
# PD10: atomic write via .tmp + os.replace so the poller
|
|
# never sees a half-written file.
|
|
tmp_path = f"{out_path}.tmp"
|
|
try:
|
|
with open(tmp_path, "w", encoding="utf-8") as f:
|
|
f.write(_json_mod.dumps(adapted))
|
|
f.write("\n")
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
os.replace(tmp_path, out_path)
|
|
except OSError as exc:
|
|
logger.warning(
|
|
"inline-output capture write failed (%s): canonical "
|
|
"poller will fall through to other channels",
|
|
exc,
|
|
)
|
|
try:
|
|
os.unlink(tmp_path)
|
|
except OSError:
|
|
pass
|
|
|
|
# ``canonical`` holds the role's V1 output JSON once it lands —
|
|
# whether the agent emitted it, or it was salvaged from the
|
|
# worktree after a session failure. None until then.
|
|
canonical: str | None = None
|
|
try:
|
|
run_opencode_session(
|
|
role=role,
|
|
tier=tier,
|
|
input_payload=input_payload,
|
|
mcp_process=spawn.process,
|
|
attempt_id=attempt_id,
|
|
instance_id=instance_id,
|
|
lost_lock_check=lost_lock_check,
|
|
inline_output_callback=_capture_inline_json,
|
|
)
|
|
except WorkerLostLock:
|
|
raise
|
|
except Exception as exc:
|
|
# The session raised — most commonly a tier-budget timeout
|
|
# (opencode_session maps timeout / transport-error to
|
|
# WorkerError). A timeout is the case MOST likely to have a
|
|
# complete committed fix sitting in the worktree: the agent
|
|
# ran out of wallclock, not correctness. Before discarding
|
|
# the attempt as worker-internal-error, run the SAME salvage
|
|
# the canonical-missing path uses — otherwise a timed-out
|
|
# implementer's committed work never reaches CI, because the
|
|
# session exception propagates before the canonical-output
|
|
# wait below ever runs (audit finding #1).
|
|
salvaged = _salvage_implementer_commits(
|
|
role=role,
|
|
input_payload=input_payload,
|
|
workspace_dir=workspace_dir,
|
|
tier=tier,
|
|
wallclock_seconds=_time.monotonic() - _session_start,
|
|
lost_lock_check=lost_lock_check,
|
|
)
|
|
if salvaged is None:
|
|
# Nothing safe to salvage — wrap so the runner's outer
|
|
# handler classifies properly.
|
|
raise WorkerError(
|
|
f"OpenCode session raised: {exc}",
|
|
outcome="worker-internal-error",
|
|
) from exc
|
|
logger.warning(
|
|
"attempt_id=%s role=%s: OpenCode session raised (%s); "
|
|
"salvaged %d commit(s) from the worktree, pushed, and "
|
|
"synthesized outcome=resolved so CI verifies the work",
|
|
attempt_id,
|
|
role,
|
|
exc,
|
|
len(salvaged.get("commit_shas") or []),
|
|
)
|
|
canonical = _json_mod.dumps(salvaged)
|
|
|
|
if canonical is None:
|
|
# The session returned normally. Wait for the canonical V1
|
|
# output to land in one of its channels (MCP path, fallback
|
|
# file, inline-chat capture).
|
|
if lost_lock_check():
|
|
raise WorkerLostLock("lost lock during MCP session")
|
|
canonical = _wait_for_canonical_output(
|
|
spawn,
|
|
out_path,
|
|
timeout_s=finalize_timeout_s,
|
|
fallback_paths=fallback_paths,
|
|
)
|
|
if canonical is None:
|
|
# The OpenCode session ended but no canonical V1 output
|
|
# landed in ANY channel (MCP path, fallback file, inline-
|
|
# chat capture). For an implementer this is often a
|
|
# finished agent that simply never emitted its report —
|
|
# while its actual work, a git commit, sits in the
|
|
# worktree. Discarding the attempt (run-25 did exactly
|
|
# this, losing a correct fix) is worse than salvaging:
|
|
# push the committed work and synthesize a minimal V1
|
|
# output so the controller routes to AWAITING_CI and lets
|
|
# CI judge the code.
|
|
salvaged = _salvage_implementer_commits(
|
|
role=role,
|
|
input_payload=input_payload,
|
|
workspace_dir=workspace_dir,
|
|
tier=tier,
|
|
wallclock_seconds=_time.monotonic() - _session_start,
|
|
lost_lock_check=lost_lock_check,
|
|
)
|
|
if salvaged is None:
|
|
raise WorkerError(
|
|
f"role={role!r} did not emit canonical output within "
|
|
f"{finalize_timeout_s}s (looked at MCP path {out_path} "
|
|
f"+ fallback paths {fallback_paths})",
|
|
outcome="worker-internal-error",
|
|
)
|
|
logger.warning(
|
|
"attempt_id=%s role=%s: canonical output missing; "
|
|
"salvaged %d commit(s) from the worktree, pushed, and "
|
|
"synthesized outcome=resolved so CI verifies the work",
|
|
attempt_id,
|
|
role,
|
|
len(salvaged.get("commit_shas") or []),
|
|
)
|
|
canonical = _json_mod.dumps(salvaged)
|
|
|
|
try:
|
|
parsed = strict_parse(model_class, canonical)
|
|
except ContractValidationError as exc:
|
|
raise WorkerError(
|
|
f"MCP emitted JSON that failed {model_class.__name__} "
|
|
f"strict-parse: {exc}",
|
|
outcome="contract-violation",
|
|
) from exc
|
|
|
|
return parsed.model_dump()
|
|
finally:
|
|
_terminate_mcp_subprocess(spawn)
|
|
# Clean up the canonical output file (best-effort).
|
|
try:
|
|
os.unlink(out_path)
|
|
except OSError:
|
|
pass
|
|
# Sidecar cleanup: the attempt completed (success or failure),
|
|
# so the janitor shouldn't see it as orphaned. Best-effort —
|
|
# if removal fails, the janitor's PID-alive check will skip
|
|
# it cleanly on the next sweep.
|
|
if sidecar_path is not None:
|
|
try:
|
|
sidecar_path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# ─── MCP subprocess lifecycle ────────────────────────────────────────
|
|
|
|
|
|
def _spawn_mcp_subprocess(
|
|
mcp_module: str,
|
|
*,
|
|
python_executable: str,
|
|
startup_timeout_s: float,
|
|
canonical_output_path: str,
|
|
) -> MCPSpawnResult:
|
|
"""Spawn ``python -m {mcp_module}`` with stdout captured via a
|
|
background reader thread.
|
|
|
|
``canonical_output_path`` is the file the MCP writes its canonical
|
|
JSON to (via the CONTROLLER_CANONICAL_OUTPUT_PATH env var).
|
|
Stdout is still captured (for transport/debug logging) but the
|
|
authoritative output goes to the file.
|
|
"""
|
|
repo_root = str(Path(__file__).resolve().parents[3])
|
|
env = dict(os.environ)
|
|
# Prepend repo_root to PYTHONPATH so ``-m tools.controller.mcp....`` works.
|
|
existing_pp = env.get("PYTHONPATH", "")
|
|
env["PYTHONPATH"] = (
|
|
f"{repo_root}{os.pathsep}{existing_pp}" if existing_pp else repo_root
|
|
)
|
|
env["CONTROLLER_CANONICAL_OUTPUT_PATH"] = canonical_output_path
|
|
proc = subprocess.Popen(
|
|
[python_executable, "-m", mcp_module],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
cwd=repo_root,
|
|
)
|
|
|
|
stdout_buffer: list[str] = []
|
|
reader_thread = threading.Thread(
|
|
target=_drain_stream_into_list,
|
|
args=(proc.stdout, stdout_buffer),
|
|
name=f"mcp-stdout-{proc.pid}",
|
|
daemon=True,
|
|
)
|
|
reader_thread.start()
|
|
|
|
# Best-effort startup check: did the process die immediately?
|
|
# Don't actually wait the full timeout — production OpenCode sessions
|
|
# are long-running; we just verify the subprocess is alive.
|
|
if proc.poll() is not None:
|
|
raise WorkerError(
|
|
f"MCP subprocess died immediately (returncode={proc.returncode})",
|
|
outcome="worker-internal-error",
|
|
)
|
|
|
|
return MCPSpawnResult(
|
|
process=proc,
|
|
stdout_buffer=stdout_buffer,
|
|
reader_thread=reader_thread,
|
|
)
|
|
|
|
|
|
def _drain_stream_into_list(stream, target: list[str]) -> None:
|
|
"""Read lines from ``stream`` into ``target`` until EOF.
|
|
|
|
Runs in a background thread. Each line (without trailing newline)
|
|
is appended. Robust against the stream closing mid-read.
|
|
"""
|
|
try:
|
|
for line in iter(stream.readline, ""):
|
|
if not line:
|
|
break
|
|
target.append(line.rstrip("\n"))
|
|
except (ValueError, OSError):
|
|
# Stream closed; thread exits.
|
|
pass
|
|
|
|
|
|
def _wait_for_canonical_output(
|
|
spawn: MCPSpawnResult,
|
|
out_path: str,
|
|
*,
|
|
timeout_s: float,
|
|
fallback_paths: list[str] | None = None,
|
|
) -> str | None:
|
|
"""Poll the canonical-output file. Returns its content on first
|
|
non-empty read, or None on timeout.
|
|
|
|
``fallback_paths`` are additional paths to poll alongside
|
|
``out_path``. Used during the trial phase to support direct
|
|
file-write output from agents that can't call the MCP (the
|
|
response-builder MCPs aren't wired into OpenCode yet — see
|
|
prompts.py for the contract the agent follows).
|
|
|
|
The MCP writes the file atomically (open + write + fsync + close)
|
|
in its finalize_and_emit. We poll until the file exists + has
|
|
content OR the subprocess has been gone long enough to indicate
|
|
it's not going to write.
|
|
"""
|
|
import json as _json
|
|
import os as _os
|
|
import time
|
|
|
|
paths_to_poll = [out_path] + list(fallback_paths or [])
|
|
|
|
def _try_read_stable(p: str) -> str | None:
|
|
"""Read ``p`` only if size is stable across two stats AND the
|
|
content parses as JSON. Defends against partial writes (agent
|
|
crashed mid-flush; the file has truncated text). Returns the
|
|
content string on success; None if the file is missing, empty,
|
|
in-flight, or unparseable.
|
|
|
|
Round-4 P1 fix: the previous version did ``f.read().strip()``
|
|
and returned anything non-empty — a half-written file's
|
|
partial JSON would then trip ``ContractValidationError`` →
|
|
``worker-internal-error`` with no record of WHICH path.
|
|
"""
|
|
try:
|
|
sz1 = _os.stat(p).st_size
|
|
except (FileNotFoundError, OSError):
|
|
return None
|
|
if sz1 == 0:
|
|
return None
|
|
# Give the writer a moment in case the flush is still landing.
|
|
time.sleep(0.02)
|
|
try:
|
|
sz2 = _os.stat(p).st_size
|
|
except (FileNotFoundError, OSError):
|
|
return None
|
|
if sz1 != sz2:
|
|
return None # size still moving; wait for the next poll
|
|
try:
|
|
with open(p, "r", encoding="utf-8") as f:
|
|
content = f.read().strip()
|
|
except (FileNotFoundError, OSError):
|
|
return None
|
|
if not content:
|
|
return None
|
|
# Validate it parses as JSON — partial writes often produce
|
|
# syntactically broken content that strict_parse would later
|
|
# reject with a confusing error far from the read site.
|
|
try:
|
|
_json.loads(content)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return content
|
|
|
|
def _try_read_any() -> tuple[str | None, str | None]:
|
|
"""Returns (content, source_path) of the first stable+valid
|
|
path, or (None, None) if nothing's ready."""
|
|
for p in paths_to_poll:
|
|
content = _try_read_stable(p)
|
|
if content:
|
|
return content, p
|
|
return None, None
|
|
|
|
start = time.monotonic()
|
|
last_source: str | None = None
|
|
while True:
|
|
content, source = _try_read_any()
|
|
if content:
|
|
last_source = source
|
|
logger.debug(
|
|
"canonical output read from %s (%d bytes)",
|
|
source,
|
|
len(content),
|
|
)
|
|
return content
|
|
|
|
# Subprocess gone + file still empty → give it one more brief
|
|
# window for the final flush to land, then give up.
|
|
if spawn.process.poll() is not None:
|
|
time.sleep(0.1)
|
|
content, source = _try_read_any()
|
|
if source:
|
|
logger.debug(
|
|
"canonical output read from %s after subprocess exit",
|
|
source,
|
|
)
|
|
return content
|
|
|
|
if time.monotonic() - start > timeout_s:
|
|
logger.warning(
|
|
"canonical output timeout after %.1fs (looked at: %s); "
|
|
"last successful read source: %s",
|
|
timeout_s,
|
|
paths_to_poll,
|
|
last_source,
|
|
)
|
|
return None
|
|
time.sleep(0.05)
|
|
|
|
|
|
def _terminate_mcp_subprocess(spawn: MCPSpawnResult) -> None:
|
|
"""Send SIGTERM + 2s grace + SIGKILL to ensure cleanup. Drain
|
|
the reader thread."""
|
|
proc = spawn.process
|
|
if proc.poll() is None:
|
|
try:
|
|
proc.terminate()
|
|
except ProcessLookupError:
|
|
pass
|
|
try:
|
|
proc.wait(timeout=2.0)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
proc.kill()
|
|
except ProcessLookupError:
|
|
pass
|
|
try:
|
|
proc.wait(timeout=2.0)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
# Reader thread will exit on its own when stdout closes.
|
|
spawn.reader_thread.join(timeout=2.0)
|
|
|
|
|
|
# ─── canonical-output salvage ────────────────────────────────────────
|
|
|
|
|
|
def _git_in_worktree(worktree: Path, *args: str, timeout: float = 120.0) -> str:
|
|
"""Run a git command in ``worktree``; return stdout. Raises
|
|
``RuntimeError`` on non-zero exit or timeout."""
|
|
try:
|
|
result = subprocess.run(
|
|
["git", "-C", str(worktree), *args],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
timeout=timeout,
|
|
)
|
|
except subprocess.CalledProcessError as exc:
|
|
raise RuntimeError(
|
|
f"git {' '.join(args)} failed: {(exc.stderr or '').strip()}"
|
|
) from exc
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise RuntimeError(f"git {' '.join(args)} timed out") from exc
|
|
return result.stdout
|
|
|
|
|
|
def _salvage_implementer_commits(
|
|
*,
|
|
role: str,
|
|
input_payload: dict[str, Any],
|
|
workspace_dir: Path | None,
|
|
tier: int | None,
|
|
wallclock_seconds: float,
|
|
lost_lock_check: Callable[[], bool],
|
|
) -> dict[str, Any] | None:
|
|
"""Recover a finished-but-unreported implementer attempt.
|
|
|
|
When an implementer's OpenCode session ends cleanly but no
|
|
canonical V1 output is emitted, the agent's actual work — a git
|
|
commit — is still in the worktree. Rather than discard the attempt
|
|
as worker-internal-error (run-25 did exactly this, losing a correct
|
|
fix), push those commits to the PR head branch and synthesize a
|
|
minimal ``ImplementerOutputV1`` with ``outcome=resolved`` so the
|
|
controller routes the workflow to AWAITING_CI and lets CI judge the
|
|
code.
|
|
|
|
``lost_lock_check`` is honored before touching the worktree and
|
|
again immediately before the push: the canonical-output wait can
|
|
span the lock TTL, and if the lock was reaped a fresh worker may
|
|
already own this workspace — racing its ``git reset``/``clean`` with
|
|
our git ops would corrupt the shared worktree. This mirrors
|
|
``_capture_inline_json``'s CA7 guard.
|
|
|
|
Returns the synthetic output dict on a successful salvage, or None
|
|
when there is nothing safe to salvage (not an implementer, lock
|
|
lost, no commits, or the push could not land — in which case the
|
|
caller raises worker-internal-error and the next attempt adopts the
|
|
work from auto-scratch/pr-<N>).
|
|
"""
|
|
if role != "implementer" or workspace_dir is None:
|
|
return None
|
|
if lost_lock_check():
|
|
logger.info(
|
|
"salvage skipped: lock lost — the workspace may have a new "
|
|
"owner; touching the shared worktree now would race it"
|
|
)
|
|
return None
|
|
worktree = Path(workspace_dir) / "worktree"
|
|
if not (worktree / ".git").exists():
|
|
return None
|
|
base = input_payload.get("head_sha")
|
|
head_ref = input_payload.get("head_ref")
|
|
if not isinstance(base, str) or not base:
|
|
return None
|
|
if not isinstance(head_ref, str) or not head_ref:
|
|
return None
|
|
try:
|
|
head = _git_in_worktree(worktree, "rev-parse", "HEAD").strip()
|
|
except RuntimeError as exc:
|
|
logger.warning("salvage: could not read worktree HEAD: %s", exc)
|
|
return None
|
|
if not head or head == base:
|
|
return None # agent produced no commits — nothing to salvage
|
|
try:
|
|
# --reverse → oldest-first, so commit_shas[-1] is the branch
|
|
# tip; runner.py derives head_sha_after from commit_shas[-1].
|
|
commit_shas = _git_in_worktree(
|
|
worktree, "rev-list", "--reverse", f"{base}..{head}"
|
|
).split()
|
|
files_touched = [
|
|
line.strip()
|
|
for line in _git_in_worktree(
|
|
worktree, "diff", "--name-only", f"{base}..{head}"
|
|
).splitlines()
|
|
if line.strip()
|
|
]
|
|
except RuntimeError as exc:
|
|
logger.warning("salvage: could not enumerate commits/files: %s", exc)
|
|
return None
|
|
if not commit_shas or not files_touched:
|
|
return None
|
|
# Re-check the lock right before the push: the rev-list/diff above
|
|
# take time, and the push is the impactful side-effect.
|
|
if lost_lock_check():
|
|
logger.info("salvage aborted before push: lock lost")
|
|
return None
|
|
# Ensure the work is on the remote — CI runs from the PR head
|
|
# branch. A plain push fast-forwards (agent never pushed) or is a
|
|
# no-op (agent already pushed); a genuine divergence fails, and we
|
|
# bail rather than clobber a concurrent push.
|
|
try:
|
|
_git_in_worktree(
|
|
worktree,
|
|
"push",
|
|
"origin",
|
|
f"HEAD:refs/heads/{head_ref}",
|
|
timeout=180.0,
|
|
)
|
|
except RuntimeError as exc:
|
|
logger.warning(
|
|
"salvage: push to origin/%s failed (%s); leaving the work for "
|
|
"the next attempt to adopt from auto-scratch/pr-<N>",
|
|
head_ref,
|
|
exc,
|
|
)
|
|
return None
|
|
return {
|
|
"output_version": "V1",
|
|
"outcome": "resolved",
|
|
"files_touched": files_touched,
|
|
"commit_shas": commit_shas,
|
|
"confidence": "low",
|
|
"blockers": [],
|
|
"used_tier": tier if isinstance(tier, int) and tier in (0, 1, 2) else 0,
|
|
"disputed_review_id": None,
|
|
"disputed_blocker_index": None,
|
|
"dispute_evidence": None,
|
|
"verified_at_sha": None,
|
|
"wallclock_seconds": max(0.0, wallclock_seconds),
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"MCPSpawnResult",
|
|
"OpenCodeSessionRunner",
|
|
"ROLE_TO_MCP_MODULE",
|
|
"ROLE_TO_OUTPUT_MODEL",
|
|
"production_agent_runner",
|
|
]
|