cleveragents-core/tools/controller/worker/agent_runner.py

"""Production agent_runner adapter — spawns the role-builder MCP +
the OpenCode session + returns the parsed worker output.

The injected ``agent_runner`` parameter to ``run_one_attempt``
(``runner.py``) is what production wires to this module. Tests use a
synthetic agent_runner.

Lifecycle per attempt (plan v9 per-attempt subprocess model):
1. Spawn the matching role-builder MCP subprocess (one of
   ``tools/controller/mcp/{role}_builder.py``) with stdout captured
   so we can read the finalize'd JSON.
2. Tell the OpenCode session to run the role's agent, pointed at the
   spawned MCP server (path passed via env / args).
3. Wait for the OpenCode session to complete + the MCP subprocess
   to emit one JSON line on stdout.
4. Strict-parse against the V1 output contract for the role.
5. Cleanup: terminate the MCP subprocess (in case it didn't exit
   cleanly after finalize).

The actual OpenCode session call is parameterized as
``run_opencode_session`` so this module is testable without a live
OpenCode. Production wires it to ``tools/_opencode_worker.run_session_blocking``.
"""
from __future__ import annotations

import logging
import os
import signal
import subprocess
import sys
import threading
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from ..contracts.parse import ContractValidationError, strict_parse
from ..contracts.v1 import (
    ConflictResolverOutputV1,
    EstimatorOutputV1,
    ImplementerOutputV1,
    ReviewerOutputV1,
    SummarizerOutputV1,
)
from .legacy_adapter import adapt_to_v1
from .runner import WorkerError, WorkerLostLock
from .session_sidecar import (
    WorkerSession,
    now_iso,
    read_proc_starttime,
    write_sidecar,
)

logger = logging.getLogger(__name__)


# Module path of each role's MCP server (Python -m runnable).
ROLE_TO_MCP_MODULE: dict[str, str] = {
    "implementer": "tools.controller.mcp.implementer_builder",
    "reviewer": "tools.controller.mcp.reviewer_builder",
    "estimator": "tools.controller.mcp.estimator_builder",
    "conflict_resolver": "tools.controller.mcp.conflict_resolver_builder",
    "summarizer": "tools.controller.mcp.summarizer_builder",
}

# The Pydantic model each role's output is parsed against.
ROLE_TO_OUTPUT_MODEL: dict[str, type] = {
    "implementer": ImplementerOutputV1,
    "reviewer": ReviewerOutputV1,
    "estimator": EstimatorOutputV1,
    "conflict_resolver": ConflictResolverOutputV1,
    "summarizer": SummarizerOutputV1,
}


# Callable signature for the injected OpenCode session runner.
# Inputs: role, mcp_subprocess (so the session can spawn-as-child or
# communicate via stdio), input_payload (already validated).
# Returns: nothing — the MCP subprocess is what emits the canonical
# JSON on its own stdout. The session is just the driver.
#
# Production wires this to a thin wrapper around
# ``_opencode_worker.run_session_blocking``.
OpenCodeSessionRunner = Callable[..., None]


@dataclass
class MCPSpawnResult:
    """Captured MCP subprocess + its stdout reader thread."""

    process: subprocess.Popen
    stdout_buffer: list[str]  # mutable; appended by reader thread
    reader_thread: threading.Thread


def production_agent_runner(
    *,
    attempt_id: int,
    role: str,
    tier: int | None,
    input_payload: dict[str, Any],
    instance_id: str,
    lost_lock_check: Callable[[], bool],
    run_opencode_session: OpenCodeSessionRunner,
    python_executable: str = sys.executable,
    mcp_startup_timeout_s: float = 5.0,
    finalize_timeout_s: float = 30.0,
    workspace_dir: Path | None = None,
    opencode_server_url: str = "",
) -> dict[str, Any]:
    """The production agent_runner used by run_one_attempt.

    Maps the role → matching MCP module + output contract; spawns
    the MCP as a subprocess; calls the injected ``run_opencode_session``
    to drive the LLM session; reads the MCP's emitted JSON; strict-
    parses; returns the parsed dict.

    Tests can call this directly with a synthetic ``run_opencode_session``
    that pokes the MCP's stdin (or otherwise simulates a session). The
    integration with ``_opencode_worker.run_session_blocking`` is the
    production wiring.

    Raises:
        ValueError: unknown role.
        WorkerError: MCP didn't start / didn't emit valid JSON /
                     strict-parse failed twice.
        WorkerLostLock: ``lost_lock_check`` returned True during the run.
    """
    if role not in ROLE_TO_MCP_MODULE:
        raise ValueError(f"unknown role {role!r}; cannot select MCP")
    mcp_module = ROLE_TO_MCP_MODULE[role]
    model_class = ROLE_TO_OUTPUT_MODEL[role]

    # Production: per-attempt canonical output path. MCP's
    # finalize_and_emit writes the canonical JSON HERE rather than
    # to stdout (which is shared with the FastMCP JSON-RPC transport).
    import tempfile
    out_fd, out_path = tempfile.mkstemp(
        prefix=f"controller-canonical-{attempt_id}-", suffix=".json",
    )
    os.close(out_fd)

    spawn = _spawn_mcp_subprocess(
        mcp_module,
        python_executable=python_executable,
        startup_timeout_s=mcp_startup_timeout_s,
        canonical_output_path=out_path,
    )

    # Patch the input_payload so the prompt builder can interpolate
    # real values (workspace_dir + attempt_id) into the agent's
    # prompt. Without these, the prompt has literal "..." placeholders
    # in the MCP tool-call signatures — the agent then guesses
    # ``attempt_id=1`` every time, the cross-session reset can't tell
    # sessions apart, stale MCP state persists, and every attempt
    # after the first fails with "already finalized". (Real bug
    # observed in trial run-2.) We mutate a shallow copy so the
    # caller's dict isn't side-effected.
    input_payload = dict(input_payload)
    input_payload["attempt_id"] = attempt_id
    if workspace_dir is not None:
        input_payload["workspace_dir"] = str(workspace_dir)

    # Phase 1k++ (N1): write a per-attempt sidecar so the startup
    # janitor can detect orphaned MCP subprocesses + workspaces if
    # this worker crashes. The starttime is read AFTER spawn so it
    # reflects the actual /proc/{pid}/stat field 22 the janitor
    # later compares against (defends against PID reuse).
    sidecar_path: Path | None = None
    if workspace_dir is not None:
        try:
            sidecar_path = Path(workspace_dir) / "worker.session"
            write_sidecar(sidecar_path, WorkerSession(
                opencode_server_url=opencode_server_url,
                session_id="",  # populated by the session adapter if/when known
                subprocess_pid=spawn.process.pid,
                spawned_by_controller_pid=os.getpid(),
                instance_id=instance_id,
                spawned_at=now_iso(),
                subprocess_starttime=read_proc_starttime(spawn.process.pid),
            ))
        except OSError as exc:
            logger.warning(
                "sidecar write failed for attempt_id=%s: %s — janitor "
                "won't be able to detect orphans for this attempt",
                attempt_id, exc,
            )
            sidecar_path = None

    # CA1 stale-file cleanup: the per-PR workspace dir is reused across
    # attempts on the same PR, so a prior attempt's
    # ``{workspace_dir}/{role}_output.json`` would be picked up by the
    # poller as if it were this attempt's output. Wipe BOTH the MCP
    # canonical path (defensive) and the fallback paths before the
    # session starts.
    fallback_paths: list[str] = []
    if workspace_dir is not None:
        fallback_paths.append(
            str(Path(workspace_dir) / f"{role}_output.json")
        )
    for stale in [out_path] + fallback_paths:
        try:
            os.unlink(stale)
        except FileNotFoundError:
            pass
        except OSError as exc:
            # PD-R2-7: a stale file we can't remove is a worse failure
            # mode than aborting — the poller would read it as if it
            # were this attempt's output. Hard-fail so the attempt
            # surfaces the underlying permissions/filesystem bug.
            raise WorkerError(
                f"could not unlink stale output file {stale} before "
                f"session: {exc}",
                outcome="worker-internal-error",
            ) from exc

    try:
        # Hand the OpenCode session the MCP we just spawned + the
        # input payload. The session's job: drive the LLM through
        # its tool-calling sequence; the MCP captures the assembled
        # state + emits canonical JSON via its finalize() tool.
        #
        # Trial-path: legacy-style agents emit a single JSON object
        # as their final response message; the OpenCode worker
        # extracts it into ``SessionResult.parsed_json``. We hand
        # the opencode_session adapter a callback that writes that
        # JSON to the canonical-output file so the post-session poller
        # picks it up uniformly with the MCP + file-write channels.
        import json as _json_mod

        import time as _time
        _session_start = _time.monotonic()

        def _capture_inline_json(parsed: dict) -> None:
            # CA7: skip if we lost the lock — file written after lock-loss
            # would poison a subsequent attempt on this same workspace.
            if lost_lock_check():
                logger.info(
                    "inline-output capture skipped (lost lock) "
                    "for attempt_id=%s role=%s", attempt_id, role,
                )
                return
            # PD3: if the MCP already wrote canonical V1 to out_path,
            # do NOT overwrite. The MCP path is authoritative; the
            # inline-chat path is a fallback for agents that don't call
            # the MCP. Inspect any existing file at out_path first.
            for p in [out_path] + list(fallback_paths):
                try:
                    with open(p, "r", encoding="utf-8") as f:
                        existing = f.read().strip()
                except (FileNotFoundError, OSError):
                    continue
                if not existing:
                    continue
                try:
                    existing_obj = _json_mod.loads(existing)
                except (TypeError, ValueError):
                    continue
                if (
                    isinstance(existing_obj, dict)
                    and existing_obj.get("output_version") == "V1"
                ):
                    logger.debug(
                        "MCP already emitted V1 to %s; skipping inline "
                        "capture (would overwrite authoritative output)",
                        p,
                    )
                    return
            # Adapt the legacy-shape JSON to the V1 contract before
            # writing. Without this, strict_parse against the V1
            # model rejects the legacy shape (missing output_version,
            # different outcome enum, etc.).
            wallclock = _time.monotonic() - _session_start
            adapted = adapt_to_v1(
                role, parsed, tier=tier, wallclock_seconds=wallclock,
            )
            # PD10: atomic write via .tmp + os.replace so the poller
            # never sees a half-written file.
            tmp_path = f"{out_path}.tmp"
            try:
                with open(tmp_path, "w", encoding="utf-8") as f:
                    f.write(_json_mod.dumps(adapted))
                    f.write("\n")
                    f.flush()
                    os.fsync(f.fileno())
                os.replace(tmp_path, out_path)
            except OSError as exc:
                logger.warning(
                    "inline-output capture write failed (%s): canonical "
                    "poller will fall through to other channels",
                    exc,
                )
                try:
                    os.unlink(tmp_path)
                except OSError:
                    pass

        try:
            run_opencode_session(
                role=role, tier=tier, input_payload=input_payload,
                mcp_process=spawn.process,
                attempt_id=attempt_id, instance_id=instance_id,
                lost_lock_check=lost_lock_check,
                inline_output_callback=_capture_inline_json,
            )
        except WorkerLostLock:
            raise
        except Exception as exc:
            # Wrap so the runner's outer handler classifies properly.
            raise WorkerError(
                f"OpenCode session raised: {exc}",
                outcome="worker-internal-error",
            ) from exc

        if lost_lock_check():
            raise WorkerLostLock("lost lock during MCP session")
        canonical = _wait_for_canonical_output(
            spawn, out_path, timeout_s=finalize_timeout_s,
            fallback_paths=fallback_paths,
        )
        if canonical is None:
            raise WorkerError(
                f"role={role!r} did not emit canonical output within "
                f"{finalize_timeout_s}s (looked at MCP path {out_path} "
                f"+ fallback paths {fallback_paths})",
                outcome="worker-internal-error",
            )

        try:
            parsed = strict_parse(model_class, canonical)
        except ContractValidationError as exc:
            raise WorkerError(
                f"MCP emitted JSON that failed {model_class.__name__} "
                f"strict-parse: {exc}",
                outcome="contract-violation",
            ) from exc

        return parsed.model_dump()
    finally:
        _terminate_mcp_subprocess(spawn)
        # Clean up the canonical output file (best-effort).
        try:
            os.unlink(out_path)
        except OSError:
            pass
        # Sidecar cleanup: the attempt completed (success or failure),
        # so the janitor shouldn't see it as orphaned. Best-effort —
        # if removal fails, the janitor's PID-alive check will skip
        # it cleanly on the next sweep.
        if sidecar_path is not None:
            try:
                sidecar_path.unlink()
            except OSError:
                pass


# ─── MCP subprocess lifecycle ────────────────────────────────────────


def _spawn_mcp_subprocess(
    mcp_module: str, *, python_executable: str, startup_timeout_s: float,
    canonical_output_path: str,
) -> MCPSpawnResult:
    """Spawn ``python -m {mcp_module}`` with stdout captured via a
    background reader thread.

    ``canonical_output_path`` is the file the MCP writes its canonical
    JSON to (via the CONTROLLER_CANONICAL_OUTPUT_PATH env var).
    Stdout is still captured (for transport/debug logging) but the
    authoritative output goes to the file.
    """
    repo_root = str(Path(__file__).resolve().parents[3])
    env = dict(os.environ)
    # Prepend repo_root to PYTHONPATH so ``-m tools.controller.mcp....`` works.
    existing_pp = env.get("PYTHONPATH", "")
    env["PYTHONPATH"] = (
        f"{repo_root}{os.pathsep}{existing_pp}" if existing_pp else repo_root
    )
    env["CONTROLLER_CANONICAL_OUTPUT_PATH"] = canonical_output_path
    proc = subprocess.Popen(
        [python_executable, "-m", mcp_module],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        env=env,
        cwd=repo_root,
    )

    stdout_buffer: list[str] = []
    reader_thread = threading.Thread(
        target=_drain_stream_into_list,
        args=(proc.stdout, stdout_buffer),
        name=f"mcp-stdout-{proc.pid}",
        daemon=True,
    )
    reader_thread.start()

    # Best-effort startup check: did the process die immediately?
    # Don't actually wait the full timeout — production OpenCode sessions
    # are long-running; we just verify the subprocess is alive.
    if proc.poll() is not None:
        raise WorkerError(
            f"MCP subprocess died immediately (returncode={proc.returncode})",
            outcome="worker-internal-error",
        )

    return MCPSpawnResult(
        process=proc, stdout_buffer=stdout_buffer,
        reader_thread=reader_thread,
    )


def _drain_stream_into_list(stream, target: list[str]) -> None:
    """Read lines from ``stream`` into ``target`` until EOF.

    Runs in a background thread. Each line (without trailing newline)
    is appended. Robust against the stream closing mid-read.
    """
    try:
        for line in iter(stream.readline, ""):
            if not line:
                break
            target.append(line.rstrip("\n"))
    except (ValueError, OSError):
        # Stream closed; thread exits.
        pass


def _wait_for_canonical_output(
    spawn: MCPSpawnResult, out_path: str, *, timeout_s: float,
    fallback_paths: list[str] | None = None,
) -> str | None:
    """Poll the canonical-output file. Returns its content on first
    non-empty read, or None on timeout.

    ``fallback_paths`` are additional paths to poll alongside
    ``out_path``. Used during the trial phase to support direct
    file-write output from agents that can't call the MCP (the
    response-builder MCPs aren't wired into OpenCode yet — see
    prompts.py for the contract the agent follows).

    The MCP writes the file atomically (open + write + fsync + close)
    in its finalize_and_emit. We poll until the file exists + has
    content OR the subprocess has been gone long enough to indicate
    it's not going to write.
    """
    import json as _json
    import os as _os
    import time

    paths_to_poll = [out_path] + list(fallback_paths or [])

    def _try_read_stable(p: str) -> str | None:
        """Read ``p`` only if size is stable across two stats AND the
        content parses as JSON. Defends against partial writes (agent
        crashed mid-flush; the file has truncated text). Returns the
        content string on success; None if the file is missing, empty,
        in-flight, or unparseable.

        Round-4 P1 fix: the previous version did ``f.read().strip()``
        and returned anything non-empty — a half-written file's
        partial JSON would then trip ``ContractValidationError`` →
        ``worker-internal-error`` with no record of WHICH path.
        """
        try:
            sz1 = _os.stat(p).st_size
        except (FileNotFoundError, OSError):
            return None
        if sz1 == 0:
            return None
        # Give the writer a moment in case the flush is still landing.
        time.sleep(0.02)
        try:
            sz2 = _os.stat(p).st_size
        except (FileNotFoundError, OSError):
            return None
        if sz1 != sz2:
            return None  # size still moving; wait for the next poll
        try:
            with open(p, "r", encoding="utf-8") as f:
                content = f.read().strip()
        except (FileNotFoundError, OSError):
            return None
        if not content:
            return None
        # Validate it parses as JSON — partial writes often produce
        # syntactically broken content that strict_parse would later
        # reject with a confusing error far from the read site.
        try:
            _json.loads(content)
        except (TypeError, ValueError):
            return None
        return content

    def _try_read_any() -> tuple[str | None, str | None]:
        """Returns (content, source_path) of the first stable+valid
        path, or (None, None) if nothing's ready."""
        for p in paths_to_poll:
            content = _try_read_stable(p)
            if content:
                return content, p
        return None, None

    start = time.monotonic()
    last_source: str | None = None
    while True:
        content, source = _try_read_any()
        if content:
            last_source = source
            logger.debug(
                "canonical output read from %s (%d bytes)",
                source, len(content),
            )
            return content

        # Subprocess gone + file still empty → give it one more brief
        # window for the final flush to land, then give up.
        if spawn.process.poll() is not None:
            time.sleep(0.1)
            content, source = _try_read_any()
            if source:
                logger.debug(
                    "canonical output read from %s after subprocess exit",
                    source,
                )
            return content

        if time.monotonic() - start > timeout_s:
            logger.warning(
                "canonical output timeout after %.1fs (looked at: %s); "
                "last successful read source: %s",
                timeout_s, paths_to_poll, last_source,
            )
            return None
        time.sleep(0.05)


def _terminate_mcp_subprocess(spawn: MCPSpawnResult) -> None:
    """Send SIGTERM + 2s grace + SIGKILL to ensure cleanup. Drain
    the reader thread."""
    proc = spawn.process
    if proc.poll() is None:
        try:
            proc.terminate()
        except ProcessLookupError:
            pass
        try:
            proc.wait(timeout=2.0)
        except subprocess.TimeoutExpired:
            try:
                proc.kill()
            except ProcessLookupError:
                pass
            try:
                proc.wait(timeout=2.0)
            except subprocess.TimeoutExpired:
                pass
    # Reader thread will exit on its own when stdout closes.
    spawn.reader_thread.join(timeout=2.0)


__all__ = [
    "MCPSpawnResult",
    "OpenCodeSessionRunner",
    "ROLE_TO_MCP_MODULE",
    "ROLE_TO_OUTPUT_MODEL",
    "production_agent_runner",
]