"""Shared runtime for deterministic review / implementation dispatchers. This module is intentionally small: it reuses the Phase A ``_claim_runtime.py`` and ``_opencode_worker.py`` primitives, runs the existing ``list_prs_*`` TypeScript wrappers, claims PR work before dispatch, and records one SQLite row per dispatched item. The LLM workers still make the review / implementation decisions; Python owns only orchestration. Extensibility: the ``WorkGroup`` seam ------------------------------------- The dispatchers (``dispatch_review.py``, ``dispatch_implementer.py``) add pipeline-specific behaviour by injecting :class:`WorkGroup` dataclass instances into this module's generic loop. Each ``WorkGroup`` carries: - ``script_name`` — the TS candidate-list script to run. - ``item_kind`` / ``claim_kind`` — what the items are and which ``auto/claimed-*`` family they get. - ``worker_agent`` / ``tag_prefix`` — the OpenCode agent name and the ``[AUTO-XXX-PR-N]`` session-tag prefix. - ``prompt_factory`` — a callable that builds the worker prompt. - ``post_session_action`` — a callable that handles the ``SessionResult`` after the worker exits. A new pipeline is added by declaring a new ``WorkGroup`` (or a list of them, one per priority bucket) and passing it to :func:`run_outer_loop`. The runtime then handles claim acquisition, single-instance locking, signal-driven release, cycle bookkeeping, SQLite rows, the cycle-failure budget, and the G5 startup PAT probe uniformly for every pipeline. This **is** the same idea ``agents/final-working``'s thin-wrapper supervisors (``implementation-supervisor.md`` ↔ ``pr-review-supervisor.md`` ↔ ``pr-merge-supervisor.md``) used to express in markdown — three wrappers over a generic ``supervisor`` agent. The dmpipeline shape expresses the same pattern in Python: three drivers over a generic runtime. Future pipelines (e.g. a grooming driver if the G1/G7 harvest extension is wired) should be added the same way — a new ``WorkGroup`` list and a thin entry-point module that calls :func:`run_outer_loop`. Do NOT replicate the cycle loop, claim machinery, lock setup, or PAT validation in a new module; reuse the seam. Refs: ``docs/development/final-working-harvest-plan.md`` (A7). """ from __future__ import annotations import json import logging import os import signal import sqlite3 import subprocess import sys import threading import time import uuid from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any # Make ``tools/`` importable so we can pull the shared sibling loader by # regular import rather than reimplementing the file-load dance inline. # Both ``python tools/...`` invocation (sys.path[0] is already tools/) and # pytest's ``spec_from_file_location`` loader (no implicit tools/ on path) # work after this idempotent insert. _TOOLS_DIR = str(Path(__file__).resolve().parent) if _TOOLS_DIR not in sys.path: sys.path.insert(0, _TOOLS_DIR) from _loader import ( # noqa: E402 type: ignore[import-not-found] load_sibling as _load_sibling, ) logger = logging.getLogger("dispatch_runtime") _claim_runtime = _load_sibling("_claim_runtime", "_claim_runtime.py") _pipeline_cache = _load_sibling("_pipeline_cache", "_pipeline_cache.py") _opencode_worker = _load_sibling("_opencode_worker", "_opencode_worker.py") _cycle_cap = _load_sibling("_cycle_cap", "_cycle_cap.py") REPO_ROOT = Path(__file__).resolve().parent.parent SCRIPT_DIR = REPO_ROOT / ".opencode" / "skills" / "auto-agents-system" / "scripts" API_BASE = _claim_runtime.API_BASE REPO_OWNER = _claim_runtime.REPO_OWNER REPO_NAME = _claim_runtime.REPO_NAME CLAIM_COMMENT_MARKER = "" def _heartbeat_log_interval_seconds() -> float: """Return the cadence between operator-visible "worker still in-flight" log lines emitted from the ``on_poll`` callback during ``run_session_blocking``. Read from ``DISPATCHER_HEARTBEAT_LOG_INTERVAL_SECONDS`` when set; falls back to 120s — long enough that a 20-minute cycle emits only 9–10 lines (readable in a terminal), short enough that a silent operator-side log past 120s really does mean the dispatcher is hung. Negative / unparseable values fall back to the default. Tests can dial this to ``0.1`` to exercise the log path without sleeping for two minutes. **Startup-only knob.** The module-level ``_HEARTBEAT_LOG_INTERVAL_SECONDS`` constant captures this function's return value at module-import time and the ``dispatch_one`` heartbeat loop reads the module-global, NOT this function. Operators changing the env var on a running dispatcher (``systemctl set-environment`` etc.) MUST restart the process for the new value to take effect. This is a deliberate trade-off: re-reading the env var on every poll (~once every 2 s) would add a syscall per poll for a knob that almost nobody tunes in production. Tests bypass the constant by reloading the runtime module (see ``_reload_runtime_with_interval`` in ``tests/auto_agents/test_dispatch_runtime.py``). """ raw = os.environ.get("DISPATCHER_HEARTBEAT_LOG_INTERVAL_SECONDS") if raw is None: return 120.0 try: value = float(raw) except (TypeError, ValueError): return 120.0 if value < 0: return 120.0 return value # Module-level constant captured at import time. See # :func:`_heartbeat_log_interval_seconds` for the startup-only caveat. _HEARTBEAT_LOG_INTERVAL_SECONDS = _heartbeat_log_interval_seconds() # Cap on the worker raw-response excerpt we quote into a Forgejo claim # release comment. Worker output can be arbitrarily long and may contain # tool-call traces, escape sequences, or partial Markdown that would # break out of the surrounding code fence; truncate before sanitizing. _RELEASE_DETAIL_MAX_CHARS = 500 def _sanitize_release_detail(raw: str | None) -> str: """Make a best-effort safe excerpt of a worker's raw response for inclusion in a Forgejo release comment. The release comment is operator-facing (and lands in a public Forgejo timeline), so we want a deterministic, readable, fence-safe excerpt rather than a verbatim dump. This: 1. Returns ``""`` when the input is empty / None. 2. Truncates to ``_RELEASE_DETAIL_MAX_CHARS`` *before* the rest of the cleanup so we never spend cycles on multi-MB inputs. 3. Strips control characters except newline and tab — a stray carriage return or escape sequence could otherwise corrupt the comment renderer. 4. Replaces backtick runs of length >= 3 so a worker emitting a triple-backtick code fence cannot close out the surrounding code fence in the release comment template. Note: sanitization is operator-facing hygiene, not a security control. The comment author identity is the bot, not the worker; nothing in the excerpt is interpreted as code. """ if not raw: return "" excerpt = raw[:_RELEASE_DETAIL_MAX_CHARS] cleaned = "".join( ch for ch in excerpt if ch in ("\n", "\t") or (0x20 <= ord(ch) < 0x7F) or ord(ch) > 0x7F ) # Single replace is sufficient: the substitute (modifier-letter # apostrophes) contains no backticks, so it can never produce a # new "```" run for a second pass to find. return cleaned.replace("```", "ʼʼʼ") @dataclass(frozen=True) class WorkGroup: name: str script_name: str item_kind: str claim_kind: str | None worker_agent: str tag_prefix: str prompt_factory: Callable[[DispatchConfig, dict[str, Any], WorkGroup], str] # Optional hook invoked by ``dispatch_one`` after the worker # session completes (and before the claim release). Positional # arguments: ``cfg``, ``item``, the parsed JSON the session # emitted (or None), the raw response text, the dispatcher- # derived ``terminal_state`` ("completed" / "timeout" / # "transport-error" / "dry-run"). Plus a single keyword-only # ``session_context: SessionContext`` carrying the work-group # name and the ISO-8601 timestamps + wall-clock that bracket # ``run_session_blocking`` (see :class:`SessionContext`). # # Hooks that don't need the context accept ``**_`` to absorb # the keyword. The dispatcher always passes ``session_context``; # ``**_kwargs`` only exists for back-compat with hooks written # against an earlier API shape. # # Returns a dict that gets merged into the dispatch outcome # under ``post_session_result``. Errors must be caught by the # action — exceptions propagate and would orphan the claim # release. The reviewer dispatcher uses this hook to POST the # worker's review verdict to Forgejo; the implementer dispatcher # uses it for cleanup + telemetry + status comments. post_session_action: Callable[..., dict[str, Any]] | None = None # When True, the dispatch loop REQUIRES the prompt_factory to # have stashed a non-empty string under # :data:`WORKER_AGENT_OVERRIDE_ITEM_KEY` on the item before the # worker session is spawned. Used by the implementer pool, whose # cycle's agent is resolved per-tier in Python (R3, # 2026-05-17) — a missing override on that path is a code bug # that would otherwise silently fall back to the static # ``worker_agent`` (the wrong tier). Review / conflict pools # leave this False because their worker is the same every cycle. requires_worker_agent_override: bool = False # Per-item override key checked by :func:`_resolve_effective_worker_agent`. # A prompt_factory MAY stash a string under this key on the item dict to # tell the dispatcher to invoke a different agent than the group's static # ``worker_agent`` for THIS cycle only. Production usage: the implementer # dispatcher resolves the cycle's escalation tier in Python and stashes # the matching ``task-implementor-tier-`` variant name here, # replacing the (retired) tier-dispatcher + tier-N wrapper agents (R3, # 2026-05-17). # # Exported (no leading underscore) so callers in higher layers # (``dispatch_implementer``) can import the SAME string literal — a # rename here would otherwise silently desync from the writer side # and the override would stop being read. WORKER_AGENT_OVERRIDE_ITEM_KEY = "_dispatcher_worker_agent_override" def _resolve_effective_worker_agent( group: "WorkGroup", item: dict[str, Any], ) -> str: """Return the agent name to invoke for this cycle's worker session. Lookup order: 1. ``item[WORKER_AGENT_OVERRIDE_ITEM_KEY]`` — set by a prompt_factory that resolved a per-cycle agent (e.g. the implementer's tier-driven variant selection). 2. ``group.worker_agent`` — the static fallback used by drivers whose worker agent is the same for every cycle (reviewer, conflict). The override value is validated as a non-empty string; anything else falls back to ``group.worker_agent`` so a malformed override cannot strand a dispatch cycle silently. Defensive: the static fallback was the only behaviour before this helper existed. When ``group.requires_worker_agent_override`` is True the fallback is treated as a code bug — a missing or malformed override on a group that requires one raises :class:`RuntimeError` immediately. This prevents the implementer pool (whose tier is Python-resolved) from silently running every cycle at the static fallback tier if the prompt_factory regressed. """ override = item.get(WORKER_AGENT_OVERRIDE_ITEM_KEY) if isinstance(override, str) and override.strip(): return override.strip() if group.requires_worker_agent_override: raise RuntimeError( f"WorkGroup {group.name!r} requires a per-cycle " f"worker-agent override under " f"item[{WORKER_AGENT_OVERRIDE_ITEM_KEY!r}], but the " f"prompt_factory did not set one. This is a code bug " f"in the prompt_factory — without the override the " f"dispatcher would silently run at the static fallback " f"agent ({group.worker_agent!r}) which is the wrong " f"tier for the implementer pool." ) return group.worker_agent @dataclass(frozen=True) class SessionContext: """Per-session context the dispatcher captures around ``run_session_blocking`` and threads into :meth:`WorkGroup.post_session_action`. Packing these four kwargs into a single dataclass replaces the earlier "pile of keyword-only arguments" call shape. The dataclass is intentionally frozen + slot-light: the dispatcher owns construction and the post-session action treats it as a read-only record. Fields: - ``work_group_name`` — canonical group name (one of ``failing_ci_pr`` / ``request_changes_pr`` / ``new_issue`` for the implementer; the reviewer-side groups follow the same convention). Equal to ``WorkGroup.name``. - ``session_started_at`` / ``session_completed_at`` — ISO-8601 UTC timestamps bracketing :func:`_opencode_worker.run_session_blocking`. Phase 4 telemetry's ``start_ts`` / ``end_ts`` keys derive from these so they reflect cycle wall-clock. - ``session_wallclock_seconds`` — ``SessionResult.wallclock_seconds`` echoed through; redundant with ``end - start`` but kept for cross-checking against the worker's self-reported timing. - ``subagent_max_depth`` — deepest BFS distance observed in the OpenCode task-tool subagent tree, captured by the ``_opencode_worker._archive_subagent_tree`` BFS walk just before the wrapper session is DELETEd. ``0`` for a flat session, ``N > 0`` for a chain, ``None`` when the walk did not run (archive disabled, transport error). Phase 4 telemetry reads this for the flatten-decision threshold; without it the field always landed as ``None`` even on real multi-tier cycles (see ``docs/development/auto-agents-tier-2-3-plan.md`` § "subagent_max_depth is NOT captured"). """ work_group_name: str session_started_at: str session_completed_at: str session_wallclock_seconds: float subagent_max_depth: int | None = None # Heartbeat refresh callback captured from ``dispatch_one``'s # poll-loop wiring. The implementer dispatcher's in-cycle # escalation runner (``_post_session_action_with_escalation``) # spawns additional worker sessions for Tier 1+; without this # callback, those sessions run with ``on_poll=lambda: None`` and # the dispatcher's heartbeat file goes stale for the duration # of each escalated session. With it threaded through, every # session in the escalation walk refreshes the heartbeat at the # same ~2 s cadence the original Tier 0 session did. ``None`` is # the legacy default; the reviewer dispatcher does not need it # (single session per cycle) and direct unit-test callers pass # ``None`` without consequence. heartbeat_callback: Callable[[], None] | None = None @dataclass(frozen=True) class DispatchConfig: token: str forgejo_url: str owner: str repo: str server_url: str lock_path: Path heartbeat_path: Path cycle_interval_seconds: int max_items_per_cycle: int worker_timeout_seconds: int claim_ttl_seconds: int api_retries: int request_timeout_s: int script_timeout_seconds: int table_name: str dry_run: bool = False cycle_failure_budget: int = 5 # Per-repo policy for ``Closes #N`` / ``ISSUES CLOSED: #N`` # resolution during pre-fetch. See # :mod:`_review_fetch.normalize_linked_issue_policy` for the # accepted values and the rationale for each mode. Defaults to # ``"strict"`` so deployments that have never set the env var # keep the historical behaviour (issue not-found is rendered as # a quality concern the reviewer is expected to surface as # blocking). Fork / test repos that harvest from an upstream # with its own issue tracker should set this to # ``"informational-on-not-found"`` so the reviewer is told the # broken link is informational rather than blocking. linked_issue_policy: str = "strict" def _configure_logging(env_var: str) -> None: level_name = os.environ.get(env_var, "INFO").upper() level = getattr(logging, level_name, logging.INFO) if not logging.getLogger().handlers: logging.basicConfig( level=level, format="%(asctime)s %(name)s %(levelname)s %(message)s", stream=sys.stderr, ) else: logging.getLogger().setLevel(level) def _read_dotenv_value(name: str) -> str | None: for path in (REPO_ROOT / ".devcontainer" / ".env", REPO_ROOT / ".env"): if not path.exists(): continue for line in path.read_text().splitlines(): stripped = line.strip() if not stripped or stripped.startswith("#") or "=" not in stripped: continue key, value = stripped.split("=", 1) if key.strip() == name: return value.strip().strip('"').strip("'") return None def load_secret(*names: str) -> str: for name in names: value = os.environ.get(name) or _read_dotenv_value(name) if value: return value joined = " / ".join(names) raise SystemExit(f"missing required token ({joined})") def derive_forgejo_url() -> str: explicit = os.environ.get("FORGEJO_URL") if explicit: return explicit.rstrip("/") if API_BASE.endswith("/api/v1"): return API_BASE[: -len("/api/v1")] return API_BASE.rstrip("/") def resolve_lock_or_heartbeat(env_var: str, basename: str) -> Path: explicit = os.environ.get(env_var) if explicit: return Path(explicit) candidates: list[Path | None] = [ Path("/var/run") / basename, Path(os.environ.get("XDG_RUNTIME_DIR", "")) / basename if os.environ.get("XDG_RUNTIME_DIR") else None, Path("/tmp") / basename, ] for candidate in candidates: if candidate is None: continue try: candidate.parent.mkdir(parents=True, exist_ok=True) candidate.touch(exist_ok=True) return candidate except (OSError, PermissionError): continue return Path("/tmp") / basename def run_list_script( script_name: str, cfg: DispatchConfig, *, token: str | None = None, ) -> list[dict[str, Any]]: script_path = SCRIPT_DIR / f"{script_name}.ts" if not script_path.exists(): raise RuntimeError(f"work-group script does not exist: {script_path}") cmd = [ "npx", "--yes", "tsx", str(script_path), "--url", cfg.forgejo_url, "--pat", token or cfg.token, "--owner", cfg.owner, "--repo", cfg.repo, ] proc = subprocess.run( cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=cfg.script_timeout_seconds, check=False, ) if proc.returncode != 0: raise RuntimeError( f"{script_name} exited {proc.returncode}: {proc.stderr.strip()}" ) try: payload = json.loads(proc.stdout or "[]") except json.JSONDecodeError as exc: raise RuntimeError( f"{script_name} emitted invalid JSON: {proc.stdout[:500]!r}" ) from exc if not isinstance(payload, list): raise RuntimeError(f"{script_name} emitted non-list JSON") return [item for item in payload if isinstance(item, dict)] def _python_filter_name_for(script_name: str) -> str | None: """Map a ``list_prs_.ts`` script name to the corresponding Python filter name in ``_pr_classification_cache.FILTER_NAMES``. The 5 reviewer work-groups follow the convention ``list_prs_`` ↔ ````. Any other script_name (e.g. ``list_prs_ready_to_merge`` used by merge_drive) returns ``None`` so the caller falls through to the legacy ``run_list_script`` path — those filters aren't in the Python cache yet. """ if not script_name.startswith("list_prs_"): return None filter_name = script_name[len("list_prs_") :] # Lazy-load the cache module so test runs that don't exercise the # cache path don't pay the import cost. try: import importlib.util as _ilu _here = Path(__file__).resolve().parent _spec = _ilu.spec_from_file_location( "_pr_classification_cache", _here / "_pr_classification_cache.py", ) if _spec is None or _spec.loader is None: return None if "_pr_classification_cache" in sys.modules: cache_mod = sys.modules["_pr_classification_cache"] else: cache_mod = _ilu.module_from_spec(_spec) sys.modules["_pr_classification_cache"] = cache_mod _spec.loader.exec_module(cache_mod) except (ImportError, OSError, AttributeError): # Module missing on disk, unreadable, or its own import chain # broke — fall through to the legacy TS-script path. # Programmer errors propagate so the test suite catches them. return None if filter_name in cache_mod.FILTER_NAMES: return filter_name return None def _use_python_filters() -> bool: """Feature flag for the Phase 2 dispatcher cutover. Default OFF in code; ``tools/launch_fork.sh`` sets it ON for fork-mode runs (same pattern as ``IMPLEMENTER_ESTIMATOR_ENABLED``). Operator can pre-export ``=0`` to opt back to the legacy TS-script path for emergency rollback without a code change.""" return os.environ.get( "REVIEW_DISPATCHER_USE_PYTHON_FILTERS", "", ).strip().lower() in ("1", "true", "yes", "on") def collect_candidates( cfg: DispatchConfig, groups: list[WorkGroup], ) -> tuple[list[tuple[WorkGroup, dict[str, Any]]], dict[str, int]]: candidates: list[tuple[WorkGroup, dict[str, Any]]] = [] counts: dict[str, int] = {} seen: set[tuple[str, int]] = set() use_python = _use_python_filters() for group in groups: # 2026-05-16 Phase 2 cutover: prefer the Python delta-cached # path when (a) the feature flag is on AND (b) the work-group's # script_name has a corresponding entry in # ``_pr_classification_cache.FILTER_NAMES``. Other groups # (e.g. merge_drive's ``list_prs_ready_to_merge``) fall through # to the legacy subprocess path — those filters haven't been # ported yet (Phase 4 in the plan). python_filter = ( _python_filter_name_for(group.script_name) if use_python else None ) if python_filter is not None: try: cache_mod = sys.modules["_pr_classification_cache"] items = cache_mod.refresh_then_filter(cfg, python_filter) except Exception as exc: # noqa: BLE001 — integration-boundary fallback # Deliberate broad catch: this is the seam between the # dispatcher and the Python filter cache module. The # contract is "ANY failure in the new path falls back # to the legacy TS-script path so the dispatcher still # runs the cycle." Narrowing here would silently drop # cycles when the cache module raises something we # didn't anticipate (a new sqlite3 subclass, a # ValueError from a schema change, etc.). The WARN # log + telemetry is the operator's regression signal. logging.getLogger("dispatch_runtime").warning( "Python filter path failed for %s (filter=%s); " "falling back to TS script: %s", group.name, python_filter, exc, ) items = run_list_script(group.script_name, cfg) else: items = run_list_script(group.script_name, cfg) # Universal triage-label exclusion (R3.4, 2026-05-17). The # cycle-cap applies ``auto/needs-human-triage`` to PRs the # reviewer has run on N times with no progress signal. The # reviewer-side Python filter already drops triage-labeled # items via ``_evaluate_filter``'s ``is_excluded`` check, but # before this guard the IMPLEMENTER (and any other dispatcher # routed through this collector) would still pick up the same # PRs because their filter scripts didn't know about the # label. Result: a triage-labeled PR with failing CI would # accrete implementer cycles indefinitely after the reviewer # gave up — the cap was half-effective. Filter here so the # label means "all automation pauses until a human looks," # regardless of which dispatcher / which filter path produced # the item. items = [ item for item in items if not _cycle_cap.labels_carry_triage(item.get("labels") or []) ] counts[group.name] = len(items) for item in items: number = _item_number(item) if number is None: continue key = (group.item_kind, number) if key in seen: continue seen.add(key) candidates.append((group, item)) return candidates, counts def _item_number(item: dict[str, Any]) -> int | None: try: number = int(item.get("number")) except (TypeError, ValueError): return None return number if number > 0 else None def _claim_label(kind: str) -> str: return f"auto/claimed-{kind}" def _existing_claim_labels( number: int, cfg: DispatchConfig ) -> tuple[list[str] | None, int]: """Return ``(labels, status)`` for issue/PR ``number``. ``labels`` is the list of every ``auto/claimed-*`` label currently attached, or ``None`` when the fetch failed at a level that means we cannot trust "no claim is present" (auth, repo-gone, persistent 5xx). ``status`` is the HTTP status code from ``_claim_runtime.idempotent_get`` (already retried for 5xx / transport errors). The caller refuses the claim on ``None`` instead of proceeding, closing the latent bug where a 401 / 403 silently let us POST a claim label we had no business attaching. """ response = _claim_runtime.get( f"/repos/{cfg.owner}/{cfg.repo}/issues/{number}/labels", cfg, ) status = int(response.get("status") or 0) if status != 200: return None, status body = response.get("body") if not isinstance(body, list): return None, status return ( [ str(entry.get("name")) for entry in body if isinstance(entry, dict) and isinstance(entry.get("name"), str) and str(entry.get("name")).startswith("auto/claimed-") ], status, ) _POST_CLAIM_VERIFY_ENV = "DISPATCHER_VERIFY_CLAIM_AFTER_APPLY" def _is_post_claim_verify_enabled() -> bool: """W5 harvest (2026-05-15): opt-in post-claim TOCTOU verification. Default OFF preserves today's accept-the-race behaviour. When set to ``1`` / ``true`` / ``yes`` / ``on`` the dispatcher re-reads each item's labels after attaching its claim and detects collisions where a sibling driver attached a different ``auto/claimed-*`` label inside the GET-then-POST race window. """ return os.environ.get(_POST_CLAIM_VERIFY_ENV, "").strip().lower() in ( "1", "true", "yes", "on", ) def _post_claim_collision_detected( number: int, cfg: DispatchConfig, *, claim_kind: str, ) -> bool: """Re-GET labels for ``number`` and return ``True`` iff a claim label OTHER than our own ``auto/claimed-{claim_kind}`` is present. A transient labels-fetch failure (status != 200, malformed body) is NOT treated as a collision — we would rather miss one race than spuriously abandon a healthy claim. The cycle-failure-budget catches persistent fetch failures separately. """ own_label = _claim_label(claim_kind) labels, _status = _existing_claim_labels(number, cfg) if labels is None: return False return any(name != own_label for name in labels) def claim_work_item( number: int, cfg: DispatchConfig, *, claim_kind: str, driver_name: str, ) -> dict[str, Any]: """Attempt to acquire the ``auto/claimed-{claim_kind}`` label on ``number``. Forgejo's label-add API is idempotent — it returns 200 whether the label was newly attached or already present — so we cannot distinguish ownership at the API layer alone. To avoid releasing a claim we did not acquire (and thereby disrupting another worker's in-flight session), we pre-check the issue's labels and refuse when *any* ``auto/claimed-*`` label is already present. This still has a TOCTOU race window (a sibling driver could attach the label between our GET and POST), which we accept and document in the same way ``conflict_drive.py`` § 3.3.1 documents the same race; the worker layer's idempotent claim helper deduplicates work even in the rare racing case. """ if cfg.dry_run: return {"applied": False, "dry_run": True, "number": number} existing, label_status = _existing_claim_labels(number, cfg) if existing is None: logger.warning( "skip claim of #%s — labels GET returned status=%s (cannot " "verify ownership; treating as foreign claim)", number, label_status, ) return { "applied": False, "number": number, "reason": "labels-fetch-failed", "label_fetch_status": label_status, } if existing: logger.info( "skip claim of #%s — already-claimed labels=%s", number, existing, ) return { "applied": False, "number": number, "reason": "already-claimed", "existing_labels": existing, } label = _claim_label(claim_kind) if not _claim_runtime._add_label(number, label, cfg): return {"applied": False, "number": number, "reason": "label-not-found"} ttl_until = ( datetime.now(UTC) + timedelta(seconds=cfg.claim_ttl_seconds) ).isoformat() body = ( f"{CLAIM_COMMENT_MARKER}\n\n" f"Claimed by `{driver_name}` (pid {os.getpid()}) until `{ttl_until}`.\n\n" "This claim is advisory and will be released when the worker exits, " "or after the TTL by a sibling driver's expired-claim sweep." ) _claim_runtime.post( f"/repos/{cfg.owner}/{cfg.repo}/issues/{number}/comments", cfg, {"body": body}, ) return {"applied": True, "number": number, "ttl_until": ttl_until} # ─── In-flight-claim registry for signal-handler cleanup (Phase 5a) ───────── # # When the dispatcher receives SIGTERM / SIGINT mid-cycle the worker session # can still be 30+ minutes from completion, but the operator (systemd, a # launcher script, or a human pressing Ctrl-C) expects the ``auto/claimed-*`` # label to clear quickly so the next dispatcher invocation — or a sibling # bot — can pick the work up. We record the in-flight claim metadata in a # module-level slot so the signal handler can release the claim # synchronously without waiting for the worker session to finish. # # Threading note: the dispatcher is single-threaded (workers run via # blocking subprocess / blocking HTTP poll), so a plain dict + lock is # sufficient. ``threading.Lock`` is reentrant-safe enough for the # Python-level signal-delivery model — handlers run between bytecodes # in the main thread, so a brief lock-then-clear-then-release sequence # under the lock cannot deadlock against itself. We deliberately keep # the critical section tiny (snapshot + flag flip) so the signal # handler doesn't hold the lock during the actual HTTP call. _INFLIGHT_CLAIM_LOCK = threading.Lock() _INFLIGHT_CLAIM: dict[str, Any] | None = None # Stop event registered by ``run_outer_loop`` / ``run_one_cycle``-via-CLI; # the signal handler sets it so the outer loop exits cleanly after the # in-flight cycle returns. _REGISTERED_STOP_EVENT: Any | None = None # Records whether ``install_signal_handlers`` has already run in this # process so a second call (e.g. ``run_outer_loop`` after a one-shot # ``run_one_cycle`` in the same process for a test) is a no-op rather # than overwriting the previous SIGTERM handler. _SIGNAL_HANDLERS_INSTALLED = False def _register_inflight_claim( *, number: int, cfg: DispatchConfig, claim_kind: str, driver_name: str, ) -> None: """Stamp the in-flight claim metadata so the signal handler can release it before the dispatcher exits. Called by ``dispatch_one`` after a successful claim, paired with :func:`_deregister_inflight_claim` in the finally block. """ global _INFLIGHT_CLAIM with _INFLIGHT_CLAIM_LOCK: _INFLIGHT_CLAIM = { "number": number, "cfg": cfg, "claim_kind": claim_kind, "driver_name": driver_name, } def _deregister_inflight_claim() -> dict[str, Any] | None: """Pop and return the in-flight claim metadata. Called by ``dispatch_one``'s finally before the regular ``release_work_item`` runs so the signal handler does not double-release. Returns the previous registration (or ``None`` if the signal handler already cleared it). """ global _INFLIGHT_CLAIM with _INFLIGHT_CLAIM_LOCK: previous = _INFLIGHT_CLAIM _INFLIGHT_CLAIM = None return previous def _signal_release_handler(signum: int, _frame: Any) -> None: """SIGTERM / SIGINT handler: release any in-flight claim and request cooperative shutdown, then re-raise as SystemExit so the outer loop's ``finally lock.release()`` runs. Sequence: 1. Snapshot ``_INFLIGHT_CLAIM`` under the lock and atomically clear it. This is the slot ``dispatch_one``'s finally block *would* otherwise read; clearing here means the finally path sees ``None`` and does NOT post a second release comment. 2. Set the registered stop event so the outer loop exits between cycles (or, if we re-raise into the cycle, the cycle's exception handler counts it against the failure budget). 3. Issue the release HTTP call. We log + swallow exceptions so a transient network blip during shutdown can't loop forever. 4. Re-raise as ``SystemExit`` with code 130 (SIGINT convention) or 143 (SIGTERM convention) so the process exit reflects the signal that triggered it. Re-raising rather than returning matters because the in-flight worker session is blocked on an HTTP poll inside ``_opencode_worker.run_session_blocking``; that loop checks the stop event at most once per poll interval. SystemExit unwinds through every ``try: ... finally:`` in the call chain (including the worker poll loop's), so the dispatcher exits within seconds rather than waiting for the worker timeout. """ global _INFLIGHT_CLAIM snapshot: dict[str, Any] | None with _INFLIGHT_CLAIM_LOCK: snapshot = _INFLIGHT_CLAIM # Clear the slot under the lock so a concurrent # ``dispatch_one`` finally that runs after we drop the lock # but before we issue the HTTP call sees ``None`` and skips # its own release. The HTTP call itself is intentionally # OUTSIDE the lock — it can take seconds and we don't want # to block the lock that long. A follow-up signal during # the release would re-snapshot ``None`` and skip cleanly. if snapshot is not None: _INFLIGHT_CLAIM = None if _REGISTERED_STOP_EVENT is not None: try: _REGISTERED_STOP_EVENT.set() except Exception: logger.exception( "signal handler: setting stop event raised; " "continuing with claim release" ) # Resolve the human-readable signal name once so both the # logger output AND the Forgejo release-comment ``terminal_state`` # field show ``signal-SIGTERM`` / ``signal-SIGINT`` rather than # the bare numeric ``signal-15`` / ``signal-2``. The ``Signals`` # enum was added in Python 3.5; falling back to the numeric # form on platforms with bespoke signal numbers (Windows, # mostly) keeps the dispatcher portable. try: signal_name = signal.Signals(signum).name except (ValueError, AttributeError): signal_name = f"sig{int(signum)}" if snapshot is not None: try: release_work_item( snapshot["number"], snapshot["cfg"], claim_kind=snapshot["claim_kind"], driver_name=snapshot["driver_name"], terminal_state=f"signal-{signal_name}", detail=( f"released by dispatcher signal handler " f"({signal_name}, signum={signum}); cooperative " f"shutdown initiated" ), ) logger.warning( "signal handler released claim for #%s (%s, signum=%s)", snapshot["number"], signal_name, signum, ) except Exception as exc: logger.exception( "signal handler: release_work_item failed for #%s: %s", snapshot["number"], exc, ) # POSIX convention: ``128 + signum``. SIGINT (=2) → 130, # SIGTERM (=15) → 143; everything else picks up the same rule. raise SystemExit(128 + int(signum)) def install_signal_handlers(stop: Any) -> None: """Install SIGTERM / SIGINT handlers for cooperative dispatcher shutdown. Idempotent: a second call from the same process is a no-op (the test suite runs many ``run_outer_loop`` invocations against different fake clocks; we don't want each to clobber the handler from the previous test). The handler is intentionally installed in the main thread by the main thread — Python's ``signal`` module raises ``ValueError`` if called from a non-main thread, so a future architecture that wraps ``run_outer_loop`` in a thread would need to install the handler before spawning the thread. """ global _REGISTERED_STOP_EVENT, _SIGNAL_HANDLERS_INSTALLED _REGISTERED_STOP_EVENT = stop if _SIGNAL_HANDLERS_INSTALLED: return try: signal.signal(signal.SIGTERM, _signal_release_handler) signal.signal(signal.SIGINT, _signal_release_handler) _SIGNAL_HANDLERS_INSTALLED = True except ValueError as exc: # Hit when called from a non-main thread. The dispatcher's # production paths always install from the main thread, but # the test suite occasionally invokes ``run_outer_loop`` from # a worker thread (e.g. when a fixture wraps it for # parallelism). We log + continue without handlers in that # case rather than failing the whole loop — the test would # have asserted the handler installation explicitly if it # cared. logger.warning( "install_signal_handlers: signal.signal raised %s " "(probably called from non-main thread); continuing " "without dispatcher signal handlers", exc, ) def release_work_item( number: int, cfg: DispatchConfig, *, claim_kind: str, driver_name: str, terminal_state: str, detail: str = "", ) -> dict[str, Any]: if cfg.dry_run: return {"released": False, "dry_run": True, "number": number} label = _claim_label(claim_kind) removed = _claim_runtime._remove_label(number, label, cfg) body = ( f"{CLAIM_COMMENT_MARKER}\n\n" f"Released by `{driver_name}` (pid {os.getpid()}). " f"terminal_state=`{terminal_state}`" + (f"\n\nDetail: {detail}" if detail else "") ) _claim_runtime.post( f"/repos/{cfg.owner}/{cfg.repo}/issues/{number}/comments", cfg, {"body": body}, ) return {"released": removed, "number": number} def sweep_own_claims(cfg: DispatchConfig, claim_kind: str) -> list[int]: return _claim_runtime.sweep_expired_claims( cfg, session_pr_numbers=set(), label=_claim_label(claim_kind), ) def dispatch_one( cfg: DispatchConfig, group: WorkGroup, item: dict[str, Any], *, driver_name: str, ) -> dict[str, Any]: """Claim, dispatch, and release a single work item. Terminal states (recorded as ``terminal_state`` in telemetry): - ``invalid-item`` — the work-group script returned an item with no usable ``number``. - ``already-claimed`` — another worker already owns an ``auto/claimed-*`` label on this item; we did NOT acquire a claim and do NOT release one. - ``labels-fetch-failed`` — the labels GET returned non-200 (auth / permission / repo-gone). Treated as a foreign claim: refused rather than silently attaching a label we have no permission to manage. The HTTP status is captured in ``claim_result.label_fetch_status`` for operator triage. - ``claim-failed`` — Forgejo accepted the labels GET but refused our label-add (label undefined in the repo, etc.). No release attempted. - ``dry-run`` — ``cfg.dry_run`` is set; no worker was dispatched. - ``completed`` — the OpenCode session reached idle without timing out or hitting a transport error. The worker's domain-specific result (review submitted, code generated, etc.) is judged by side effects, not by a JSON outcome key. Reviewer / implementer workers do NOT emit the conflict-driver JSON schema. - ``timeout`` / ``transport-error`` — propagated verbatim from :class:`_opencode_worker.SessionResult.status`. The release in the finally block runs only when ``claimed`` is true (i.e. :func:`claim_work_item` returned ``applied=True``). This is what makes ``already-claimed`` and ``labels-fetch-failed`` safe: we never remove a label we did not just attach. """ number = _item_number(item) if number is None: return {"terminal_state": "invalid-item", "item": item} claimed = False claim_result: dict[str, Any] | None = None started = time.monotonic() session = None terminal_state = "unknown" try: if group.claim_kind is not None: claim_result = claim_work_item( number, cfg, claim_kind=group.claim_kind, driver_name=driver_name, ) claimed = bool(claim_result.get("applied")) if not claimed and not cfg.dry_run: reason = claim_result.get("reason") if reason == "already-claimed": terminal_state = "already-claimed" elif reason == "labels-fetch-failed": terminal_state = "labels-fetch-failed" else: terminal_state = "claim-failed" return { "terminal_state": terminal_state, "claim_result": claim_result, "item_number": number, } # W5 harvest (2026-05-15): opt-in post-claim TOCTOU # verification. Forgejo's label-add is idempotent — # two concurrent drivers can BOTH come back with HTTP # 200 even though only one was first. The single-instance # fcntl lock prevents same-driver races, but cross-driver # (implementer ↔ reviewer ↔ merge) and cross-host races # remain. When ``DISPATCHER_VERIFY_CLAIM_AFTER_APPLY=1``, # re-GET the item's labels right after our claim landed; # if a *different* ``auto/claimed-*`` label is present, # a sibling driver raced us — release our claim cleanly # and skip the item this cycle. Default OFF preserves # today's accept-the-race behaviour byte-for-byte. if ( claimed and not cfg.dry_run and _is_post_claim_verify_enabled() and _post_claim_collision_detected( number, cfg, claim_kind=group.claim_kind, ) ): logger.warning( "%s post-claim collision on #%s (sibling driver " "claim label present); releasing and skipping", driver_name, number, ) release_work_item( number, cfg, claim_kind=group.claim_kind, driver_name=driver_name, terminal_state="claim-collision", detail="post-claim-verify", ) return { "terminal_state": "claim-collision", "claim_result": claim_result, "item_number": number, } # Register the in-flight claim so a SIGTERM / SIGINT # handler can release it before this dispatcher exits. # Skipped on dry-run (no real claim was acquired) and on # the ``new_issue`` work group whose ``claim_kind`` is # ``None`` (the outer ``if group.claim_kind is not None`` # already gates this branch correctly). The matching # deregister fires in the finally block before # ``release_work_item`` so the signal handler's # synchronous release and the finally's release cannot # both post release comments. if claimed and not cfg.dry_run: _register_inflight_claim( number=number, cfg=cfg, claim_kind=group.claim_kind, driver_name=driver_name, ) prompt = group.prompt_factory(cfg, item, group) if cfg.dry_run: terminal_state = "dry-run" return { "terminal_state": terminal_state, "claim_result": claim_result, "item_number": number, "prompt": prompt, } tag = _tag_for(group, number) # Refresh our liveness heartbeat once per polling iteration so a # 30-minute review session doesn't look like a hung dispatcher # process to the launcher / systemd unit. Without this, the # watchdog above the dispatcher would SIGTERM us mid-cycle and # orphan both the OpenCode session and the auto/claimed-* lock # we hold on this PR. Errors from the callback are swallowed # inside run_session_blocking so a transient disk-full / EROFS # on the heartbeat write cannot mask a successful worker # completion. heartbeat_path = cfg.heartbeat_path # Operator-visible heartbeat (Tier-1 R3, 2026-05-12). The # ``on_poll`` callback fires every ~2s (the polling cadence), # but spamming the log every 2s for 20-minute worker turns # would be unreadable. Throttle to one INFO line every # ``_HEARTBEAT_LOG_INTERVAL_SECONDS`` (default 120s). Before # this change the dispatcher emitted log lines at session # start and session end but NOTHING in between — an operator # tail-ing the log during a 20-minute worker run had no # signal that the dispatcher was still alive vs. hung. The # state-transition logs inside ``run_session_blocking`` # already fire on busy↔idle flips but a long-running worker # turn is one continuous "busy" episode, so they emit at # most once per turn. The interval is configurable via # ``DISPATCHER_HEARTBEAT_LOG_INTERVAL_SECONDS`` for tests # and for operators who want a denser trace. # ``last_logged_at_monotonic`` is seeded to the start time so # the FIRST poll (which fires ~2s into the cycle) does NOT # immediately emit a log line — only the second poll past the # interval threshold does. Seeding to ``0.0`` would log on # every cycle's first poll because (now - 0) ≫ interval for # any post-epoch monotonic clock. _heartbeat_started_at_monotonic = time.monotonic() heartbeat_state = { "started_at_monotonic": _heartbeat_started_at_monotonic, "last_logged_at_monotonic": _heartbeat_started_at_monotonic, } # Resolve the effective worker-agent name once, after the # prompt_factory ran (so any per-cycle override it stashed on # ``item`` is visible). The implementer dispatcher uses this # to route each cycle to the matching # ``task-implementor-tier-`` variant resolved from the # estimator or the cross-cycle tier label, replacing the # (retired) tier-dispatcher + tier-N wrapper agents. Other # work groups (review, conflict) leave the override unset # and continue using the static ``group.worker_agent``. effective_worker_agent = _resolve_effective_worker_agent( group, item, ) def _refresh_heartbeat() -> None: _claim_runtime.write_heartbeat(heartbeat_path) now_mono = time.monotonic() if ( now_mono - heartbeat_state["last_logged_at_monotonic"] >= _HEARTBEAT_LOG_INTERVAL_SECONDS ): elapsed = now_mono - heartbeat_state["started_at_monotonic"] logger.info( "%s worker still in-flight: PR #%s elapsed=%.0fs " "tag=%s agent=%s — heartbeat refreshed", group.name, number, elapsed, tag, effective_worker_agent, ) heartbeat_state["last_logged_at_monotonic"] = now_mono # Bracket ``run_session_blocking`` with ISO-8601 UTC # timestamps so the post-session hook (Phase 4 telemetry, # specifically) can record actual cycle wall-clock instead # of post-hoc ``now_iso()`` placeholders. The worker's # self-reported ``session.wallclock_seconds`` is also # threaded through for cross-checking. # Redact the dispatcher's Forgejo PAT from the on-disk session # archive. ``cfg.token`` is the same value the implementer # prompt builder embeds into the worker prompt (so the worker # does not have to burn turns on ``printenv FORGEJO_PAT``); we # never want that value to land in # ``.dispatcher-logs/sessions/*.json``. ``run_session_blocking`` # short-circuits when the list is empty / values are too short, # so a dry-run cycle (token may be a stub) is still safe. redact_values = [cfg.token] if getattr(cfg, "token", None) else [] session_started_at = _now() # Short-circuit hook: a prompt_factory MAY stash a # ``_short_circuit_result`` on the item's # ``_dispatcher_implementer_context`` dict to indicate the # dispatcher has already handled this PR deterministically # and no LLM session is needed. The value is a dict shaped # like a :class:`_opencode_worker.SessionResult` with at # minimum ``status``, ``parsed_json``, and ``raw_response``. # See dispatch_implementer's ``_maybe_short_circuit_with_auto_fix`` # for the production callsite. short_circuit = _maybe_read_short_circuit(item) if short_circuit is not None: session = short_circuit logger.info( "%s item #%s short-circuited (no LLM session): %s", group.name, number, short_circuit.status, ) else: session = _opencode_worker.run_session_blocking( server_url=cfg.server_url, agent=effective_worker_agent, tag=tag, prompt=prompt, timeout_seconds=cfg.worker_timeout_seconds, on_poll=_refresh_heartbeat, redact_values=redact_values, ) session_completed_at = _now() if session.status == "completed": terminal_state = "completed" else: terminal_state = session.status # "timeout" | "transport-error" worker_outcome: str | None = None if isinstance(session.parsed_json, dict): outcome_value = session.parsed_json.get("outcome") if isinstance(outcome_value, str) and outcome_value: worker_outcome = outcome_value post_session_result: dict[str, Any] | None = None if group.post_session_action is not None: # Pack the work-group name and session-timing context # into a single ``SessionContext`` dataclass so the post- # session action sees a single read-only record instead # of a fan of four loose kwargs (and so future fields # land in one place). Earlier iteration stamped these # onto ``item`` directly, then a flat ``**kwargs`` — # both made the data flow hard to follow. session_context = SessionContext( work_group_name=group.name, session_started_at=session_started_at, session_completed_at=session_completed_at, session_wallclock_seconds=session.wallclock_seconds, subagent_max_depth=session.subagent_max_depth, heartbeat_callback=_refresh_heartbeat, ) try: post_session_result = group.post_session_action( cfg, item, session.parsed_json, session.raw_response, terminal_state, session_context=session_context, ) except Exception as exc: logger.exception( "%s post_session_action raised on item #%s: %s", group.name, number, exc, ) post_session_result = { "review_action": "failed", "review_action_reason": ( f"post_session_action_exception: {type(exc).__name__}: {exc}" ), } return { "terminal_state": terminal_state, "session_status": session.status, "worker_outcome": worker_outcome, "session_id": session.session_id, "worker_wallclock_seconds": session.wallclock_seconds, "raw_response": session.raw_response, "parsed_json": session.parsed_json, "claim_result": claim_result, "item_number": number, "post_session_result": post_session_result, # Surfaces ``error_kind`` from _opencode_worker so the # cycle log can pivot on transport-create-session vs # transport-poll-status vs startup-grace-elapsed without # parsing free-text raw_response. ``None`` on # ``completed``. "session_error_kind": session.error_kind, } finally: # Deregister the in-flight claim BEFORE issuing the regular # release so a signal that fires between ``_register_inflight_claim`` # and here triggers exactly one release path, not two. The # signal handler clears the slot and posts the release; this # path checks the previous registration and only posts when # the slot is still our claim (i.e. no signal raced us). If # the snapshot is None the signal handler already released — # we skip the duplicate release comment. previous_registration = _deregister_inflight_claim() if ( claimed and group.claim_kind is not None and previous_registration is not None ): release_work_item( number, cfg, claim_kind=group.claim_kind, driver_name=driver_name, terminal_state=terminal_state, detail=_sanitize_release_detail( session.raw_response if session is not None else "" ), ) elapsed = time.monotonic() - started logger.info( "%s item #%s finished terminal_state=%s elapsed=%.1fs", group.name, number, terminal_state, elapsed, ) def _tag_for(group: WorkGroup, number: int) -> str: suffix = "PR" if group.item_kind == "pr" else "ISSUE" return f"{group.tag_prefix}-{suffix}-{number}" def run_one_cycle( cfg: DispatchConfig, groups: list[WorkGroup], *, driver_name: str, sweep_claim_kind: str | None, ) -> dict[str, Any]: cycle_id = str(uuid.uuid4()) started_at = _now() # Insert the in-flight row before any work so the telemetry console # can show this cycle as "running" the moment it starts. Without # this, the cycle would only appear after a multi-minute worker # session finished, leaving the dashboard looking like the # dispatcher had stopped working. The companion ``finish_cycle`` call # in the finally block UPDATEs the same row with aggregate counts. begin_cycle(cfg, cycle_id=cycle_id, started_at=started_at, driver_name=driver_name) swept: list[int] = [] candidates: list[tuple[WorkGroup, dict[str, Any]]] = [] group_counts: dict[str, int] = {} processed: list[dict[str, Any]] = [] claims_acquired = 0 try: swept = sweep_own_claims(cfg, sweep_claim_kind) if sweep_claim_kind else [] candidates, group_counts = collect_candidates(cfg, groups) for group, item in candidates[: cfg.max_items_per_cycle]: outcome = dispatch_one(cfg, group, item, driver_name=driver_name) if (outcome.get("claim_result") or {}).get("applied"): claims_acquired += 1 processed.append({"group": group.name, **outcome}) return { "cycle_id": cycle_id, "started_at": started_at, "ended_at": None, # set by the finally block via finish_cycle "candidates_count": len(candidates), "group_counts": group_counts, "claims_acquired": claims_acquired, "swept": swept, "processed": processed, } finally: ended_at = _now() # Use a defensive try/except so a finish-time write failure # (disk full, transient SQLite lock contention, etc.) does not # mask whatever exception actually drove us into the finally # block. The in-flight row stays in place if the UPDATE fails # — operators can spot the orphan via ``ended_at IS NULL`` and # the next ensure_cycle_table run will not delete it. try: finish_cycle( cfg, cycle_id=cycle_id, ended_at=ended_at, candidates_count=len(candidates), group_counts=group_counts, claims_acquired=claims_acquired, swept=swept, processed=processed, # Pass started_at + driver_name so the INSERT-fallback # path (which fires only if begin_cycle's row got lost # — e.g. SQLite WAL corruption + recovery between # begin_cycle and finish_cycle) still records an # accurate started_at and driver instead of synthesising # zero-duration ``unknown`` rows. Under the normal # UPDATE path these kwargs are unused. started_at=started_at, driver_name=driver_name, ) except sqlite3.Error: logger.exception( "%s finish_cycle UPDATE failed for cycle_id=%s — orphan " "in-flight row will be visible in dispatch_*_cycles", driver_name, cycle_id, ) _PAT_AUTH_FAILURE_STATUSES: frozenset[int] = frozenset({401, 403}) def _validate_pat_or_die( cfg: DispatchConfig, *, driver_name: str, ) -> None: """G5 harvest (2026-05-15): probe ``GET /user`` at dispatcher startup to fail fast when the configured PAT is dead. Rationale: without this probe, a rotated / revoked PAT lets the driver loop forever — every work-item claim returns ``labels-fetch-failed`` (a "soft" outcome that counts as a successful cycle with zero dispatches), so ``consecutive_failures`` is reset every iteration and the cycle-failure-budget escape hatch never trips. Operators see timed-out heartbeats but no SystemExit, no log alert with the actual cause. Behaviour: - 200 → log INFO with the resolved login (auditable identity at process start), return normally. - 401 / 403 → ``raise SystemExit(2)`` with a loud message naming the driver. The exit code matches :func:`run_outer_loop`'s existing cycle-failure-budget exit so a supervisor (systemd, launcher) restarts the process with the same semantics. - Any other response (transport error, 5xx, malformed body) → log WARNING and return. We *do not* hard-stop on transient failures; the cycle-failure-budget covers persistent ones. Skippable via ``DISPATCHER_SKIP_PAT_VALIDATION=1`` for tests and bisect scenarios. The skip is logged so a missing validation step in production telemetry is obvious. """ if os.environ.get("DISPATCHER_SKIP_PAT_VALIDATION") in ( "1", "true", "yes", "on", ): logger.warning( "%s startup PAT validation SKIPPED via " "DISPATCHER_SKIP_PAT_VALIDATION — dead PATs will spin " "idle until cycle-failure-budget trips", driver_name, ) return try: response = _claim_runtime.idempotent_get("/user", cfg) except Exception as exc: # noqa: BLE001 - want to log + return logger.warning( "%s startup PAT probe raised %s — assuming transient; " "the cycle-failure-budget will catch persistent failures", driver_name, type(exc).__name__, ) return status = int(response.get("status") or 0) if isinstance(response, dict) else 0 if status in _PAT_AUTH_FAILURE_STATUSES: logger.error( "%s startup PAT validation got HTTP %s from /user — the " "configured Forgejo PAT appears dead (rotated, revoked, " "or scoped wrong). Exiting (rather than spinning idle) " "so the supervisor surfaces the error and an operator " "rotates the credential.", driver_name, status, ) raise SystemExit(2) if status != 200: logger.warning( "%s startup PAT probe returned HTTP %s (neither 200 nor " "401/403) — proceeding; the cycle-failure-budget will " "catch persistent failures", driver_name, status, ) return body = response.get("body") if isinstance(response, dict) else None login = body.get("login") if isinstance(body, dict) else None logger.info( "%s startup PAT validated as %s", driver_name, login if isinstance(login, str) and login else "", ) def run_outer_loop( cfg: DispatchConfig, groups: list[WorkGroup], *, driver_name: str, sweep_claim_kind: str | None, stop: Any | None = None, ) -> None: """Hold the single-instance lock and run cycles until ``stop`` fires. Each cycle's exceptions are logged and counted against ``cfg.cycle_failure_budget``. After that many consecutive failures we exit with code 2 — matching ``merge_drive.py`` / ``conflict_drive.py`` so an upstream supervisor (systemd, a launcher script, etc.) restarts us with fresh state instead of letting a partial-output work-group script wedge the loop indefinitely. A successful cycle resets the counter. """ lock = _claim_runtime.SingleInstanceLock(cfg.lock_path) if not lock.acquire(): raise SystemExit(f"another {driver_name} instance holds {cfg.lock_path}") # G5 harvest (2026-05-15): hard-stop at startup if the PAT is # dead. Without this, a 401/403 returns ``labels-fetch-failed`` # per work-item and the cycle "succeeds with zero dispatches" # — resetting ``consecutive_failures`` to 0 every cycle and # spinning idle forever. The probe is a single ``GET /user`` # round-trip; transient failures don't trip it (only 401/403 # do), and a 2xx is logged at INFO so an operator can confirm # which identity the dispatcher started under. _validate_pat_or_die(cfg, driver_name=driver_name) stop = stop or _claim_runtime.StopEvent() # Install SIGTERM / SIGINT handlers so an operator stop (systemd, # Ctrl-C, supervisor restart) releases any in-flight # ``auto/claimed-*`` label before the dispatcher exits. Without # this, the merge driver's ``sweep_expired_claims`` would only # release the orphan label after ``claim_ttl_seconds`` (default # 7200 s = 2 h) — long enough for a sibling dispatcher to wait # noticeably for the next cycle. Test seam: a test that calls # ``run_outer_loop`` with its own ``stop`` event still gets the # handler installation, but tests that exercise the handler # directly should call :func:`install_signal_handlers` themselves # against a controlled stop event. install_signal_handlers(stop) consecutive_failures = 0 try: while not stop.is_set(): try: run_one_cycle( cfg, groups, driver_name=driver_name, sweep_claim_kind=sweep_claim_kind, ) consecutive_failures = 0 except Exception: consecutive_failures += 1 logger.exception( "%s cycle failed (consecutive=%s/%s)", driver_name, consecutive_failures, cfg.cycle_failure_budget, ) if consecutive_failures >= cfg.cycle_failure_budget: logger.error( "%s exceeded cycle failure budget (%s); exiting " "for supervisor restart", driver_name, cfg.cycle_failure_budget, ) raise SystemExit(2) _claim_runtime.write_heartbeat(cfg.heartbeat_path) stop.sleep(cfg.cycle_interval_seconds) finally: lock.release() def _now() -> str: return datetime.now(UTC).isoformat() def _maybe_read_short_circuit( item: dict[str, Any], ) -> "_opencode_worker.SessionResult | None": """Read the prompt-factory's short-circuit stash if present. A prompt factory MAY decide that the dispatcher has already handled the work-item deterministically (e.g. by applying compliance fixes directly) and that no LLM session needs to run. It signals this by stashing a ``SessionResult``-shaped object under ``item["_dispatcher_implementer_context"]["_short_circuit_result"]`` BEFORE returning from the prompt factory. Returns the ``SessionResult`` if present and shaped correctly; ``None`` otherwise (in which case the dispatcher spawns the worker as usual). The key is consumed (popped) so a subsequent cycle reading the same item dict does not re-trigger the short-circuit. The short-circuit machinery preserves the rest of the post- session pipeline (telemetry, status comment, claim release) — only the LLM session is skipped. Telemetry rows will carry ``terminal_state`` from the synthesised result and downstream consumers see the same row shape they expect. """ context = item.get("_dispatcher_implementer_context") if not isinstance(context, dict): return None stash = context.pop("_short_circuit_result", None) if stash is None: return None # Mark consumption so the post-session action's cycle-archive # field can report ``short_circuit=True`` without re-reading the # stash (which is gone after the .pop above). context["_short_circuit_consumed"] = True # Validate shape: must have at least ``status`` and be coercible # to a SessionResult. We accept either a SessionResult instance # directly OR a plain dict (so test code doesn't have to import # the dataclass). if isinstance(stash, _opencode_worker.SessionResult): return stash if isinstance(stash, dict) and "status" in stash: return _opencode_worker.SessionResult( status=stash["status"], wallclock_seconds=float(stash.get("wallclock_seconds", 0.0)), session_id=str(stash.get("session_id", "")), raw_response=str(stash.get("raw_response", "")), parsed_json=stash.get("parsed_json"), error_kind=stash.get("error_kind"), subagent_max_depth=stash.get("subagent_max_depth", 0), ) return None def ensure_cycle_table(table_name: str) -> None: """Create or repair the per-driver cycle table. Delegates to :func:`_pipeline_cache.ensure_dispatch_cycle_schema` so the v5 schema (``ended_at`` nullable, UNIQUE INDEX on ``cycle_id``) is defined in exactly one place. Both this function and ``PipelineCache._migrate_to_v5_in_flight_rows`` import their DDL from there, eliminating drift risk if a future migration adds a column. The dispatcher writes cycle rows via raw ``sqlite3.connect`` rather than going through ``PipelineCache``, so this function must run the creation + migration itself; ``PipelineCache`` may not have been opened in this process before the first dispatcher cycle. """ if table_name not in _pipeline_cache.DISPATCH_CYCLE_TABLES: raise ValueError(f"unexpected dispatch table: {table_name}") path = _pipeline_cache.DEFAULT_CACHE_PATH path.parent.mkdir(parents=True, exist_ok=True) with sqlite3.connect(path) as conn: conn.row_factory = sqlite3.Row _pipeline_cache.ensure_dispatch_cycle_schema(conn, table_name) def begin_cycle( cfg: DispatchConfig, *, cycle_id: str, started_at: str, driver_name: str, ) -> None: """Insert an in-flight row for ``cycle_id`` (``ended_at IS NULL``). Pairs with :func:`finish_cycle`, which UPDATEs the same row with the final ``ended_at`` and aggregate fields when the cycle resolves. Inserting at start fixes the telemetry-blind-spot where a multi- minute review session would not appear anywhere in the cycle table until it finished — making the dispatcher look idle even though it was actively polling an OpenCode worker. The cycle row carries placeholder counts (``0``) and an empty raw blob until :func:`finish_cycle` overwrites them. The UNIQUE index on ``cycle_id`` prevents accidental double-inserts; a re-entrant crash-loop that attempted ``begin_cycle`` twice would surface a ``sqlite3.IntegrityError`` rather than silently fork the row. """ ensure_cycle_table(cfg.table_name) raw_placeholder = json.dumps( {"group_counts": {}, "swept": [], "processed": [], "in_flight": True}, sort_keys=True, ) with sqlite3.connect(_pipeline_cache.DEFAULT_CACHE_PATH) as conn: conn.execute( f""" INSERT INTO {cfg.table_name} ( cycle_id, started_at, ended_at, driver, candidates_count, claims_acquired, swept_count, processed_count, terminal_state, worker_outcome, session_id, worker_wallclock_seconds, raw ) VALUES (?, ?, NULL, ?, 0, 0, 0, 0, NULL, NULL, NULL, NULL, ?) """, ( cycle_id, started_at, driver_name, raw_placeholder, ), ) def finish_cycle( cfg: DispatchConfig, *, cycle_id: str, ended_at: str, candidates_count: int, group_counts: dict[str, int], claims_acquired: int, swept: list[int], processed: list[dict[str, Any]], started_at: str | None = None, driver_name: str | None = None, ) -> None: """Overwrite the in-flight row for ``cycle_id`` with the resolved aggregate fields. Falls back to ``INSERT`` if no row exists for ``cycle_id`` — this handles drivers that never called :func:`begin_cycle` (e.g. test fixtures that build a complete row in one shot) and keeps the function compatible with the legacy ``record_cycle`` semantics. The optional ``started_at`` and ``driver_name`` kwargs feed the INSERT-fallback path. When provided, the synthesised row carries accurate timestamps + driver attribution. When omitted, the row uses ``ended_at`` as both ``started_at`` (collapsing wallclock duration to zero) and ``"unknown"`` for the driver, and the raw JSON blob carries a ``"synthetic_started_at": true`` marker so cycle-time analytics can exclude the row from duration stats. """ ensure_cycle_table(cfg.table_name) terminal_state = None worker_outcome = None session_id = None worker_wallclock = None if processed: last = processed[-1] terminal_state = str(last.get("terminal_state") or "") or None worker_outcome = ( str(last.get("worker_outcome")) if last.get("worker_outcome") else None ) session_id = str(last.get("session_id")) if last.get("session_id") else None worker_wallclock = last.get("worker_wallclock_seconds") raw = { "group_counts": group_counts, "swept": swept, "processed": processed, } with sqlite3.connect(_pipeline_cache.DEFAULT_CACHE_PATH) as conn: cursor = conn.execute( f""" UPDATE {cfg.table_name} SET ended_at = ?, candidates_count = ?, claims_acquired = ?, swept_count = ?, processed_count = ?, terminal_state = ?, worker_outcome = ?, session_id = ?, worker_wallclock_seconds = ?, raw = ? WHERE cycle_id = ? """, ( ended_at, candidates_count, claims_acquired, len(swept), len(processed), terminal_state, worker_outcome, session_id, worker_wallclock, json.dumps(raw, sort_keys=True), cycle_id, ), ) if cursor.rowcount == 0: # No matching in-flight row — fall back to a full INSERT # so callers that skip begin_cycle (legacy tests, # crash-recovery paths that lost their begin_cycle row) # still get a complete record. Use ``started_at`` / # ``driver_name`` if the caller passed them (modern # ``run_one_cycle`` does); otherwise stamp the row with the # ``ended_at`` value for ``started_at`` (collapsing the # cycle to zero duration) and ``"unknown"`` for the # driver, plus a ``synthetic_started_at`` flag in raw so # cycle-time analytics can spot and exclude these rows. insert_started_at = started_at if started_at is not None else ended_at insert_driver = driver_name if driver_name is not None else "unknown" insert_raw = dict(raw) if started_at is None: insert_raw["synthetic_started_at"] = True conn.execute( f""" INSERT INTO {cfg.table_name} ( cycle_id, started_at, ended_at, driver, candidates_count, claims_acquired, swept_count, processed_count, terminal_state, worker_outcome, session_id, worker_wallclock_seconds, raw ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( cycle_id, insert_started_at, ended_at, insert_driver, candidates_count, claims_acquired, len(swept), len(processed), terminal_state, worker_outcome, session_id, worker_wallclock, json.dumps(insert_raw, sort_keys=True), ), ) def status_payload(cfg: DispatchConfig, *, driver_name: str) -> dict[str, Any]: ensure_cycle_table(cfg.table_name) return { "driver": driver_name, "owner": cfg.owner, "repo": cfg.repo, "forgejo_url": cfg.forgejo_url, "server_url": cfg.server_url, "lock_path": str(cfg.lock_path), "heartbeat_path": str(cfg.heartbeat_path), "cycle_interval_seconds": cfg.cycle_interval_seconds, "max_items_per_cycle": cfg.max_items_per_cycle, "worker_timeout_seconds": cfg.worker_timeout_seconds, "claim_ttl_seconds": cfg.claim_ttl_seconds, "cycle_failure_budget": cfg.cycle_failure_budget, "table_name": cfg.table_name, "dry_run": cfg.dry_run, } def json_line(data: Any) -> None: print(json.dumps(data, indent=2, sort_keys=True)) __all__ = ( "API_BASE", "CLAIM_COMMENT_MARKER", "DispatchConfig", "REPO_NAME", "REPO_OWNER", "REPO_ROOT", "SCRIPT_DIR", "SessionContext", "WorkGroup", "begin_cycle", "claim_work_item", "collect_candidates", "derive_forgejo_url", "dispatch_one", "ensure_cycle_table", "finish_cycle", "install_signal_handlers", "json_line", "load_secret", "release_work_item", "resolve_lock_or_heartbeat", "run_list_script", "run_one_cycle", "run_outer_loop", "status_payload", "sweep_own_claims", )