cleveragents-core/tools/controller/master/local_ci.py

"""Local-CI verdict source — the ``RUN_CI_LOCAL=TRUE`` path.

When the cluster's Forgejo CI is unhealthy (every job dies at the
``actions/checkout`` step on a ``git fetch`` connection-reset), the
controller's ``get_ci_status`` / ``get_failure_logs`` callbacks return
garbage red verdicts and the pipeline cannot make progress. Setting
``RUN_CI_LOCAL=TRUE`` swaps those two callbacks for this module, which
runs the *real* ``.forgejo/workflows/ci.yml`` locally via
``tools/run-ci-full-local.sh`` (``forgejo-runner exec``) and reports
the result. ``forgejo-runner exec`` sources the code from a local
checkout, so it produces real gate verdicts even when the cluster's
checkout path is broken.

Async by necessity
------------------
A full CI run takes minutes; a master tick must not block. So the
verdict is served from an on-disk, per-``(owner, repo, head_sha)`` job
cache:

    get_ci_status(owner, repo, head_sha):
      - no run for this SHA  -> spawn a detached runner, return pending
      - run in progress      -> return pending
      - run finished         -> return success / failure

The detached runner (``_worker_main``) checks out ``head_sha`` into a
git worktree off a shared per-repo cache clone, runs
``run-ci-full-local.sh --repo-root <worktree>``, and writes the verdict
+ full log into the run dir. Results persist on disk, so they survive a
master restart and are reused across the ci_gate / ci_status_poll /
prefetch consumers without re-running CI.

CI failure detail
-----------------
The worker parses the forgejo-runner combined log into per-job results
and surfaces the failure detail through the SAME channels the remote
pipeline uses, so the implementer sees local CI's errors exactly as it
sees Forgejo's:

- ``get_ci_status`` returns a combined-status dict with one entry per
  CI job, so prefetch's ``ci_summary`` / ``failing_gates`` show the
  real per-gate breakdown.
- The worker seeds the unified ``_ci_logs`` bundle
  (``/tmp/cleveragents-ci-logs-cache/{head_sha}.full.json``,
  ``source="local"``) with every job's full log — the same store the
  implementer's ``ci_fetch_pr_failure_logs`` MCP tool and the
  ci_summary prefetch read for Forgejo CI.

Log tails are curated: forgejo-runner orchestration lines (``⭐ Run``,
``actions/checkout`` setup, ``🏁 Job``, ...) are stripped so the agent
sees the genuine ruff/pytest/mypy output, and the infra-vs-real
classifier never false-matches an ``actions/checkout`` step name as an
infra failure. The full raw run log stays at ``<run_dir>/ci.log``.
"""

from __future__ import annotations

import json
import logging
import os
import re
import shutil
import subprocess
import sys
import time
from collections.abc import Callable
from contextlib import contextmanager
from pathlib import Path

from .ci_freshness import INFRA_FAILURE_SIGNATURES

logger = logging.getLogger(__name__)


# Where per-(owner,repo,head_sha) run dirs + the shared per-repo cache
# clone live. Env-tunable so a deployment can point it at fast disk.
DEFAULT_RUNS_ROOT = Path(
    os.environ.get("CONTROLLER_LOCAL_CI_ROOT", "/tmp/cleveragents-local-ci")
)

# Hard ceiling on how long one local CI run may take before the worker
# gives up and records a failure. The fork's full CI is ~25 min; 60 min
# leaves generous headroom for a cold Docker image pull.
DEFAULT_RUN_TIMEOUT_S: float = float(
    os.environ.get("CONTROLLER_LOCAL_CI_TIMEOUT_S", "3600")
)

# Backpressure: at most this many local CI runs in flight at once.
# Default 1 — SERIAL. Each run already parallelizes its own jobs across
# ~13 containers (the workflow's needs-graph), which is the real win.
# Running two *workflows* at once instead races the Docker daemon:
# concurrent container create/remove across ~26 containers produces
# "No such container" / "RWLayer is unexpectedly nil" errors that fail
# jobs spuriously. One workflow at a time mirrors how a cluster runner
# executes CI. Operators with a beefy Docker host can raise
# CONTROLLER_LOCAL_CI_MAX_CONCURRENT.
DEFAULT_MAX_CONCURRENT: int = int(
    os.environ.get("CONTROLLER_LOCAL_CI_MAX_CONCURRENT", "1")
)

# How long a finished run dir is kept before the GC sweep removes it.
# Finished runs are retained so a re-poll of the same head_sha is served
# from the on-disk cache instead of re-running CI; past this age that
# reuse is moot (the workflow has long since moved on) and reclaiming
# the disk matters more. Env-tunable.
DEFAULT_RETENTION_S: float = float(
    os.environ.get("CONTROLLER_LOCAL_CI_RETENTION_S", "86400")
)

# git credential.helper that sources the Forgejo token from
# ``$FORGEJO_TOKEN`` at fetch time — keeps the token out of
# ``.git/config``. Same pattern as worker/workspace.py's PerPRWorkspace.
_CREDENTIAL_HELPER = (
    '!f() { test "$1" = "get" && '
    'echo "username=x" && '
    'echo "password=${FORGEJO_TOKEN:-}"; }; f'
)

_STATUS_RUNNING = "running"
_STATUS_SUCCESS = "success"
_STATUS_FAILURE = "failure"

# Per-git-op timeout for the clone/fetch/worktree steps.
_GIT_TIMEOUT_S = 300

# Per-job curated-log-tail cap. Matches _ci_logs' own default
# (_DEFAULT_MAX_CHARS_PER_JOB) — enough for the failing assertion +
# stack-trace tail of any gate (ruff / mypy / pytest / nox).
_LOG_TAIL_MAX_CHARS = 4000


# get_ci_status(owner, repo, head_sha) -> Forgejo-shaped combined-status
# dict or None. get_failure_logs(owner, repo, head_sha) -> str.
GetCIStatusCallback = Callable[[str, str, str], dict | None]
GetFailureLogsCallback = Callable[[str, str, str], str]
SpawnCallback = Callable[..., None]


# ─── run-dir helpers ─────────────────────────────────────────────────


def _run_key(owner: str, repo: str, head_sha: str) -> str:
    """Filesystem-safe per-(owner,repo,head_sha) run-dir name."""
    safe_owner = owner.replace("/", "_")
    safe_repo = repo.replace("/", "_")
    return f"{safe_owner}__{safe_repo}__{head_sha[:16]}"


def _read_status(run_dir: Path) -> str | None:
    """Read the run's ``status`` file; None when absent/empty."""
    try:
        return (run_dir / "status").read_text(encoding="utf-8").strip() or None
    except (FileNotFoundError, NotADirectoryError):
        return None
    except OSError:
        return None


def _write_status(run_dir: Path, status: str) -> None:
    """Atomically write the run's ``status`` file (temp + rename) so a
    concurrent reader never sees a half-written value."""
    tmp = run_dir / "status.tmp"
    tmp.write_text(status, encoding="utf-8")
    os.replace(tmp, run_dir / "status")


def _append_log(run_dir: Path, text: str) -> None:
    try:
        with (run_dir / "ci.log").open("a", encoding="utf-8") as fh:
            fh.write(text)
    except OSError:
        logger.warning("local_ci: could not append to %s/ci.log", run_dir)


def _runner_is_dead(run_dir: Path) -> bool:
    """True when the run's detached runner process is gone.

    Returns False when no ``pid`` file exists yet (the runner is still
    starting) — conservative: only a confirmed-dead process is "dead".
    """
    try:
        pid = int((run_dir / "pid").read_text(encoding="utf-8").strip())
    except (FileNotFoundError, NotADirectoryError, ValueError, OSError):
        return False
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return True
    except PermissionError:
        return False
    return False


def _count_running(runs_root: Path) -> int:
    """Count live in-flight runs under ``runs_root`` (status=running and
    the runner process still alive). Dead-but-running dirs are skipped
    so a crashed run never permanently consumes a concurrency slot."""
    try:
        children = list(runs_root.iterdir())
    except (FileNotFoundError, NotADirectoryError):
        return 0
    n = 0
    for child in children:
        if not child.is_dir() or child.name == "_repos":
            continue
        if _read_status(child) == _STATUS_RUNNING and not _runner_is_dead(child):
            n += 1
    return n


def any_run_in_flight(runs_root: Path | str | None = None) -> bool:
    """True when at least one local CI run is currently executing
    (status=running with a live runner process).

    The master's ci_poll_exhaustion sweep consults this: while local CI
    is busy, every AWAITING_CI workflow is either being verified now or
    queued behind a run (local CI is serial by default), so its wait is
    legitimate progress — not a wedge. The poll-exhaustion timer must
    not STUCK a workflow whose on-demand verdict is minutes away."""
    root = Path(runs_root) if runs_root else DEFAULT_RUNS_ROOT
    return _count_running(root) > 0


def _gc_old_runs(runs_root: Path, *, max_age_s: float = DEFAULT_RETENTION_S) -> int:
    """Remove finished run dirs whose last activity is older than
    ``max_age_s``. Never touches a still-running run nor the shared
    ``_repos`` cache clone. Best-effort: per-dir errors are logged, not
    raised. Returns the count removed."""
    try:
        children = list(runs_root.iterdir())
    except OSError:
        return 0
    now = time.time()
    removed = 0
    for child in children:
        if not child.is_dir() or child.name == "_repos":
            continue
        if _read_status(child) == _STATUS_RUNNING and not _runner_is_dead(child):
            continue  # genuinely in flight — keep
        try:
            age = now - child.stat().st_mtime
        except OSError:
            continue
        if age < max_age_s:
            continue
        try:
            shutil.rmtree(child)
        except OSError as exc:
            logger.warning("local_ci: GC could not remove %s: %s", child, exc)
            continue
        removed += 1
    if removed:
        logger.info(
            "local_ci: GC removed %d finished run dir(s) older than %.0fs",
            removed,
            max_age_s,
        )
    return removed


def _status_dict(head_sha: str, state: str, gates: list[dict] | None = None) -> dict:
    """Synthesize a Forgejo-shaped combined-status dict.

    ``classify_ci_result`` / ``ci_status_poll`` read ``state`` + ``sha``
    + ``statuses``. ``statuses`` MUST be non-empty or the classifier
    returns ``no_ci`` (it filters statuses by SHA before anything else).
    Every entry deliberately carries NO timestamp so the classifier's
    staleness check is a no-op.

    ``gates`` (set once a run has finished) yields one ``statuses``
    entry per CI job, so prefetch's ``ci_summary`` / ``failing_gates``
    show the real per-gate breakdown. A pending run has no gates yet —
    one synthetic entry keeps ``statuses`` non-empty.
    """
    if gates:
        statuses = [
            {
                "context": g.get("context") or "local-ci",
                "state": g.get("state") or state,
                "description": f"local CI: {g.get('state') or state}",
            }
            for g in gates
        ]
    else:
        statuses = [
            {
                "context": "local-ci",
                "state": state,
                "description": f"tools/run-ci-full-local.sh: {state}",
            }
        ]
    return {"state": state, "sha": head_sha, "statuses": statuses}


# ─── forgejo-runner log parsing ──────────────────────────────────────


# forgejo-runner (act) prefixes every line "[<workflow>/<job>] ".
_JOB_LINE_RE = re.compile(r"^\[([^\]]+)\]\s?(.*)$")

# act step/job-boundary markers (leading-whitespace-tolerant).
_STEP_START = "⭐"
_STEP_PASS = "✅"
_STEP_FAIL = "❌"
_JOB_END = "🏁"
# Lines starting with one of these are act orchestration, not real
# command output — dropped from curated log tails.
_NOISE_MARKERS = (_STEP_START, _STEP_PASS, _STEP_FAIL, _JOB_END, "☁️", "🐳", "💬")


def _is_noise_line(line: str) -> bool:
    s = line.lstrip()
    if any(s.startswith(m) for m in _NOISE_MARKERS):
        return True
    low = s.lower()
    return "skipping post step" in low or low.startswith("cleaning up")


def _drop_infra_lines(text: str) -> str:
    """Drop any line carrying an infra-failure signature. Defensive
    belt-and-suspenders so a curated tail can never false-trip the
    infra-vs-real classifier — local CI has no infra failure mode."""
    sigs = [s.lower() for s in INFRA_FAILURE_SIGNATURES]
    return "\n".join(
        ln for ln in text.splitlines() if not any(sig in ln.lower() for sig in sigs)
    )


def _truncate_tail(text: str) -> str | None:
    text = text.strip()
    if not text:
        return None
    if len(text) > _LOG_TAIL_MAX_CHARS:
        text = "...[truncated head]...\n" + text[-_LOG_TAIL_MAX_CHARS:]
    return text


def _job_verdict(lines: list[str]) -> str:
    """Per-job verdict from the act '🏁 Job ...' line(s)."""
    seen: set[str] = set()
    for ln in lines:
        s = ln.strip()
        if _JOB_END not in s:
            continue
        if "Job failed" in s:
            seen.add("failure")
        elif "Job succeeded" in s:
            seen.add("success")
        elif "Job skipped" in s:
            seen.add("skipped")
    if "failure" in seen:
        return "failure"
    if "success" in seen:
        return "success"
    if "skipped" in seen:
        return "skipped"
    return "unknown"


def _failing_step_output(lines: list[str]) -> str | None:
    """Curated output of a failed job's failing steps.

    Walks the job's lines, buffering each step's output between its
    ``⭐ Run`` start and its verdict; a ``❌ Failure`` flushes the
    buffer (plus the failure header) into the result, a ``✅ Success``
    discards it. Orchestration + infra-signature lines are stripped.
    Falls back to the whole job body when no per-step ``❌`` is found.
    """
    failed: list[str] = []
    current: list[str] = []
    for ln in lines:
        s = ln.lstrip()
        if s.startswith(_STEP_START):
            current = []
            continue
        if s.startswith(_STEP_PASS):
            current = []
            continue
        if s.startswith(_STEP_FAIL):
            failed.append(s)  # "❌  Failure - <step>" header
            failed.extend(current)
            current = []
            continue
        if _is_noise_line(ln):
            continue
        current.append(ln)
    text = "\n".join(failed).strip()
    if not text:
        text = "\n".join(ln for ln in lines if not _is_noise_line(ln)).strip()
    return _truncate_tail(_drop_infra_lines(text))


def _fallback_log_tail(log_text: str) -> str | None:
    """Curated tail of the whole run log — used when per-job parsing
    finds no recognizable job sections."""
    return _truncate_tail(_drop_infra_lines(log_text))


def _parse_forgejo_runner_log(log_text: str) -> list[dict]:
    """Parse a ``forgejo-runner exec`` combined log into per-job results.

    Returns ``[{job, context, state, log_tail}]`` — ``state`` is
    success / failure / skipped / unknown; ``log_tail`` is the curated
    failing-step output for a failed job, else None. An unrecognized
    format yields ``[]`` (the caller then falls back to the overall
    exit code).
    """
    jobs: dict[str, dict] = {}
    order: list[str] = []
    for raw in log_text.splitlines():
        m = _JOB_LINE_RE.match(raw)
        if not m:
            continue
        tag, content = m.group(1), m.group(2)
        if "/" not in tag:
            continue
        workflow, _, job = tag.rpartition("/")
        # forgejo-runner pads the [workflow/job] tag with spaces to
        # column-align its log output — strip that padding.
        workflow, job = workflow.strip(), job.strip()
        if not job:
            continue
        if job not in jobs:
            jobs[job] = {"workflow": workflow, "lines": []}
            order.append(job)
        jobs[job]["lines"].append(content)

    results: list[dict] = []
    for job in order:
        info = jobs[job]
        state = _job_verdict(info["lines"])
        results.append(
            {
                "job": job,
                "context": f"{info['workflow']} / {job}",
                "state": state,
                "log_tail": (
                    _failing_step_output(info["lines"]) if state == "failure" else None
                ),
            }
        )
    return results


def _per_job_full_logs(log_text: str) -> dict[str, str]:
    """Per-job FULL log text from a forgejo-runner combined log, keyed
    by the ``"workflow / job"`` context (the commit-status context
    shape). Unlike :func:`_parse_forgejo_runner_log` this keeps the
    whole job body, untruncated — for the get_ci_logs bundle."""
    jobs: dict[str, list[str]] = {}
    for raw in log_text.splitlines():
        m = _JOB_LINE_RE.match(raw)
        if not m:
            continue
        tag, content = m.group(1), m.group(2)
        if "/" not in tag:
            continue
        workflow, _, job = tag.rpartition("/")
        workflow, job = workflow.strip(), job.strip()
        if not job:
            continue
        jobs.setdefault(f"{workflow} / {job}", []).append(content)
    return {ctx: "\n".join(lines) for ctx, lines in jobs.items()}


# ─── per-run result file (result.json) ───────────────────────────────


def _write_result(run_dir: Path, overall: str, jobs: list[dict]) -> None:
    """Atomically write result.json — the internal per-run summary read
    by ``get_ci_status`` (per-gate breakdown) + ``get_failure_logs``."""
    payload = {"overall": overall, "jobs": jobs}
    tmp = run_dir / "result.json.tmp"
    tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8")
    os.replace(tmp, run_dir / "result.json")


def _read_result(run_dir: Path) -> dict | None:
    try:
        raw = (run_dir / "result.json").read_text(encoding="utf-8")
    except (FileNotFoundError, NotADirectoryError, OSError):
        return None
    try:
        data = json.loads(raw)
    except ValueError:
        return None
    return data if isinstance(data, dict) else None


def _gates_from_result(run_dir: Path) -> list[dict] | None:
    """Per-gate ``[{context, state}]`` from result.json, for the
    combined-status ``statuses`` list. None when the run produced no
    parsable result."""
    result = _read_result(run_dir)
    if not result:
        return None
    jobs = result.get("jobs")
    if not isinstance(jobs, list) or not jobs:
        return None
    return [
        {
            "context": j.get("context") or "local-ci",
            "state": j.get("state") or "unknown",
        }
        for j in jobs
        if isinstance(j, dict)
    ]


# ─── clone-URL helper ────────────────────────────────────────────────


def default_clone_url(owner: str, repo: str) -> str | None:
    """Build the Forgejo clone URL from ``FORGEJO_URL`` (or a stripped
    ``FORGEJO_API_BASE``). Returns None when neither yields a usable
    base — the caller then refuses to start local-CI mode.

    The token is NOT embedded in the URL; the runner configures a
    ``credential.helper`` sourcing ``$FORGEJO_TOKEN`` instead.
    """
    base = os.environ.get("FORGEJO_URL", "").rstrip("/")
    if not base:
        api = os.environ.get("FORGEJO_API_BASE", "").rstrip("/")
        suffix = "/api/v1"
        base = api[: -len(suffix)] if api.endswith(suffix) else api
    base = base.rstrip("/")
    if not base or "://" not in base:
        return None
    return f"{base}/{owner}/{repo}.git"


# ─── startup preflight ───────────────────────────────────────────────


def resolve_runner_bin(repo_root: str | Path) -> str | None:
    """Locate the ``forgejo-runner`` binary the same way
    ``run-ci-full-local.sh`` does: ``$FORGEJO_RUNNER_BIN`` → ``PATH`` →
    ``<repo>/tools/.bin/forgejo-runner``. None when none resolve."""
    env_bin = os.environ.get("FORGEJO_RUNNER_BIN", "").strip()
    if env_bin:
        return env_bin if os.access(env_bin, os.X_OK) else None
    on_path = shutil.which("forgejo-runner")
    if on_path:
        return on_path
    local = Path(repo_root) / "tools" / ".bin" / "forgejo-runner"
    return str(local) if os.access(local, os.X_OK) else None


def preflight_local_ci(repo_root: str | Path) -> str | None:
    """Verify the host can actually run local CI — the forgejo-runner
    binary resolves AND the Docker daemon is reachable. Returns an
    operator-facing error string when something is missing, else None.

    Called at master startup so a ``RUN_CI_LOCAL`` misconfiguration
    fails loudly there instead of silently turning into a red CI
    verdict on every PR (the runner ``die``s mid-run otherwise)."""
    if resolve_runner_bin(repo_root) is None:
        return (
            "forgejo-runner binary not found — install it (single Go "
            "binary: https://code.forgejo.org/forgejo/runner/releases) "
            "then put it on PATH, at tools/.bin/forgejo-runner, or set "
            "FORGEJO_RUNNER_BIN"
        )
    if shutil.which("docker") is None:
        return "docker not found on PATH (local CI runs each CI job in a container)"
    try:
        proc = subprocess.run(
            ["docker", "info"],
            capture_output=True,
            text=True,
            timeout=30,
        )
    except (OSError, subprocess.SubprocessError) as exc:
        return f"could not probe the Docker daemon: {exc}"
    if proc.returncode != 0:
        return "Docker daemon not reachable (`docker info` failed) — is it running?"
    return None


# ─── public factory ──────────────────────────────────────────────────


def build_local_ci_callbacks(
    *,
    clone_url: str,
    script_path: str,
    repo_root: str,
    forgejo_token: str = "",
    runs_root: Path | str | None = None,
    max_concurrent: int = DEFAULT_MAX_CONCURRENT,
    _spawn: SpawnCallback | None = None,
) -> tuple[GetCIStatusCallback, GetFailureLogsCallback]:
    """Build the ``(get_ci_status, get_failure_logs)`` pair backed by
    local CI runs.

    Args:
        clone_url: Forgejo clone URL (no embedded credentials).
        script_path: absolute path to ``tools/run-ci-full-local.sh``.
        repo_root: the controller repo root — the cwd for the detached
            ``python -m tools.controller.master.local_ci worker`` spawn.
        forgejo_token: the Forgejo token; passed to the runner via
            ``$FORGEJO_TOKEN`` for the credential helper.
        runs_root: override the on-disk run-cache root (tests).
        max_concurrent: ceiling on simultaneous local CI runs.
        _spawn: injectable runner-spawn for tests; defaults to the
            real detached-subprocess spawner.
    """
    root = Path(runs_root) if runs_root else DEFAULT_RUNS_ROOT
    spawn = _spawn or _default_spawn

    def get_ci_status(owner: str, repo: str, head_sha: str) -> dict | None:
        if not head_sha:
            return None
        run_dir = root / _run_key(owner, repo, head_sha)
        status = _read_status(run_dir)

        # No run for this SHA yet — claim + spawn (respecting the cap).
        if status is None and not run_dir.exists():
            running = _count_running(root)
            if running >= max_concurrent:
                logger.info(
                    "local_ci: %d/%d runs in flight; deferring %s/%s @%s",
                    running,
                    max_concurrent,
                    owner,
                    repo,
                    head_sha[:12],
                )
                return _status_dict(head_sha, "pending")
            # A new head_sha — opportunistically GC stale finished runs
            # so the on-disk run cache doesn't grow without bound.
            _gc_old_runs(root)
            try:
                run_dir.mkdir(parents=True, exist_ok=False)
            except FileExistsError:
                # Another tick claimed it between the read and the
                # mkdir — fall through to the status re-read below.
                pass
            else:
                _write_status(run_dir, _STATUS_RUNNING)
                try:
                    spawn(
                        run_dir=run_dir,
                        owner=owner,
                        repo=repo,
                        head_sha=head_sha,
                        clone_url=clone_url,
                        script_path=script_path,
                        repo_root=repo_root,
                        runs_root=root,
                        forgejo_token=forgejo_token,
                    )
                    logger.info(
                        "local_ci: spawned CI run for %s/%s @%s",
                        owner,
                        repo,
                        head_sha[:12],
                    )
                except Exception:
                    logger.exception(
                        "local_ci: failed to spawn runner for %s", head_sha[:12]
                    )
                    _write_status(run_dir, _STATUS_FAILURE)
                    return _status_dict(head_sha, "failure")
                return _status_dict(head_sha, "pending")

        # Re-read (covers the claim-race fall-through above).
        status = _read_status(run_dir)
        if status == _STATUS_SUCCESS:
            return _status_dict(head_sha, "success", _gates_from_result(run_dir))
        if status == _STATUS_FAILURE:
            return _status_dict(head_sha, "failure", _gates_from_result(run_dir))

        # status is running (or not yet written). If the runner died
        # without recording a verdict, record failure — but re-read
        # first: the runner may have written its verdict in the window
        # between our status read and the liveness check.
        if _runner_is_dead(run_dir):
            final = _read_status(run_dir)
            if final in (_STATUS_SUCCESS, _STATUS_FAILURE):
                return _status_dict(head_sha, final, _gates_from_result(run_dir))
            logger.warning(
                "local_ci: runner for %s/%s @%s exited without a verdict; "
                "recording failure",
                owner,
                repo,
                head_sha[:12],
            )
            _write_status(run_dir, _STATUS_FAILURE)
            return _status_dict(head_sha, "failure")
        return _status_dict(head_sha, "pending")

    def get_failure_logs(owner: str, repo: str, head_sha: str) -> str:
        """Concatenate the curated log tails of the run's failing jobs.

        Read from the run's result.json — the tails are already
        stripped of forgejo-runner orchestration + infra-signature
        noise, so the infra-vs-real classifier sees genuine error text
        and lands on ``fresh_real`` (correct: local CI has no infra
        failure mode). Returns "" when the run hasn't finished or had
        no failing job.
        """
        if not head_sha:
            return ""
        run_dir = root / _run_key(owner, repo, head_sha)
        result = _read_result(run_dir)
        if not result:
            return ""
        chunks: list[str] = []
        for job in result.get("jobs") or []:
            if not isinstance(job, dict) or job.get("state") != "failure":
                continue
            tail = job.get("log_tail")
            if isinstance(tail, str) and tail.strip():
                label = job.get("context") or job.get("job") or "ci"
                chunks.append(f"=== {label} ===\n{tail}")
        return "\n\n".join(chunks)

    return get_ci_status, get_failure_logs


# ─── detached-runner spawn ───────────────────────────────────────────


def _default_spawn(
    *,
    run_dir: Path,
    owner: str,
    repo: str,
    head_sha: str,
    clone_url: str,
    script_path: str,
    repo_root: str,
    runs_root: Path,
    forgejo_token: str,
) -> None:
    """Spawn the detached ``worker`` subprocess that actually runs CI.

    ``start_new_session=True`` puts the runner in its own session so an
    ordinary master shutdown doesn't kill an in-flight CI run — the
    result lands on disk and is reused after a restart. (An explicit
    ``--stop`` teardown still reaps it via ``pkill -f``.)
    """
    env = dict(os.environ)
    if forgejo_token:
        env["FORGEJO_TOKEN"] = forgejo_token
    args = [
        sys.executable,
        "-m",
        "tools.controller.master.local_ci",
        "worker",
        "--run-dir",
        str(run_dir),
        "--owner",
        owner,
        "--repo",
        repo,
        "--sha",
        head_sha,
        "--clone-url",
        clone_url,
        "--script",
        script_path,
        "--runs-root",
        str(runs_root),
    ]
    out = (run_dir / "runner.out").open("w", encoding="utf-8")
    try:
        subprocess.Popen(
            args,
            cwd=repo_root,
            env=env,
            stdout=out,
            stderr=subprocess.STDOUT,
            stdin=subprocess.DEVNULL,
            start_new_session=True,
            close_fds=True,
        )
    finally:
        out.close()


# ─── git helpers (runner side) ───────────────────────────────────────


@contextmanager
def _file_lock(lock_path: Path):
    """Coarse advisory lock over the shared per-repo cache clone, held
    only for the cheap clone/fetch/worktree steps — NOT the long CI
    run."""
    import fcntl

    lock_path.parent.mkdir(parents=True, exist_ok=True)
    fh = lock_path.open("w")
    try:
        fcntl.flock(fh, fcntl.LOCK_EX)
        yield
    finally:
        try:
            fcntl.flock(fh, fcntl.LOCK_UN)
        finally:
            fh.close()


def _git(args: list[str], *, cwd: Path | None = None) -> None:
    """Run a git command; raise RuntimeError on failure."""
    try:
        subprocess.run(
            ["git", *args],
            cwd=str(cwd) if cwd else None,
            capture_output=True,
            text=True,
            check=True,
            timeout=_GIT_TIMEOUT_S,
        )
    except subprocess.CalledProcessError as exc:
        raise RuntimeError(
            f"git {' '.join(args)} failed: {exc.stderr.strip()}"
        ) from exc
    except subprocess.TimeoutExpired as exc:
        raise RuntimeError(
            f"git {' '.join(args)} timed out ({_GIT_TIMEOUT_S}s)"
        ) from exc


def _ensure_cache_clone(cache: Path, clone_url: str) -> None:
    """Clone the repo into ``cache`` if absent; configure the
    token-sourcing credential helper. Idempotent."""
    if (cache / ".git").exists():
        return
    if cache.exists():
        shutil.rmtree(cache)
    cache.parent.mkdir(parents=True, exist_ok=True)
    _git(
        [
            "-c",
            f"credential.helper={_CREDENTIAL_HELPER}",
            "clone",
            "--no-single-branch",
            clone_url,
            str(cache),
        ]
    )
    _git(
        ["config", "--local", "credential.helper", _CREDENTIAL_HELPER],
        cwd=cache,
    )


def _fetch_all(cache: Path) -> None:
    """Refresh remote refs in the cache clone — branch heads plus PR
    head refs (so a fork-origin PR's head SHA is also reachable)."""
    _git(["fetch", "origin", "--prune"], cwd=cache)
    # PR head refs are best-effort: a same-repo PR head is already a
    # branch, but a fork-origin PR's SHA only lives under refs/pull/*.
    try:
        _git(
            ["fetch", "origin", "+refs/pull/*/head:refs/remotes/pr/*"],
            cwd=cache,
        )
    except RuntimeError as exc:
        logger.info("local_ci: PR-ref fetch skipped (%s)", exc)


def _add_worktree(cache: Path, checkout: Path, head_sha: str) -> None:
    """Create a detached worktree at ``head_sha`` for an isolated
    checkout, clearing any residue from a crashed prior run."""
    if checkout.exists():
        shutil.rmtree(checkout, ignore_errors=True)
    _git(["worktree", "prune"], cwd=cache)
    _git(
        ["worktree", "add", "--detach", "--force", str(checkout), head_sha],
        cwd=cache,
    )


def _remove_worktree(cache: Path, checkout: Path) -> None:
    """Best-effort worktree teardown."""
    try:
        _git(["worktree", "remove", "--force", str(checkout)], cwd=cache)
    except RuntimeError as exc:
        logger.info("local_ci: worktree remove failed (%s)", exc)
    shutil.rmtree(checkout, ignore_errors=True)


# ─── detached worker entry point ─────────────────────────────────────


def _execute_run(
    *,
    owner: str,
    repo: str,
    head_sha: str,
    clone_url: str,
    script_path: str,
    runs_root: Path,
    run_dir: Path,
    timeout_s: float,
) -> str:
    """Do the actual work: checkout ``head_sha``, run CI, return the
    verdict (``success`` / ``failure``)."""
    repos_dir = runs_root / "_repos"
    cache = repos_dir / f"{owner.replace('/', '_')}__{repo.replace('/', '_')}"
    lock_path = repos_dir / f"{owner.replace('/', '_')}__{repo.replace('/', '_')}.lock"
    checkout = run_dir / "checkout"

    # The lock is held only for the cheap clone/fetch/worktree steps so
    # concurrent runs for other PRs aren't blocked by a long CI run.
    with _file_lock(lock_path):
        _ensure_cache_clone(cache, clone_url)
        _fetch_all(cache)
        _add_worktree(cache, checkout, head_sha)

    try:
        rc = _run_ci_script(script_path, checkout, run_dir, timeout_s)
    finally:
        with _file_lock(lock_path):
            _remove_worktree(cache, checkout)
        # The per-run $HOME held forgejo-runner's act cache (a multi-
        # hundred-MB bolt.db) — needed only while CI runs. Drop it now so
        # finished run dirs stay small until the GC sweep removes them.
        shutil.rmtree(run_dir / "home", ignore_errors=True)
    overall = _STATUS_SUCCESS if rc == 0 else _STATUS_FAILURE
    # Parse the run log into per-job results + surface them. Best
    # effort — the status verdict is the load-bearing output, so a
    # summary failure must not flip the verdict.
    try:
        _summarize_run(run_dir, head_sha=head_sha, overall=overall)
    except Exception:
        logger.exception("local_ci: failed to summarize run for %s", head_sha[:12])
    return overall


def _summarize_run(run_dir: Path, *, head_sha: str, overall: str) -> None:
    """Parse the run's ci.log into per-job results, then write
    result.json (consumed by get_ci_status / get_failure_logs) and the
    shared _ci_logs per-SHA cache (consumed by the implementer's CI
    MCP tool)."""
    try:
        log_text = (run_dir / "ci.log").read_text(encoding="utf-8", errors="replace")
    except OSError:
        log_text = ""
    jobs = _parse_forgejo_runner_log(log_text)
    if not jobs:
        # No recognizable job sections — synthesize one gate from the
        # exit code so consumers still get a verdict + (on failure)
        # the raw log tail.
        jobs = [
            {
                "job": "ci",
                "context": "CI / ci",
                "state": overall,
                "log_tail": (
                    _fallback_log_tail(log_text) if overall == _STATUS_FAILURE else None
                ),
            }
        ]
    elif overall == _STATUS_FAILURE and not any(j["state"] == "failure" for j in jobs):
        # Exit code says failure but no job parsed as failed — append a
        # synthetic failing gate so the implementer still gets detail.
        jobs.append(
            {
                "job": "ci",
                "context": "CI / ci",
                "state": "failure",
                "log_tail": _fallback_log_tail(log_text),
            }
        )
    _write_result(run_dir, overall, jobs)
    _write_ci_logs_cache(head_sha, jobs, log_text)


def _write_ci_logs_cache(head_sha: str, jobs: list[dict], log_text: str) -> None:
    """Populate the unified ``get_ci_logs`` bundle from a local CI run
    (``source="local"``), so every consumer — the implementer/reviewer
    ``ci_summary`` AND the implementer's ``ci_fetch_pr_failure_logs``
    MCP tool — reads local-CI logs through the same entry point as
    Forgejo CI. (B1: the legacy ``{sha}.json`` failing-tail cache is
    retired; ``fetch_pr_failure_logs`` is now a view over the bundle.)

    Best effort: a failure here only costs a log channel; the
    implementer prompt's ``ci_summary`` still works off get_ci_status.
    """
    try:
        from tools import _ci_logs  # type: ignore[import-not-found]
    except Exception as exc:
        logger.warning(
            "local_ci: _ci_logs import failed; CI MCP tool won't see local logs: %s",
            exc,
        )
        return

    # The unified get_ci_logs bundle — ALL jobs, full per-job logs
    # from ci.log, source="local". This is what the implementer/
    # reviewer ci_summary reads (via the get_ci_logs callback) and what
    # fetch_pr_failure_logs projects for the MCP tool.
    try:
        full_by_ctx = _per_job_full_logs(log_text)
        bundle_jobs = [
            {
                "context": j.get("context") or j.get("job") or "ci",
                "state": j.get("state"),
                "description": "local CI (forgejo-runner exec)",
                "log": (
                    full_by_ctx.get(j.get("context") or "") or j.get("log_tail") or ""
                ),
            }
            for j in jobs
        ]
        _ci_logs.put_local_bundle(head_sha, bundle_jobs)
    except Exception as exc:  # noqa: BLE001
        logger.warning("local_ci: get_ci_logs bundle write failed: %s", exc)


def _run_ci_script(
    script_path: str,
    checkout: Path,
    run_dir: Path,
    timeout_s: float,
) -> int:
    """Run ``run-ci-full-local.sh --repo-root <checkout>``, teeing the
    combined output into ``<run_dir>/ci.log``. Returns the exit code
    (non-zero on CI failure or timeout)."""
    cmd = ["bash", script_path, "--repo-root", str(checkout)]
    # Per-run HOME so concurrent runs don't contend on forgejo-runner's
    # act cache: ``~/.cache/actcache/bolt.db`` is single-writer bolt —
    # a second concurrent ``forgejo-runner exec`` otherwise dies at
    # startup with ``Open(...bolt.db): timeout``. forgejo-runner derives
    # the cache path from $HOME and ignores $XDG_CACHE_HOME, so an
    # isolated HOME is the only lever.
    run_home = run_dir / "home"
    run_home.mkdir(parents=True, exist_ok=True)
    env = dict(os.environ)
    env["HOME"] = str(run_home)
    with (run_dir / "ci.log").open("w", encoding="utf-8") as logf:
        logf.write(f"[local_ci] {' '.join(cmd)}\n\n")
        logf.flush()
        try:
            proc = subprocess.run(
                cmd,
                cwd=str(checkout),
                stdout=logf,
                stderr=subprocess.STDOUT,
                timeout=timeout_s,
                env=env,
            )
        except subprocess.TimeoutExpired:
            logf.write(f"\n[local_ci] CI run timed out after {timeout_s:.0f}s\n")
            return 124
    return proc.returncode


def _worker_main(argv: list[str]) -> int:
    """Detached-runner entry point — invoked as
    ``python -m tools.controller.master.local_ci worker ...``."""
    import argparse

    parser = argparse.ArgumentParser(prog="local_ci worker")
    parser.add_argument("--run-dir", required=True)
    parser.add_argument("--owner", required=True)
    parser.add_argument("--repo", required=True)
    parser.add_argument("--sha", required=True)
    parser.add_argument("--clone-url", required=True)
    parser.add_argument("--script", required=True)
    parser.add_argument("--runs-root", required=True)
    parser.add_argument("--timeout", type=float, default=DEFAULT_RUN_TIMEOUT_S)
    args = parser.parse_args(argv)

    run_dir = Path(args.run_dir)
    run_dir.mkdir(parents=True, exist_ok=True)
    (run_dir / "pid").write_text(str(os.getpid()), encoding="utf-8")

    status = _STATUS_FAILURE
    try:
        status = _execute_run(
            owner=args.owner,
            repo=args.repo,
            head_sha=args.sha,
            clone_url=args.clone_url,
            script_path=args.script,
            runs_root=Path(args.runs_root),
            run_dir=run_dir,
            timeout_s=args.timeout,
        )
    except Exception as exc:
        logger.exception("local_ci: worker failed for %s", args.sha[:12])
        _append_log(run_dir, f"\n[local_ci] worker error: {exc}\n")
        status = _STATUS_FAILURE
    finally:
        _write_status(run_dir, status)
    logger.info("local_ci: run %s -> %s", args.sha[:12], status)
    return 0 if status == _STATUS_SUCCESS else 1


def main(argv: list[str] | None = None) -> int:
    argv = list(sys.argv[1:] if argv is None else argv)
    if argv and argv[0] == "worker":
        logging.basicConfig(
            level=logging.INFO,
            format="%(asctime)s %(levelname)s %(name)s: %(message)s",
        )
        return _worker_main(argv[1:])
    print(
        "usage: python -m tools.controller.master.local_ci worker "
        "--run-dir D --owner O --repo R --sha S --clone-url U "
        "--script P --runs-root D",
        file=sys.stderr,
    )
    return 2


__all__ = [
    "DEFAULT_MAX_CONCURRENT",
    "DEFAULT_RETENTION_S",
    "DEFAULT_RUNS_ROOT",
    "DEFAULT_RUN_TIMEOUT_S",
    "GetCIStatusCallback",
    "GetFailureLogsCallback",
    "any_run_in_flight",
    "build_local_ci_callbacks",
    "default_clone_url",
    "preflight_local_ci",
    "resolve_runner_bin",
]


if __name__ == "__main__":
    raise SystemExit(main())