cleveragents-core/tools/_ci_logs.py

"""Unified per-(head_sha) on-disk cache of CI job logs.

Why this exists
---------------

The dispatcher pre-fetches the per-check *statuses* (context + state
+ target_url) but not the log text — so a worker told ``CI / lint:
failure`` had to chase the cause through file reads and gate runs,
most of which never finished inside the worker window. This module
fetches the CI job logs ONCE per head_sha and serves them — to the
implementer/reviewer prefetch, the freshness gate, and the
``ci_fetch_pr_failure_logs`` MCP tool — through one shared cache.

The single store: the bundle
----------------------------

:func:`get_ci_logs` is the one entry point every consumer should
call. It returns a BUNDLE — EVERY job of the run (passing and
failing), each with its FULL untruncated log — cached at
``/tmp/cleveragents-ci-logs-cache/{head_sha}.full.json``.

:func:`fetch_pr_failure_logs` is a thin VIEW over the bundle that
projects it to the legacy failing-jobs-with-tails envelope (B1),
retained for the MCP tool + legacy prefetch callers. There is one
cache and one fetch path.

Freshness
---------

- A run still in-flight is cached ``partial=True`` and re-fetched on
  a minimum interval (``CI_LOGS_PARTIAL_MIN_REFETCH_S``, B2).
- A terminal run with a clean fetch is frozen — served straight from
  cache — but only while its ``run_id`` matches the live run: a SHA
  can be re-run, and a bundle whose run_id no longer matches is
  re-fetched (B3, last-run-wins).
- A terminal run with per-job fetch errors retries under an
  exponential backoff (``consecutive_failures`` /
  ``next_attempt_after``), mirroring :mod:`_pr_comments_cache`.

RUN_CI_LOCAL writes into the SAME cache via :func:`put_local_bundle`
(``source="local"``), so every consumer is source-agnostic. A 7-day
cleanup ceiling keeps the cache directory bounded.
"""

from __future__ import annotations

import datetime as _dt
import http.cookiejar
import json
import logging
import os
import re
import sys
import threading
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any

_TOOLS_DIR = str(Path(__file__).resolve().parent)
if _TOOLS_DIR not in sys.path:
    sys.path.insert(0, _TOOLS_DIR)
from _loader import (  # noqa: E402  type: ignore[import-not-found]
    load_sibling as _load_sibling,
)

_review_fetch = _load_sibling("_review_fetch", "_review_fetch.py")
_claim_runtime = _load_sibling("_claim_runtime", "_claim_runtime.py")
_backoff = _load_sibling("_backoff", "_backoff.py")

_logger = logging.getLogger("ci_logs")

SCHEMA_VERSION = 1

# ─── Session-cookie helper for the UI logs endpoint ──────────────────
#
# Forgejo's REST API does NOT expose Actions job logs on this server
# version (live-probed 2026-05-17: every ``/api/v1/.../actions/.../logs``
# variant returns 404). The UI route DOES serve raw text at
# ``/{owner}/{repo}/actions/runs/{run_index}/jobs/{job_index}/attempt/{N}/logs``
# but requires a session cookie (token-auth ignored — verified live).
#
# This helper logs in once via ``FORGEJO_PASSWORD`` (env-set by the
# launcher) and caches the cookie-jar at module scope. Subsequent
# fetches reuse the cookie until the session expires; on 401/302-to-
# login we re-login transparently.
#
# Thread-safe via ``_login_lock``. Disabled when ``FORGEJO_PASSWORD``
# is unset — the fetcher returns ``no-session-password`` errors so the
# graceful-degradation path in ``collect_failing_jobs`` continues to
# work.

_session_cookies: http.cookiejar.CookieJar | None = None
_login_lock = threading.Lock()


def _do_session_login(cfg: Any) -> http.cookiejar.CookieJar | None:
    """Acquire a session cookie via the UI login form. Returns a
    populated cookie-jar on success, ``None`` on any failure
    (missing password / CSRF parse error / non-2xx login)."""
    password = os.environ.get("FORGEJO_PASSWORD", "").strip()
    if not password:
        return None
    username = os.environ.get("FORGEJO_USERNAME", "").strip() or "HAL9000"
    base_url = str(cfg.forgejo_url).rstrip("/")
    jar = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(
        urllib.request.HTTPCookieProcessor(jar),
        urllib.request.HTTPRedirectHandler(),
    )
    # Fetch the login form to seed CSRF + initial cookie. Some
    # Forgejo configs require the CSRF token; others ignore it.
    csrf = ""
    try:
        req = urllib.request.Request(
            f"{base_url}/user/login",
            headers={"User-Agent": "cleveragents-ci-logs/1.0"},
        )
        with opener.open(req, timeout=15) as resp:
            html = resp.read().decode("utf-8", errors="replace")
        m = re.search(r'name="_csrf"\s+value="([^"]+)"', html)
        if m:
            csrf = m.group(1)
    except (urllib.error.URLError, OSError, ValueError) as exc:
        _logger.warning("ci-logs session: GET /user/login failed: %s", exc)
        return None
    # POST credentials. Include CSRF if we got one; many builds
    # accept the POST without it.
    form_fields = {"user_name": username, "password": password}
    if csrf:
        form_fields["_csrf"] = csrf
    body = urllib.parse.urlencode(form_fields).encode("utf-8")
    try:
        req = urllib.request.Request(
            f"{base_url}/user/login",
            data=body,
            headers={
                "User-Agent": "cleveragents-ci-logs/1.0",
                "Content-Type": "application/x-www-form-urlencoded",
            },
        )
        with opener.open(req, timeout=15) as resp:
            status = resp.status
    except urllib.error.HTTPError as exc:
        status = exc.code
    except (urllib.error.URLError, OSError) as exc:
        _logger.warning("ci-logs session: POST /user/login failed: %s", exc)
        return None
    # Success path: status 200 with a session cookie set. A redirect
    # to /user/login again means auth failed.
    has_session = any(
        c.name in {"i_like_gitea", "session", "gitea_incredible"} for c in jar
    )
    if status >= 400 or not has_session:
        _logger.warning(
            "ci-logs session: login returned status=%s session_cookie=%s",
            status,
            has_session,
        )
        return None
    return jar


def _get_session_cookies(cfg: Any) -> http.cookiejar.CookieJar | None:
    """Return the cached cookie-jar, logging in once on first call.
    Thread-safe."""
    global _session_cookies
    if _session_cookies is not None:
        return _session_cookies
    with _login_lock:
        if _session_cookies is not None:
            return _session_cookies
        _session_cookies = _do_session_login(cfg)
        return _session_cookies


def _invalidate_session() -> None:
    """Drop the cached cookie-jar — forces a fresh login on next
    call. Used when a request returns 401/302-to-login."""
    global _session_cookies
    with _login_lock:
        _session_cookies = None


def _ui_fetch_with_session(cfg: Any, path: str) -> dict[str, Any]:
    """Fetch ``base_url + path`` using the cached session cookie.
    Returns a dict shaped like ``_claim_runtime.get``'s return:
    ``{"status": int, "body": bytes|str}``.

    Auto-relogin once on 401 or login-page redirect. After the second
    failure, returns the error status so the caller's backoff logic
    fires normally."""
    base_url = str(cfg.forgejo_url).rstrip("/")
    for attempt in (1, 2):
        jar = _get_session_cookies(cfg)
        if jar is None:
            return {"status": 0, "body": b"", "error": "no-session-password"}
        opener = urllib.request.build_opener(
            urllib.request.HTTPCookieProcessor(jar),
            urllib.request.HTTPRedirectHandler(),
        )
        try:
            req = urllib.request.Request(
                f"{base_url}{path}",
                headers={"User-Agent": "cleveragents-ci-logs/1.0"},
            )
            with opener.open(req, timeout=30) as resp:
                status = resp.status
                body = resp.read()
                final_url = resp.geturl()
        except urllib.error.HTTPError as exc:
            status = exc.code
            body = b""
            final_url = path
        except (urllib.error.URLError, OSError) as exc:
            return {"status": 0, "body": b"", "error": str(exc)}
        # Detect session expiry: a 302 redirect chain ended at the
        # login page, OR a 401. Invalidate + retry once.
        if attempt == 1 and (
            status == 401
            or (final_url.endswith("/user/login") and not path.endswith("/user/login"))
        ):
            _invalidate_session()
            continue
        return {"status": status, "body": body}
    return {"status": 0, "body": b"", "error": "session-retry-exhausted"}


# Defaults — overridable via env vars.
_DEFAULT_CACHE_DIR = Path("/tmp/cleveragents-ci-logs-cache")
_CACHE_DIR_ENV = "CI_LOGS_CACHE_DIR"

# Kill switch for tests / operators bypassing the cache.
_DISABLE_ENV = "CI_LOGS_CACHE_DISABLE"

# Per-job log-tail size cap. 4000 chars (~1000 tokens) is enough
# to see the failing assertion + last few lines of stack trace for
# every CI gate we've observed (ruff, mypy, pytest, behave, nox
# coverage). Operators can override for unusually verbose gates.
_DEFAULT_MAX_CHARS_PER_JOB = 4000
_MAX_CHARS_ENV = "CI_LOGS_MAX_CHARS_PER_JOB"

# Cap on jobs fetched per run. ``get_ci_logs`` fetches ALL jobs (not
# just failing), so the cap must clear a full CI matrix — a 12-gate run
# is normal here. ``collect_all_jobs`` additionally orders failing jobs
# first, so even if the cap bites it never drops a failing gate.
_DEFAULT_MAX_JOBS = 20
_MAX_JOBS_ENV = "CI_LOGS_MAX_JOBS"

# Exponential backoff for repeated live-fetch failures. Shared shape
# via :class:`_backoff.Backoff` so the three cache modules pull from
# one source of truth for the curve + env vocabulary.
_BACKOFF = _backoff.Backoff(
    base_env="CI_LOGS_BACKOFF_BASE_S",
    max_env="CI_LOGS_BACKOFF_MAX_S",
    base_default=60,
    max_default=1800,
)

# B2: minimum interval between live re-fetches of a PARTIAL bundle
# (CI still in-flight). A partial bundle is expected to change, so it
# is never frozen — but without a floor every ``get_ci_logs`` call
# would re-hit the Forgejo UI login + per-job log endpoints. Inside
# this window the cached partial is served as-is. 60s is well under
# any CI stage duration, so the in-flight view is never meaningfully
# stale. Env-tunable for tests / operators.
_PARTIAL_MIN_REFETCH_ENV = "CI_LOGS_PARTIAL_MIN_REFETCH_S"
_DEFAULT_PARTIAL_MIN_REFETCH_S = 60

# Cache file cleanup ceiling. Older than this -> safe to delete on
# a janitor pass (not implemented here; this is just the contract).
_DEFAULT_MAX_STALENESS_S = 7 * 24 * 3600
_STALENESS_ENV = "CI_LOGS_MAX_STALENESS_S"


# target_url shapes we know how to parse:
#   1. https://<host>/<owner>/<repo>/actions/runs/<run_id>/jobs/<job_id>
#      Forgejo Actions UI link. Has both ids inline.
#   2. https://<host>/<owner>/<repo>/actions/runs/<run_id>
#      Forgejo Actions UI link without per-job index. Need to list
#      jobs in the run to map a check context to a specific job_id.
#   3. Anything else (Codecov, external CI, custom Forgejo plugin):
#      we can't fetch its logs through the Forgejo Actions API.
#      Emit the job entry with ``fetch_error="unsupported-url-shape"``
#      so the agent can see it and follow the link manually if needed.
_TARGET_URL_RUN_JOB_RE = re.compile(r"/actions/runs/(\d+)/jobs/(\d+)\b")
_TARGET_URL_RUN_ONLY_RE = re.compile(r"/actions/runs/(\d+)\b")


def cache_dir() -> Path:
    return Path(os.environ.get(_CACHE_DIR_ENV) or str(_DEFAULT_CACHE_DIR))


def _max_chars_per_job() -> int:
    raw = os.environ.get(_MAX_CHARS_ENV)
    if raw:
        try:
            return max(256, int(raw))
        except ValueError:
            pass
    return _DEFAULT_MAX_CHARS_PER_JOB


def _max_jobs() -> int:
    raw = os.environ.get(_MAX_JOBS_ENV)
    if raw:
        try:
            return max(1, int(raw))
        except ValueError:
            pass
    return _DEFAULT_MAX_JOBS


def is_disabled() -> bool:
    raw = os.environ.get(_DISABLE_ENV, "").strip().lower()
    return raw in {"1", "true", "yes", "on"}


def _now() -> str:
    return _dt.datetime.now(_dt.timezone.utc).isoformat()


# Thin module-level wrappers preserved so the existing test suite +
# any direct imports continue to work. Both delegate to the shared
# :class:`_backoff.Backoff` instance above.
def _compute_next_attempt_after(
    failures: int,
    now_dt: _dt.datetime,
) -> str | None:
    return _BACKOFF.next_attempt_after(failures, now_dt)


def _backoff_active(cached: dict[str, Any] | None, now_dt: _dt.datetime) -> bool:
    return _BACKOFF.is_active(cached, now_dt)


def parse_run_job_ids(target_url: str | None) -> tuple[int | None, int | None]:
    """Extract ``(run_id, job_id)`` from a Forgejo Actions
    ``target_url`` if it matches one of the shapes we know.

    Returns ``(run_id, None)`` for the run-only shape so the caller
    can fall through to a per-run jobs listing if it cares to.
    Returns ``(None, None)`` for unsupported shapes (Codecov, etc.).
    """
    if not target_url:
        return None, None
    s = str(target_url)
    m = _TARGET_URL_RUN_JOB_RE.search(s)
    if m:
        try:
            return int(m.group(1)), int(m.group(2))
        except (TypeError, ValueError):
            return None, None
    m = _TARGET_URL_RUN_ONLY_RE.search(s)
    if m:
        try:
            return int(m.group(1)), None
        except (TypeError, ValueError):
            return None, None
    return None, None


def _fetch_run_jobs(cfg: Any, run_id: int) -> tuple[list[dict[str, Any]], str | None]:
    """List jobs in an actions run via
    ``/repos/{owner}/{repo}/actions/runs/{run_id}/jobs``.

    Returns ``(jobs, error)``. ``jobs`` is empty on any failure;
    ``error`` carries a short reason string for the caller to stamp
    on the per-job entry. Forgejo wraps the list in ``{"jobs": [...]}``
    or returns the array directly depending on version; we accept
    both."""
    path = f"/repos/{cfg.owner}/{cfg.repo}/actions/runs/{int(run_id)}/jobs"
    try:
        response = _claim_runtime.get(path, cfg)
    except Exception as exc:  # noqa: BLE001
        return [], f"run-jobs:{type(exc).__name__}"
    status = int(response.get("status") or 0)
    if status != 200:
        return [], f"run-jobs:status={status}"
    body = response.get("body")
    if isinstance(body, dict) and isinstance(body.get("jobs"), list):
        return [j for j in body["jobs"] if isinstance(j, dict)], None
    if isinstance(body, list):
        return [j for j in body if isinstance(j, dict)], None
    return [], "run-jobs:malformed-body"


def _resolve_job_id_for_context(
    cfg: Any,
    run_id: int,
    context: str,
) -> tuple[int | None, str | None]:
    """When ``target_url`` only carries the run_id, list jobs in the
    run and match by name/context. The Forgejo Actions API
    typically calls the field ``name``; the commit status's
    ``context`` is conventionally ``"CI / {job-name}"`` so we
    strip the ``"CI / "`` prefix before matching."""
    jobs, err = _fetch_run_jobs(cfg, run_id)
    if err:
        return None, err
    if not jobs:
        return None, "run-jobs:empty"
    # Try exact-name match first, then suffix match.
    name_to_id: dict[str, int] = {}
    for j in jobs:
        try:
            jid = int(j.get("id"))
        except (TypeError, ValueError):
            continue
        name = str(j.get("name") or "")
        if name:
            name_to_id[name] = jid
    if context in name_to_id:
        return name_to_id[context], None
    # Strip common ``"CI / "`` prefix from the status context.
    if " / " in context:
        suffix = context.split(" / ", 1)[1]
        if suffix in name_to_id:
            return name_to_id[suffix], None
    return None, "run-jobs:no-context-match"


def _fetch_job_log(
    cfg: Any,
    run_id: int,
    job_id: int,
    *,
    max_chars: int | None,
) -> tuple[str | None, int | None, bool, str | None]:
    """Fetch the raw log text for a single job. Returns
    ``(log_text, bytes_seen, truncated, error)``.

    ``max_chars=None`` returns the FULL untruncated log — the
    :func:`get_ci_logs` bundle path. An integer keeps only the last
    ``max_chars`` characters (the legacy failing-tail path).

    Forgejo's REST API does NOT expose action job logs (live-probed
    2026-05-17: every variant of ``/api/v1/.../actions/.../logs``
    returns 404 on this version). The UI route DOES serve raw text
    at ``/{owner}/{repo}/actions/runs/{run_index}/jobs/{job_index}/attempt/{attempt}/logs``
    but requires a session cookie (no token-auth — verified live).

    Args:
        run_id: the run **index_in_repo** (the small number from
            target_url), not the global PK.
        job_id: the per-run job INDEX (the trailing number in
            target_url's ``/jobs/{N}``).

    Both come from ``parse_run_job_ids(target_url)``.

    We truncate to the LAST ``max_chars`` characters — the failing
    assertion + stack trace are always at the tail."""
    path = (
        f"/{cfg.owner}/{cfg.repo}/actions/runs/{int(run_id)}/jobs/"
        f"{int(job_id)}/attempt/1/logs"
    )
    try:
        response = _ui_fetch_with_session(cfg, path)
    except Exception as exc:  # noqa: BLE001
        return None, None, False, f"log:{type(exc).__name__}"
    status = int(response.get("status") or 0)
    if status != 200:
        return None, None, False, f"log:status={status}"
    body = response.get("body")
    if isinstance(body, bytes):
        try:
            text = body.decode("utf-8", errors="replace")
        except Exception:  # noqa: BLE001
            return None, len(body), False, "log:decode-error"
    elif isinstance(body, str):
        text = body
    else:
        return None, None, False, "log:unexpected-body-shape"
    bytes_seen = len(text.encode("utf-8"))
    if max_chars is not None and len(text) > max_chars:
        # Keep the tail — failing assertions and stack traces live
        # at the end of CI logs. Add a marker so the worker can tell
        # this isn't the full log.
        return (
            "...[truncated head; showing last "
            f"{max_chars} chars]...\n" + text[-max_chars:],
            bytes_seen,
            True,
            None,
        )
    return text, bytes_seen, False, None


def _is_failing(state: str | None) -> bool:
    """Forgejo emits both ``state`` (statuses) and ``conclusion``
    (actions jobs) with overlapping vocabularies. The set below is
    everything we treat as "this check did not pass" — pending and
    in_progress are NOT failing (yet)."""
    if not state:
        return False
    return str(state).lower() in {
        "failure",
        "failed",
        "error",
        "cancelled",
        "canceled",
        "timed_out",
    }


def collect_failing_jobs(
    cfg: Any,
    ci_detail: list[dict[str, Any]],
    *,
    max_jobs: int,
    max_chars_per_job: int,
) -> tuple[list[dict[str, Any]], bool]:
    """Build the failing-jobs payload from a CI detail list (the
    list returned by :func:`_review_fetch.fetch_ci_check_detail`).
    Returns ``(failing_jobs, completed)``. ``completed`` is True iff
    every failing-status' log was fetched cleanly."""
    failing_statuses = [
        s
        for s in ci_detail
        if isinstance(s, dict) and _is_failing(s.get("state") or s.get("status"))
    ]
    failing_statuses = failing_statuses[:max_jobs]
    out: list[dict[str, Any]] = []
    all_clean = True
    for s in failing_statuses:
        context = s.get("context") or ""
        target_url = s.get("target_url") or ""
        run_id, job_id = parse_run_job_ids(target_url)
        entry: dict[str, Any] = {
            "context": context,
            "state": s.get("state") or s.get("status"),
            "description": s.get("description"),
            "run_id": run_id,
            "job_id": job_id,
            "log_url": target_url,
            "log_tail": None,
            "log_bytes_seen": None,
            "log_truncated": False,
            "fetch_error": None,
        }
        if run_id is None:
            # Unsupported URL shape (e.g. external Codecov link). The
            # worker can still follow the link manually; we just
            # can't pre-fetch its log content.
            entry["fetch_error"] = "unsupported-url-shape"
            all_clean = False
            out.append(entry)
            continue
        if job_id is None:
            resolved_job_id, resolve_err = _resolve_job_id_for_context(
                cfg,
                run_id,
                context,
            )
            if resolve_err is not None or resolved_job_id is None:
                entry["fetch_error"] = resolve_err or "run-jobs:no-job-id"
                all_clean = False
                out.append(entry)
                continue
            entry["job_id"] = job_id = resolved_job_id
        log_tail, bytes_seen, truncated, log_err = _fetch_job_log(
            cfg,
            int(run_id),
            int(job_id),
            max_chars=max_chars_per_job,
        )
        entry["log_tail"] = log_tail
        entry["log_bytes_seen"] = bytes_seen
        entry["log_truncated"] = truncated
        entry["fetch_error"] = log_err
        if log_err is not None:
            all_clean = False
        out.append(entry)
    return out, all_clean


def _tail_with_marker(
    log: str | None, max_chars: int | None
) -> tuple[str | None, bool]:
    """Project a full job log to its last ``max_chars`` characters with
    the legacy truncation marker. Returns ``(text, truncated)``;
    ``(None, False)`` when there is no log. Matches the truncation
    ``_fetch_job_log`` applied in the pre-bundle code path."""
    if not isinstance(log, str):
        return None, False
    if max_chars is not None and len(log) > max_chars:
        return (
            "...[truncated head; showing last "
            f"{max_chars} chars]...\n" + log[-max_chars:],
            True,
        )
    return log, False


def fetch_pr_failure_logs(
    cfg: Any,
    head_sha: str,
    *,
    ci_detail: list[dict[str, Any]] | None = None,
    max_jobs: int | None = None,
    max_chars_per_job: int | None = None,
) -> tuple[dict[str, Any], bool]:
    """Failing-CI log *tails* for ``head_sha`` — a VIEW over
    :func:`get_ci_logs`.

    B1 (2026-05-20): this was once a parallel fetch path with its own
    ``{sha}.json`` cache, its own backoff, and 4000-char tail
    truncation — a second CI-log cache that could drift from the
    bundle. It is now a thin projection of the unified
    :func:`get_ci_logs` bundle (``{sha}.full.json``): ONE CI-log
    cache, ONE fetch path. Retained for the ``ci_fetch_pr_failure_logs``
    MCP tool and the legacy review/implementer prefetch callers, which
    consume the failing-jobs-with-tails envelope.

    Returns ``(payload, completed)``; ``payload`` carries the legacy
    schema (``failing_jobs`` each with a ``log_tail`` truncated to
    ``max_chars_per_job``). ``completed`` is True iff the CI run is
    terminal AND every job log fetched cleanly (or there were no
    failing jobs) — a partial (in-flight) run is never ``completed``.
    """
    effective_max_jobs = max_jobs if max_jobs is not None else _max_jobs()
    effective_max_chars = (
        max_chars_per_job if max_chars_per_job is not None else _max_chars_per_job()
    )
    if not head_sha:
        # No SHA → no cache key → no fetch. Return an empty envelope
        # rather than raising so the caller doesn't have to defend.
        empty: dict[str, Any] = {
            "schema_version": SCHEMA_VERSION,
            "head_sha": "",
            "fetched_at": _now(),
            "failing_jobs": [],
            "completed": True,
            "consecutive_failures": 0,
            "next_attempt_after": None,
        }
        return empty, True

    bundle = get_ci_logs(
        cfg, head_sha, ci_detail=ci_detail, max_jobs=effective_max_jobs
    )
    failing_jobs: list[dict[str, Any]] = []
    for j in bundle.get("jobs") or []:
        if not isinstance(j, dict):
            continue
        if not _is_failing(j.get("state")):
            continue
        log_tail, truncated = _tail_with_marker(j.get("log"), effective_max_chars)
        failing_jobs.append(
            {
                "context": j.get("context"),
                "state": j.get("state"),
                "description": j.get("description"),
                "run_id": j.get("run_id"),
                "job_id": j.get("job_id"),
                "log_url": j.get("log_url"),
                "log_tail": log_tail,
                "log_bytes_seen": j.get("log_bytes"),
                "log_truncated": truncated,
                "fetch_error": j.get("fetch_error"),
            }
        )
    # Legacy ``completed`` = the bundle is terminal (CI not in-flight)
    # AND every job log fetched cleanly. A partial (in-flight) run is
    # never "completed" — the caller must not freeze on it.
    completed = bool(bundle.get("completed")) and not bundle.get("partial")
    payload = {
        "schema_version": SCHEMA_VERSION,
        "head_sha": head_sha,
        "fetched_at": bundle.get("fetched_at") or _now(),
        "failing_jobs": failing_jobs,
        "completed": completed,
        "consecutive_failures": int(bundle.get("consecutive_failures") or 0),
        "next_attempt_after": bundle.get("next_attempt_after"),
    }
    return payload, completed


def invalidate(head_sha: str) -> None:
    """Force-remove the cached bundle for ``head_sha``. Idempotent.
    Use to force the next :func:`get_ci_logs` to repopulate, or in
    tests. (A genuine CI re-run is auto-detected via the run_id
    staleness check, so explicit invalidation is rarely needed.)"""
    target = bundle_cache_path(head_sha)
    try:
        target.unlink(missing_ok=True)
    except OSError as exc:
        _logger.warning(
            "ci-logs bundle invalidate failed for %s at %s: %s",
            head_sha,
            target,
            exc,
        )


# ─── Unified full-log bundle (get_ci_logs) ───────────────────────────
#
# ``get_ci_logs`` is the single entry point every CI-log consumer
# should use. Unlike ``fetch_pr_failure_logs`` (failing jobs only,
# 4000-char tails) it returns EVERY job of the run with FULL,
# untruncated logs, in one cache. ``partial`` is True while the CI run
# is still in-flight; a terminal run with a clean fetch is frozen
# forever (per-SHA immutability). Truncation, when a consumer needs it
# for a prompt budget, is the consumer's job at render time — never in
# this cache.

_BUNDLE_SCHEMA_VERSION = 1

# Check states that mean the CI run has not reached a terminal verdict.
_PENDING_STATES = frozenset(
    {"pending", "running", "in_progress", "in-progress", "queued", "waiting"}
)


def bundle_cache_path(head_sha: str) -> Path:
    """Per-SHA full-log bundle cache file. A distinct filename from the
    legacy ``fetch_pr_failure_logs`` cache so the two never collide
    while consumers are being migrated onto the bundle."""
    safe = re.sub(r"[^a-fA-F0-9]", "", str(head_sha))[:64] or "INVALID"
    return cache_dir() / f"{safe}.full.json"


def _read_bundle(head_sha: str) -> dict[str, Any] | None:
    target = bundle_cache_path(head_sha)
    if not target.exists():
        return None
    try:
        payload = json.loads(target.read_text(encoding="utf-8"))
    except (OSError, ValueError) as exc:
        _logger.warning("ci-logs bundle read failed for %s: %s", head_sha, exc)
        return None
    if not isinstance(payload, dict):
        return None
    if payload.get("schema_version") != _BUNDLE_SCHEMA_VERSION:
        return None
    if not isinstance(payload.get("jobs"), list):
        return None
    return payload


def _write_bundle(head_sha: str, payload: dict[str, Any]) -> None:
    target = bundle_cache_path(head_sha)
    tmp = target.with_suffix(target.suffix + ".tmp")
    try:
        target.parent.mkdir(parents=True, exist_ok=True)
        tmp.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
        tmp.replace(target)
    except (OSError, TypeError, ValueError) as exc:
        _logger.warning("ci-logs bundle write failed for %s: %s", head_sha, exc)
        try:
            tmp.unlink(missing_ok=True)
        except OSError:
            pass


def _ci_run_partial(ci_detail: list[dict[str, Any]] | None) -> bool:
    """True iff any check is still pending/running — i.e. the CI run
    has not reached a terminal verdict for this head_sha."""
    for s in ci_detail or []:
        if not isinstance(s, dict):
            continue
        state = str(s.get("state") or s.get("status") or "").lower()
        if state in _PENDING_STATES:
            return True
    return False


def _run_id_from_detail(ci_detail: list[dict[str, Any]] | None) -> int | None:
    """The Forgejo Actions run index for this CI detail — the first
    parseable ``run_id`` across the statuses' ``target_url`` fields.

    Used as the cache-staleness discriminator: a head_sha can be
    re-run (Forgejo "re-run", a re-pushed identical tree), producing a
    new run with a new index. A cached terminal bundle whose ``run_id``
    no longer matches the current run is stale and must be re-fetched —
    otherwise the first run's verdict shadows the re-run forever.
    Returns ``None`` when no status carries a parseable run index."""
    for s in ci_detail or []:
        if not isinstance(s, dict):
            continue
        run_id, _job_id = parse_run_job_ids(s.get("target_url") or "")
        if run_id is not None:
            return run_id
    return None


def _partial_min_refetch_s() -> int:
    """Minimum seconds between live re-fetches of a partial bundle
    (B2). Env-overridable; clamped to >= 0."""
    raw = os.environ.get(_PARTIAL_MIN_REFETCH_ENV, "").strip()
    if not raw:
        return _DEFAULT_PARTIAL_MIN_REFETCH_S
    try:
        return max(0, int(float(raw)))
    except (TypeError, ValueError):
        return _DEFAULT_PARTIAL_MIN_REFETCH_S


def _partial_refetch_due(cached: dict[str, Any] | None, now_dt: _dt.datetime) -> bool:
    """B2: True when a cached PARTIAL bundle is due for a live
    re-fetch — i.e. its ``next_attempt_after`` is absent or already
    in the past. False means serve the cached partial as-is."""
    if not isinstance(cached, dict):
        return True
    raw = cached.get("next_attempt_after")
    if not raw:
        return True
    try:
        nxt = _dt.datetime.fromisoformat(str(raw))
    except (TypeError, ValueError):
        return True
    if nxt.tzinfo is None:
        nxt = nxt.replace(tzinfo=_dt.timezone.utc)
    return now_dt >= nxt


def collect_all_jobs(
    cfg: Any,
    ci_detail: list[dict[str, Any]],
    *,
    max_jobs: int,
) -> tuple[list[dict[str, Any]], bool]:
    """Fetch the FULL (untruncated) log for EVERY job in ``ci_detail`` —
    passing and failing alike. Returns ``(jobs, all_clean)``;
    ``all_clean`` is True iff every job's log was fetched cleanly.

    Failing jobs are ordered first so that, if ``max_jobs`` bites on a
    very wide CI matrix, the cap only ever drops *passing* jobs — a
    failing gate (the one a worker actually needs) is never lost."""
    ordered = sorted(
        (s for s in ci_detail if isinstance(s, dict)),
        key=lambda s: 0 if _is_failing(s.get("state") or s.get("status")) else 1,
    )
    statuses = ordered[:max_jobs]
    out: list[dict[str, Any]] = []
    all_clean = True
    for s in statuses:
        context = s.get("context") or ""
        target_url = s.get("target_url") or ""
        run_id, job_id = parse_run_job_ids(target_url)
        entry: dict[str, Any] = {
            "context": context,
            "state": s.get("state") or s.get("status"),
            "description": s.get("description"),
            "run_id": run_id,
            "job_id": job_id,
            "log_url": target_url,
            "log": None,
            "log_bytes": None,
            "fetch_error": None,
        }
        if run_id is None:
            entry["fetch_error"] = "unsupported-url-shape"
            all_clean = False
            out.append(entry)
            continue
        if job_id is None:
            resolved, err = _resolve_job_id_for_context(cfg, run_id, context)
            if err is not None or resolved is None:
                entry["fetch_error"] = err or "run-jobs:no-job-id"
                all_clean = False
                out.append(entry)
                continue
            entry["job_id"] = job_id = resolved
        log_text, bytes_seen, _trunc, log_err = _fetch_job_log(
            cfg,
            int(run_id),
            int(job_id),
            max_chars=None,
        )
        entry["log"] = log_text
        entry["log_bytes"] = bytes_seen
        entry["fetch_error"] = log_err
        if log_err is not None:
            all_clean = False
        out.append(entry)
    return out, all_clean


def _bundle(
    head_sha: str,
    *,
    jobs: list[dict[str, Any]],
    run_id: int | None,
    partial: bool,
    completed: bool,
    consecutive_failures: int,
    next_attempt_after: str | None,
    source: str = "forgejo",
) -> dict[str, Any]:
    return {
        "schema_version": _BUNDLE_SCHEMA_VERSION,
        "head_sha": head_sha,
        "fetched_at": _now(),
        "source": source,
        "run_id": run_id,
        "jobs": jobs,
        "partial": partial,
        "completed": completed,
        "consecutive_failures": consecutive_failures,
        "next_attempt_after": next_attempt_after,
    }


def get_ci_logs(
    cfg: Any,
    head_sha: str,
    *,
    ci_detail: list[dict[str, Any]] | None = None,
    max_jobs: int | None = None,
) -> dict[str, Any]:
    """Unified CI-log entry point — EVERY job of the run, FULL logs, one
    cache. The single fetcher all CI-log consumers should call.

    Returns a bundle dict::

        {schema_version, head_sha, fetched_at, source, run_id,
         partial, completed, consecutive_failures, next_attempt_after,
         jobs: [{context, state, description, run_id, job_id, log_url,
                 log, log_bytes, fetch_error}]}

    - ``partial`` — True while the CI run is still in-flight (some check
      pending/running). A partial bundle is always re-fetched on the
      next call (CI is expected to change) — no backoff.
    - ``completed`` — every reachable job log fetched cleanly.
    - A terminal run (``not partial``) with a clean fetch
      (``completed``) is frozen — served straight from cache — for as
      long as it is the CURRENT run. The cache is keyed by ``head_sha``
      but a SHA can be re-run; a cached bundle whose ``run_id`` no
      longer matches the live run is treated as stale and re-fetched
      (last-run-wins). Resolving the live run costs one cheap
      ci-detail call; the expensive per-job log fetches are still
      skipped on a cache hit.
    - A terminal run with fetch errors is retried under the shared
      exponential backoff.
    """
    effective_max_jobs = max_jobs if max_jobs is not None else _max_jobs()
    if not head_sha:
        return _bundle(
            "",
            jobs=[],
            run_id=None,
            partial=False,
            completed=True,
            consecutive_failures=0,
            next_attempt_after=None,
        )

    now_dt = _dt.datetime.now(_dt.timezone.utc)
    disabled = is_disabled()
    cached = None if disabled else _read_bundle(head_sha)

    # A RUN_CI_LOCAL bundle (source="local") is authoritative: it has
    # no Forgejo run to re-check, so serve it frozen with no fetch.
    if (
        cached is not None
        and cached.get("source") == "local"
        and not cached.get("partial")
        and cached.get("completed")
    ):
        return cached

    # B2: a cached PARTIAL bundle (CI in-flight) is served as-is until
    # its minimum re-fetch interval elapses — bounds polling load on
    # the Forgejo UI endpoints without ever freezing an in-flight run.
    if (
        cached is not None
        and cached.get("partial")
        and not _partial_refetch_due(cached, now_dt)
    ):
        return cached

    # Resolve ci_detail FIRST, so the cache decision is run-aware. The
    # bundle is keyed by head_sha, but a SHA can be re-run (Forgejo
    # "re-run", a re-pushed identical tree) — a NEW run with a new
    # index. A terminal bundle frozen for the OLD run must not shadow
    # the re-run: it is served from cache only when its ``run_id``
    # still matches the current run. (B3: multi-run cache correctness.)
    detail = ci_detail
    if detail is None:
        try:
            detail, _ok = _review_fetch.fetch_ci_check_detail(cfg, head_sha)
        except Exception as exc:  # noqa: BLE001
            # ci-detail fetch failed — we cannot identify the current
            # run. A previously-cached bundle (even an older run's)
            # beats nothing; otherwise record the failure for backoff.
            if cached is not None:
                return cached
            return _record_bundle_failure(
                head_sha,
                None,
                now_dt,
                error=f"ci-detail:{type(exc).__name__}",
                disabled=disabled,
            )

    current_run_id = _run_id_from_detail(detail)

    if cached is not None and not cached.get("partial"):
        cache_run_id = cached.get("run_id")
        # Serve the frozen / backoff bundle only when it is for the
        # CURRENT run. An unknown run_id on either side → assume a
        # match (conservative: preserves the legacy per-SHA freeze when
        # the run simply cannot be identified).
        run_matches = (
            current_run_id is None
            or cache_run_id is None
            or cache_run_id == current_run_id
        )
        if run_matches:
            if cached.get("completed"):
                return cached  # terminal + clean + current run → frozen
            if _backoff_active(cached, now_dt):
                return cached  # terminal, fetch errors, inside backoff
        else:
            _logger.info(
                "ci-logs bundle for %s is stale (cached run_id=%s, "
                "current run_id=%s) — CI was re-run; re-fetching",
                head_sha,
                cache_run_id,
                current_run_id,
            )
            # Stale run → clean slate: do not chain the old run's
            # consecutive_failures into the re-fetch's backoff.
            cached = None

    partial = _ci_run_partial(detail or [])
    jobs, ok = collect_all_jobs(cfg, detail or [], max_jobs=effective_max_jobs)
    run_id = next((j["run_id"] for j in jobs if j.get("run_id") is not None), None)

    if partial:
        # In-flight: cache it (consumers still see what exists). Never
        # frozen — but B2 stamps a minimum re-fetch interval so a tight
        # caller loop doesn't re-poll Forgejo every tick.
        next_refetch = (
            now_dt + _dt.timedelta(seconds=_partial_min_refetch_s())
        ).isoformat()
        payload = _bundle(
            head_sha,
            jobs=jobs,
            run_id=run_id,
            partial=True,
            completed=ok,
            consecutive_failures=0,
            next_attempt_after=next_refetch,
        )
    elif ok:
        payload = _bundle(
            head_sha,
            jobs=jobs,
            run_id=run_id,
            partial=False,
            completed=True,
            consecutive_failures=0,
            next_attempt_after=None,
        )
    else:
        # Terminal run, but >=1 job log unreachable -> backoff retry.
        nxt = int((cached or {}).get("consecutive_failures") or 0) + 1
        payload = _bundle(
            head_sha,
            jobs=jobs,
            run_id=run_id,
            partial=False,
            completed=False,
            consecutive_failures=nxt,
            next_attempt_after=_compute_next_attempt_after(nxt, now_dt),
        )
        _logger.warning(
            "ci-logs bundle partial fetch for %s (%s/%s jobs missing "
            "logs); consecutive_failures=%s",
            head_sha,
            sum(1 for j in jobs if j.get("fetch_error")),
            len(jobs),
            nxt,
        )
    if not disabled:
        _write_bundle(head_sha, payload)
    return payload


def _record_bundle_failure(
    head_sha: str,
    prior_cache: dict[str, Any] | None,
    now_dt: _dt.datetime,
    *,
    error: str,
    disabled: bool,
) -> dict[str, Any]:
    """ "The live attempt couldn't even start" path for the bundle —
    e.g. the ci-detail fetch raised. Persists a backoff record + any
    previously-cached jobs so the loop converges."""
    nxt = int((prior_cache or {}).get("consecutive_failures") or 0) + 1
    payload = _bundle(
        head_sha,
        jobs=list((prior_cache or {}).get("jobs") or []),
        run_id=(prior_cache or {}).get("run_id"),
        partial=False,
        completed=False,
        consecutive_failures=nxt,
        next_attempt_after=_compute_next_attempt_after(nxt, now_dt),
    )
    payload["last_error"] = error
    if not disabled:
        _write_bundle(head_sha, payload)
    _logger.warning(
        "ci-logs bundle live fetch failed for %s (%s); consecutive_failures=%s",
        head_sha,
        error,
        nxt,
    )
    return payload


def put_local_bundle(head_sha: str, jobs: list[dict[str, Any]]) -> None:
    """Seed the :func:`get_ci_logs` bundle cache from a LOCAL CI run
    (RUN_CI_LOCAL), so every consumer reads local-CI logs through the
    same entry point as Forgejo-CI logs — the cache is source-abstracted.

    ``jobs`` is ``[{context, state, log, ...}]``. The bundle is marked
    ``source="local"``, terminal and complete: a finished local run is
    frozen, exactly like a terminal Forgejo run.
    """
    if not head_sha:
        return
    norm: list[dict[str, Any]] = []
    for j in jobs:
        if not isinstance(j, dict):
            continue
        log = j.get("log")
        norm.append(
            {
                "context": j.get("context"),
                "state": j.get("state"),
                "description": j.get("description"),
                "run_id": None,
                "job_id": None,
                "log_url": None,
                "log": log if isinstance(log, str) else None,
                "log_bytes": (
                    len(log.encode("utf-8")) if isinstance(log, str) else None
                ),
                "fetch_error": None,
            }
        )
    payload = _bundle(
        head_sha,
        jobs=norm,
        run_id=None,
        partial=False,
        completed=True,
        consecutive_failures=0,
        next_attempt_after=None,
        source="local",
    )
    _write_bundle(head_sha, payload)


def logs_by_context(bundle: dict[str, Any]) -> dict[str, str]:
    """Flatten a :func:`get_ci_logs` bundle to ``{gate_context:
    full_log}`` — the shape a CISummary ``log_fetcher`` consumes. Jobs
    whose log is missing (fetch error) are omitted."""
    out: dict[str, str] = {}
    for j in bundle.get("jobs") or []:
        if not isinstance(j, dict):
            continue
        ctx, log = j.get("context"), j.get("log")
        if ctx and isinstance(log, str):
            out[ctx] = log
    return out


__all__ = (
    "SCHEMA_VERSION",
    "bundle_cache_path",
    "cache_dir",
    "collect_all_jobs",
    "collect_failing_jobs",
    "fetch_pr_failure_logs",
    "get_ci_logs",
    "invalidate",
    "is_disabled",
    "logs_by_context",
    "parse_run_job_ids",
    "put_local_bundle",
)