drew
6879c68f4e
Live-observed bug from PR #40 run-7 (2026-05-17): a SHA's CI is NOT terminally frozen at the first cache write — Forgejo runs checks asynchronously, and a check that's PENDING at fetch time can later transition to FAILURE. The original cache logic marked ``completed=True`` as soon as the currently-failing jobs' logs were fetched cleanly, then refused to ever re-fetch (per-SHA immutability claim). Result observed: PR #40 cache sealed at 20:38 with 1 failing job (push-validation, the only one transitioned by then). Lint and unit_tests transitioned pending→failure later in the same run. The cache was treated as frozen, never re-fetched, and BOTH downstream consumers (reviewer + implementer) saw only push-validation's log. The reviewer's RC review listed three failing checks by name but could only describe one in detail because the prefetch envelope literally didn't have the other two logs. The implementer, even with its R3.4 visibility into the reviewer's review, had the same gap. Fix: when the cached payload says ``completed=True`` AND the caller-provided ``ci_detail`` shows additional failing contexts not in the cached ``failing_jobs``, treat the cache as stale and live-fetch. Same-set comparison = cache still valid. The per-SHA immutability claim still holds in the steady state: once every check has reached a terminal state, the set of failing contexts stops changing, and the comparison becomes a no-op. The fix only triggers re-fetch during the window where checks are still transitioning. New helper: ``_cache_covers_all_current_failures(cached, ci_detail)`` returns True iff every distinct failing-context in ci_detail is already represented in cached.failing_jobs. When ci_detail is None (caller couldn't provide), conservatively assumes the cache is still valid (preserves prior behaviour for that path). Tests: - ``test_completed_cache_reinvalidated_when_new_failures_appear`` pins the regression: seeded cache with 1 failure, current ci_detail has 2, re-fetch must occur and capture both contexts. - ``test_completed_cache_still_served_when_ci_detail_unchanged`` counter-test: same failing set = cache preserved, no live call (the live-fetch path is stubbed to raise so any live call surfaces as a hard test failure). Full auto_agents suite: 2307 passing. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
902 lines
33 KiB
Python
902 lines
33 KiB
Python
"""Per-(head_sha) on-disk cache of failing-CI job log tails.
|
||
|
||
Why this exists
|
||
---------------
|
||
|
||
Multiple recent implementer cycles (PR #28, 2026-05-16 runs 18-20)
|
||
burned their 30-min worker budget largely on:
|
||
|
||
- ~14 file reads + ~14 bash calls just to *infer* what CI was
|
||
failing on, from the diff alone.
|
||
- Two ``bash curl ... /actions/runs/{n}/jobs`` attempts that were
|
||
blocked by the task-implementor's bash allowlist.
|
||
- One ``webfetch`` attempt at the same Forgejo Actions URL that
|
||
errored.
|
||
- One ``ci_run_local_gate`` call (a ``nox -s coverage_report`` run)
|
||
that took ~10 minutes by itself.
|
||
|
||
The root pathology is that the dispatcher pre-fetches the per-check
|
||
*statuses* (context + state + target_url) but NOT the actual log
|
||
text. The worker is told ``CI / lint: failure`` but has to chase
|
||
the cause through file reads and gate runs — most of which never
|
||
finish inside the worker window.
|
||
|
||
This module fetches and caches the failing jobs' log tails ONCE per
|
||
head_sha and exposes them to:
|
||
|
||
1. The reviewer + implementer dispatchers (pre-fetch into prompt).
|
||
2. The ``ci_fetch_pr_failure_logs`` MCP tool (agent ad-hoc query).
|
||
|
||
Both consumers share one cache directory so the second consumer is
|
||
free even if it runs in a different process.
|
||
|
||
Cache layout
|
||
------------
|
||
|
||
``/tmp/cleveragents-ci-logs-cache/{head_sha}.json`` — per-SHA JSON.
|
||
SHAs are immutable, and a workflow run against a SHA freezes its
|
||
job log content once the run reaches a terminal state, so once a
|
||
cache entry is ``completed=True`` it is good *forever* — no TTL
|
||
required for staleness. A 7-day cleanup ceiling keeps the cache
|
||
directory bounded under sustained operation.
|
||
|
||
Schema::
|
||
|
||
{
|
||
"schema_version": 1,
|
||
"head_sha": "d423e5f9...",
|
||
"fetched_at": "2026-05-16T18:03:27+00:00",
|
||
"failing_jobs": [
|
||
{
|
||
"context": "CI / lint",
|
||
"state": "failure",
|
||
"run_id": 83,
|
||
"job_id": 7,
|
||
"log_url": "https://git.cleverthis.com/.../actions/runs/83/jobs/7",
|
||
"log_tail": "...last 4000 chars of job log...",
|
||
"log_bytes_seen": 12345,
|
||
"log_truncated": true,
|
||
"fetch_error": null
|
||
},
|
||
...
|
||
],
|
||
"completed": true,
|
||
"consecutive_failures": 0,
|
||
"next_attempt_after": null
|
||
}
|
||
|
||
``completed=True`` iff EVERY failing-job log fetch returned 200 with
|
||
text content. Any per-job error sets the job's ``fetch_error`` field
|
||
AND flips ``completed`` to False so the caller knows the picture is
|
||
partial.
|
||
|
||
Errors and backoff
|
||
------------------
|
||
|
||
A failed live fetch increments ``consecutive_failures`` and stamps
|
||
``next_attempt_after = now + min(BASE * 2**(failures-1), MAX)``.
|
||
Inside the deferred window, ``fetch_pr_failure_logs`` short-circuits
|
||
and returns the cached payload with ``completed=False`` without
|
||
hitting the live endpoint again. A subsequent successful fetch
|
||
resets the counter so a transient outage doesn't permanently
|
||
throttle a PR.
|
||
|
||
Mirrors the backoff design in :mod:`_pr_comments_cache`.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import datetime as _dt
|
||
import http.cookiejar
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import sys
|
||
import threading
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
_TOOLS_DIR = str(Path(__file__).resolve().parent)
|
||
if _TOOLS_DIR not in sys.path:
|
||
sys.path.insert(0, _TOOLS_DIR)
|
||
from _loader import ( # noqa: E402 type: ignore[import-not-found]
|
||
load_sibling as _load_sibling,
|
||
)
|
||
|
||
_review_fetch = _load_sibling("_review_fetch", "_review_fetch.py")
|
||
_claim_runtime = _load_sibling("_claim_runtime", "_claim_runtime.py")
|
||
_backoff = _load_sibling("_backoff", "_backoff.py")
|
||
|
||
_logger = logging.getLogger("ci_logs")
|
||
|
||
SCHEMA_VERSION = 1
|
||
|
||
# ─── Session-cookie helper for the UI logs endpoint ──────────────────
|
||
#
|
||
# Forgejo's REST API does NOT expose Actions job logs on this server
|
||
# version (live-probed 2026-05-17: every ``/api/v1/.../actions/.../logs``
|
||
# variant returns 404). The UI route DOES serve raw text at
|
||
# ``/{owner}/{repo}/actions/runs/{run_index}/jobs/{job_index}/attempt/{N}/logs``
|
||
# but requires a session cookie (token-auth ignored — verified live).
|
||
#
|
||
# This helper logs in once via ``FORGEJO_PASSWORD`` (env-set by the
|
||
# launcher) and caches the cookie-jar at module scope. Subsequent
|
||
# fetches reuse the cookie until the session expires; on 401/302-to-
|
||
# login we re-login transparently.
|
||
#
|
||
# Thread-safe via ``_login_lock``. Disabled when ``FORGEJO_PASSWORD``
|
||
# is unset — the fetcher returns ``no-session-password`` errors so the
|
||
# graceful-degradation path in ``collect_failing_jobs`` continues to
|
||
# work.
|
||
|
||
_session_cookies: http.cookiejar.CookieJar | None = None
|
||
_login_lock = threading.Lock()
|
||
|
||
|
||
def _do_session_login(cfg: Any) -> http.cookiejar.CookieJar | None:
|
||
"""Acquire a session cookie via the UI login form. Returns a
|
||
populated cookie-jar on success, ``None`` on any failure
|
||
(missing password / CSRF parse error / non-2xx login)."""
|
||
password = os.environ.get("FORGEJO_PASSWORD", "").strip()
|
||
if not password:
|
||
return None
|
||
username = os.environ.get("FORGEJO_USERNAME", "").strip() or "HAL9000"
|
||
base_url = str(cfg.forgejo_url).rstrip("/")
|
||
jar = http.cookiejar.CookieJar()
|
||
opener = urllib.request.build_opener(
|
||
urllib.request.HTTPCookieProcessor(jar),
|
||
urllib.request.HTTPRedirectHandler(),
|
||
)
|
||
# Fetch the login form to seed CSRF + initial cookie. Some
|
||
# Forgejo configs require the CSRF token; others ignore it.
|
||
csrf = ""
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{base_url}/user/login",
|
||
headers={"User-Agent": "cleveragents-ci-logs/1.0"},
|
||
)
|
||
with opener.open(req, timeout=15) as resp:
|
||
html = resp.read().decode("utf-8", errors="replace")
|
||
m = re.search(r'name="_csrf"\s+value="([^"]+)"', html)
|
||
if m:
|
||
csrf = m.group(1)
|
||
except (urllib.error.URLError, OSError, ValueError) as exc:
|
||
_logger.warning("ci-logs session: GET /user/login failed: %s", exc)
|
||
return None
|
||
# POST credentials. Include CSRF if we got one; many builds
|
||
# accept the POST without it.
|
||
form_fields = {"user_name": username, "password": password}
|
||
if csrf:
|
||
form_fields["_csrf"] = csrf
|
||
body = urllib.parse.urlencode(form_fields).encode("utf-8")
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{base_url}/user/login",
|
||
data=body,
|
||
headers={
|
||
"User-Agent": "cleveragents-ci-logs/1.0",
|
||
"Content-Type": "application/x-www-form-urlencoded",
|
||
},
|
||
)
|
||
with opener.open(req, timeout=15) as resp:
|
||
status = resp.status
|
||
except urllib.error.HTTPError as exc:
|
||
status = exc.code
|
||
except (urllib.error.URLError, OSError) as exc:
|
||
_logger.warning("ci-logs session: POST /user/login failed: %s", exc)
|
||
return None
|
||
# Success path: status 200 with a session cookie set. A redirect
|
||
# to /user/login again means auth failed.
|
||
has_session = any(
|
||
c.name in {"i_like_gitea", "session", "gitea_incredible"}
|
||
for c in jar
|
||
)
|
||
if status >= 400 or not has_session:
|
||
_logger.warning(
|
||
"ci-logs session: login returned status=%s session_cookie=%s",
|
||
status, has_session,
|
||
)
|
||
return None
|
||
return jar
|
||
|
||
|
||
def _get_session_cookies(cfg: Any) -> http.cookiejar.CookieJar | None:
|
||
"""Return the cached cookie-jar, logging in once on first call.
|
||
Thread-safe."""
|
||
global _session_cookies
|
||
if _session_cookies is not None:
|
||
return _session_cookies
|
||
with _login_lock:
|
||
if _session_cookies is not None:
|
||
return _session_cookies
|
||
_session_cookies = _do_session_login(cfg)
|
||
return _session_cookies
|
||
|
||
|
||
def _invalidate_session() -> None:
|
||
"""Drop the cached cookie-jar — forces a fresh login on next
|
||
call. Used when a request returns 401/302-to-login."""
|
||
global _session_cookies
|
||
with _login_lock:
|
||
_session_cookies = None
|
||
|
||
|
||
def _ui_fetch_with_session(cfg: Any, path: str) -> dict[str, Any]:
|
||
"""Fetch ``base_url + path`` using the cached session cookie.
|
||
Returns a dict shaped like ``_claim_runtime.get``'s return:
|
||
``{"status": int, "body": bytes|str}``.
|
||
|
||
Auto-relogin once on 401 or login-page redirect. After the second
|
||
failure, returns the error status so the caller's backoff logic
|
||
fires normally."""
|
||
base_url = str(cfg.forgejo_url).rstrip("/")
|
||
for attempt in (1, 2):
|
||
jar = _get_session_cookies(cfg)
|
||
if jar is None:
|
||
return {"status": 0, "body": b"", "error": "no-session-password"}
|
||
opener = urllib.request.build_opener(
|
||
urllib.request.HTTPCookieProcessor(jar),
|
||
urllib.request.HTTPRedirectHandler(),
|
||
)
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{base_url}{path}",
|
||
headers={"User-Agent": "cleveragents-ci-logs/1.0"},
|
||
)
|
||
with opener.open(req, timeout=30) as resp:
|
||
status = resp.status
|
||
body = resp.read()
|
||
final_url = resp.geturl()
|
||
except urllib.error.HTTPError as exc:
|
||
status = exc.code
|
||
body = b""
|
||
final_url = path
|
||
except (urllib.error.URLError, OSError) as exc:
|
||
return {"status": 0, "body": b"", "error": str(exc)}
|
||
# Detect session expiry: a 302 redirect chain ended at the
|
||
# login page, OR a 401. Invalidate + retry once.
|
||
if attempt == 1 and (
|
||
status == 401
|
||
or (final_url.endswith("/user/login")
|
||
and not path.endswith("/user/login"))
|
||
):
|
||
_invalidate_session()
|
||
continue
|
||
return {"status": status, "body": body}
|
||
return {"status": 0, "body": b"", "error": "session-retry-exhausted"}
|
||
|
||
# Defaults — overridable via env vars.
|
||
_DEFAULT_CACHE_DIR = Path("/tmp/cleveragents-ci-logs-cache")
|
||
_CACHE_DIR_ENV = "CI_LOGS_CACHE_DIR"
|
||
|
||
# Kill switch for tests / operators bypassing the cache.
|
||
_DISABLE_ENV = "CI_LOGS_CACHE_DISABLE"
|
||
|
||
# Per-job log-tail size cap. 4000 chars (~1000 tokens) is enough
|
||
# to see the failing assertion + last few lines of stack trace for
|
||
# every CI gate we've observed (ruff, mypy, pytest, behave, nox
|
||
# coverage). Operators can override for unusually verbose gates.
|
||
_DEFAULT_MAX_CHARS_PER_JOB = 4000
|
||
_MAX_CHARS_ENV = "CI_LOGS_MAX_CHARS_PER_JOB"
|
||
|
||
# Cap on number of failing jobs per PR. Most PRs have ≤ 10. A pre-fetch
|
||
# of 10 × 4000 chars = ~40 KB ≈ 10K tokens; bounded but informative.
|
||
_DEFAULT_MAX_JOBS = 10
|
||
_MAX_JOBS_ENV = "CI_LOGS_MAX_JOBS"
|
||
|
||
# Exponential backoff for repeated live-fetch failures. Shared shape
|
||
# via :class:`_backoff.Backoff` so the three cache modules pull from
|
||
# one source of truth for the curve + env vocabulary.
|
||
_BACKOFF = _backoff.Backoff(
|
||
base_env="CI_LOGS_BACKOFF_BASE_S",
|
||
max_env="CI_LOGS_BACKOFF_MAX_S",
|
||
base_default=60,
|
||
max_default=1800,
|
||
)
|
||
|
||
# Cache file cleanup ceiling. Older than this -> safe to delete on
|
||
# a janitor pass (not implemented here; this is just the contract).
|
||
_DEFAULT_MAX_STALENESS_S = 7 * 24 * 3600
|
||
_STALENESS_ENV = "CI_LOGS_MAX_STALENESS_S"
|
||
|
||
|
||
# target_url shapes we know how to parse:
|
||
# 1. https://<host>/<owner>/<repo>/actions/runs/<run_id>/jobs/<job_id>
|
||
# Forgejo Actions UI link. Has both ids inline.
|
||
# 2. https://<host>/<owner>/<repo>/actions/runs/<run_id>
|
||
# Forgejo Actions UI link without per-job index. Need to list
|
||
# jobs in the run to map a check context to a specific job_id.
|
||
# 3. Anything else (Codecov, external CI, custom Forgejo plugin):
|
||
# we can't fetch its logs through the Forgejo Actions API.
|
||
# Emit the job entry with ``fetch_error="unsupported-url-shape"``
|
||
# so the agent can see it and follow the link manually if needed.
|
||
_TARGET_URL_RUN_JOB_RE = re.compile(
|
||
r"/actions/runs/(\d+)/jobs/(\d+)\b"
|
||
)
|
||
_TARGET_URL_RUN_ONLY_RE = re.compile(
|
||
r"/actions/runs/(\d+)\b"
|
||
)
|
||
|
||
|
||
def cache_dir() -> Path:
|
||
return Path(os.environ.get(_CACHE_DIR_ENV) or str(_DEFAULT_CACHE_DIR))
|
||
|
||
|
||
def cache_path(head_sha: str) -> Path:
|
||
"""Per-SHA cache file path. We sanitize the SHA defensively even
|
||
though Forgejo only ever emits hex — a malformed SHA from a
|
||
caller bug shouldn't be able to write outside the cache dir."""
|
||
safe = re.sub(r"[^a-fA-F0-9]", "", str(head_sha))[:64]
|
||
if not safe:
|
||
safe = "INVALID"
|
||
return cache_dir() / f"{safe}.json"
|
||
|
||
|
||
def _max_chars_per_job() -> int:
|
||
raw = os.environ.get(_MAX_CHARS_ENV)
|
||
if raw:
|
||
try:
|
||
return max(256, int(raw))
|
||
except ValueError:
|
||
pass
|
||
return _DEFAULT_MAX_CHARS_PER_JOB
|
||
|
||
|
||
def _max_jobs() -> int:
|
||
raw = os.environ.get(_MAX_JOBS_ENV)
|
||
if raw:
|
||
try:
|
||
return max(1, int(raw))
|
||
except ValueError:
|
||
pass
|
||
return _DEFAULT_MAX_JOBS
|
||
|
||
|
||
def is_disabled() -> bool:
|
||
raw = os.environ.get(_DISABLE_ENV, "").strip().lower()
|
||
return raw in {"1", "true", "yes", "on"}
|
||
|
||
|
||
def _now() -> str:
|
||
return _dt.datetime.now(_dt.timezone.utc).isoformat()
|
||
|
||
|
||
# Thin module-level wrappers preserved so the existing test suite +
|
||
# any direct imports continue to work. Both delegate to the shared
|
||
# :class:`_backoff.Backoff` instance above.
|
||
def _compute_next_attempt_after(
|
||
failures: int, now_dt: _dt.datetime,
|
||
) -> str | None:
|
||
return _BACKOFF.next_attempt_after(failures, now_dt)
|
||
|
||
|
||
def _backoff_active(cached: dict[str, Any] | None, now_dt: _dt.datetime) -> bool:
|
||
return _BACKOFF.is_active(cached, now_dt)
|
||
|
||
|
||
def _read_cache(head_sha: str) -> dict[str, Any] | None:
|
||
"""Load cache file. Returns None on missing / malformed /
|
||
schema-mismatch / corrupted. Schema-mismatched files are NOT
|
||
auto-deleted (a janitor task or future migration handles it)."""
|
||
target = cache_path(head_sha)
|
||
if not target.exists():
|
||
return None
|
||
try:
|
||
text = target.read_text(encoding="utf-8")
|
||
payload = json.loads(text)
|
||
except (OSError, ValueError) as exc:
|
||
_logger.warning(
|
||
"ci-logs cache read failed for %s at %s: %s",
|
||
head_sha, target, exc,
|
||
)
|
||
return None
|
||
if not isinstance(payload, dict):
|
||
return None
|
||
if payload.get("schema_version") != SCHEMA_VERSION:
|
||
return None
|
||
if not isinstance(payload.get("failing_jobs"), list):
|
||
return None
|
||
return payload
|
||
|
||
|
||
def _write_cache(head_sha: str, payload: dict[str, Any]) -> None:
|
||
"""Atomic write."""
|
||
target = cache_path(head_sha)
|
||
tmp = target.with_suffix(target.suffix + ".tmp")
|
||
try:
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp.write_text(
|
||
json.dumps(payload, indent=2, default=str),
|
||
encoding="utf-8",
|
||
)
|
||
tmp.replace(target)
|
||
except (OSError, TypeError, ValueError) as exc:
|
||
_logger.warning(
|
||
"ci-logs cache write failed for %s at %s: %s",
|
||
head_sha, target, exc,
|
||
)
|
||
try:
|
||
tmp.unlink(missing_ok=True)
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def parse_run_job_ids(target_url: str | None) -> tuple[int | None, int | None]:
|
||
"""Extract ``(run_id, job_id)`` from a Forgejo Actions
|
||
``target_url`` if it matches one of the shapes we know.
|
||
|
||
Returns ``(run_id, None)`` for the run-only shape so the caller
|
||
can fall through to a per-run jobs listing if it cares to.
|
||
Returns ``(None, None)`` for unsupported shapes (Codecov, etc.).
|
||
"""
|
||
if not target_url:
|
||
return None, None
|
||
s = str(target_url)
|
||
m = _TARGET_URL_RUN_JOB_RE.search(s)
|
||
if m:
|
||
try:
|
||
return int(m.group(1)), int(m.group(2))
|
||
except (TypeError, ValueError):
|
||
return None, None
|
||
m = _TARGET_URL_RUN_ONLY_RE.search(s)
|
||
if m:
|
||
try:
|
||
return int(m.group(1)), None
|
||
except (TypeError, ValueError):
|
||
return None, None
|
||
return None, None
|
||
|
||
|
||
def _fetch_run_jobs(cfg: Any, run_id: int) -> tuple[list[dict[str, Any]], str | None]:
|
||
"""List jobs in an actions run via
|
||
``/repos/{owner}/{repo}/actions/runs/{run_id}/jobs``.
|
||
|
||
Returns ``(jobs, error)``. ``jobs`` is empty on any failure;
|
||
``error`` carries a short reason string for the caller to stamp
|
||
on the per-job entry. Forgejo wraps the list in ``{"jobs": [...]}``
|
||
or returns the array directly depending on version; we accept
|
||
both."""
|
||
path = f"/repos/{cfg.owner}/{cfg.repo}/actions/runs/{int(run_id)}/jobs"
|
||
try:
|
||
response = _claim_runtime.get(path, cfg)
|
||
except Exception as exc: # noqa: BLE001
|
||
return [], f"run-jobs:{type(exc).__name__}"
|
||
status = int(response.get("status") or 0)
|
||
if status != 200:
|
||
return [], f"run-jobs:status={status}"
|
||
body = response.get("body")
|
||
if isinstance(body, dict) and isinstance(body.get("jobs"), list):
|
||
return [j for j in body["jobs"] if isinstance(j, dict)], None
|
||
if isinstance(body, list):
|
||
return [j for j in body if isinstance(j, dict)], None
|
||
return [], "run-jobs:malformed-body"
|
||
|
||
|
||
def _resolve_job_id_for_context(
|
||
cfg: Any, run_id: int, context: str,
|
||
) -> tuple[int | None, str | None]:
|
||
"""When ``target_url`` only carries the run_id, list jobs in the
|
||
run and match by name/context. The Forgejo Actions API
|
||
typically calls the field ``name``; the commit status's
|
||
``context`` is conventionally ``"CI / {job-name}"`` so we
|
||
strip the ``"CI / "`` prefix before matching."""
|
||
jobs, err = _fetch_run_jobs(cfg, run_id)
|
||
if err:
|
||
return None, err
|
||
if not jobs:
|
||
return None, "run-jobs:empty"
|
||
# Try exact-name match first, then suffix match.
|
||
name_to_id: dict[str, int] = {}
|
||
for j in jobs:
|
||
try:
|
||
jid = int(j.get("id"))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
name = str(j.get("name") or "")
|
||
if name:
|
||
name_to_id[name] = jid
|
||
if context in name_to_id:
|
||
return name_to_id[context], None
|
||
# Strip common ``"CI / "`` prefix from the status context.
|
||
if " / " in context:
|
||
suffix = context.split(" / ", 1)[1]
|
||
if suffix in name_to_id:
|
||
return name_to_id[suffix], None
|
||
return None, "run-jobs:no-context-match"
|
||
|
||
|
||
def _fetch_job_log(
|
||
cfg: Any, run_id: int, job_id: int, *, max_chars: int,
|
||
) -> tuple[str | None, int | None, bool, str | None]:
|
||
"""Fetch the raw log text for a single job. Returns
|
||
``(log_tail, bytes_seen, truncated, error)``.
|
||
|
||
Forgejo's REST API does NOT expose action job logs (live-probed
|
||
2026-05-17: every variant of ``/api/v1/.../actions/.../logs``
|
||
returns 404 on this version). The UI route DOES serve raw text
|
||
at ``/{owner}/{repo}/actions/runs/{run_index}/jobs/{job_index}/attempt/{attempt}/logs``
|
||
but requires a session cookie (no token-auth — verified live).
|
||
|
||
Args:
|
||
run_id: the run **index_in_repo** (the small number from
|
||
target_url), not the global PK.
|
||
job_id: the per-run job INDEX (the trailing number in
|
||
target_url's ``/jobs/{N}``).
|
||
|
||
Both come from ``parse_run_job_ids(target_url)``.
|
||
|
||
We truncate to the LAST ``max_chars`` characters — the failing
|
||
assertion + stack trace are always at the tail."""
|
||
path = (
|
||
f"/{cfg.owner}/{cfg.repo}/actions/runs/{int(run_id)}/jobs/"
|
||
f"{int(job_id)}/attempt/1/logs"
|
||
)
|
||
try:
|
||
response = _ui_fetch_with_session(cfg, path)
|
||
except Exception as exc: # noqa: BLE001
|
||
return None, None, False, f"log:{type(exc).__name__}"
|
||
status = int(response.get("status") or 0)
|
||
if status != 200:
|
||
return None, None, False, f"log:status={status}"
|
||
body = response.get("body")
|
||
if isinstance(body, bytes):
|
||
try:
|
||
text = body.decode("utf-8", errors="replace")
|
||
except Exception: # noqa: BLE001
|
||
return None, len(body), False, "log:decode-error"
|
||
elif isinstance(body, str):
|
||
text = body
|
||
else:
|
||
return None, None, False, "log:unexpected-body-shape"
|
||
bytes_seen = len(text.encode("utf-8"))
|
||
if len(text) > max_chars:
|
||
# Keep the tail — failing assertions and stack traces live
|
||
# at the end of CI logs. Add a marker so the worker can tell
|
||
# this isn't the full log.
|
||
return (
|
||
"...[truncated head; showing last "
|
||
f"{max_chars} chars]...\n" + text[-max_chars:],
|
||
bytes_seen, True, None,
|
||
)
|
||
return text, bytes_seen, False, None
|
||
|
||
|
||
def _is_failing(state: str | None) -> bool:
|
||
"""Forgejo emits both ``state`` (statuses) and ``conclusion``
|
||
(actions jobs) with overlapping vocabularies. The set below is
|
||
everything we treat as "this check did not pass" — pending and
|
||
in_progress are NOT failing (yet)."""
|
||
if not state:
|
||
return False
|
||
return str(state).lower() in {
|
||
"failure", "failed", "error", "cancelled", "canceled", "timed_out",
|
||
}
|
||
|
||
|
||
def collect_failing_jobs(
|
||
cfg: Any,
|
||
ci_detail: list[dict[str, Any]],
|
||
*,
|
||
max_jobs: int,
|
||
max_chars_per_job: int,
|
||
) -> tuple[list[dict[str, Any]], bool]:
|
||
"""Build the failing-jobs payload from a CI detail list (the
|
||
list returned by :func:`_review_fetch.fetch_ci_check_detail`).
|
||
Returns ``(failing_jobs, completed)``. ``completed`` is True iff
|
||
every failing-status' log was fetched cleanly."""
|
||
failing_statuses = [
|
||
s for s in ci_detail
|
||
if isinstance(s, dict) and _is_failing(s.get("state") or s.get("status"))
|
||
]
|
||
failing_statuses = failing_statuses[:max_jobs]
|
||
out: list[dict[str, Any]] = []
|
||
all_clean = True
|
||
for s in failing_statuses:
|
||
context = s.get("context") or ""
|
||
target_url = s.get("target_url") or ""
|
||
run_id, job_id = parse_run_job_ids(target_url)
|
||
entry: dict[str, Any] = {
|
||
"context": context,
|
||
"state": s.get("state") or s.get("status"),
|
||
"description": s.get("description"),
|
||
"run_id": run_id,
|
||
"job_id": job_id,
|
||
"log_url": target_url,
|
||
"log_tail": None,
|
||
"log_bytes_seen": None,
|
||
"log_truncated": False,
|
||
"fetch_error": None,
|
||
}
|
||
if run_id is None:
|
||
# Unsupported URL shape (e.g. external Codecov link). The
|
||
# worker can still follow the link manually; we just
|
||
# can't pre-fetch its log content.
|
||
entry["fetch_error"] = "unsupported-url-shape"
|
||
all_clean = False
|
||
out.append(entry)
|
||
continue
|
||
if job_id is None:
|
||
resolved_job_id, resolve_err = _resolve_job_id_for_context(
|
||
cfg, run_id, context,
|
||
)
|
||
if resolve_err is not None or resolved_job_id is None:
|
||
entry["fetch_error"] = resolve_err or "run-jobs:no-job-id"
|
||
all_clean = False
|
||
out.append(entry)
|
||
continue
|
||
entry["job_id"] = job_id = resolved_job_id
|
||
log_tail, bytes_seen, truncated, log_err = _fetch_job_log(
|
||
cfg, int(run_id), int(job_id), max_chars=max_chars_per_job,
|
||
)
|
||
entry["log_tail"] = log_tail
|
||
entry["log_bytes_seen"] = bytes_seen
|
||
entry["log_truncated"] = truncated
|
||
entry["fetch_error"] = log_err
|
||
if log_err is not None:
|
||
all_clean = False
|
||
out.append(entry)
|
||
return out, all_clean
|
||
|
||
|
||
def _cache_covers_all_current_failures(
|
||
cached: dict[str, Any], ci_detail: list[dict[str, Any]] | None,
|
||
) -> bool:
|
||
"""True iff every distinct failing-context in ``ci_detail`` is
|
||
already present in the cached ``failing_jobs`` list.
|
||
|
||
The cache is per-SHA but Forgejo runs checks asynchronously —
|
||
a SHA's CI is NOT final until every workflow has reached a
|
||
terminal state. Marking the cache ``completed=True`` after the
|
||
first fetch of currently-failing jobs is wrong if any checks
|
||
were still pending at fetch time and later transitioned to
|
||
failure. This predicate lets the cache stay sealed when the
|
||
failure set hasn't grown, and triggers a re-fetch when it has.
|
||
|
||
When ``ci_detail`` is None (caller couldn't provide it), we
|
||
conservatively assume the cache is still valid — the prior
|
||
behaviour. The caller upstream can still pass a fresh detail
|
||
list to force the comparison.
|
||
"""
|
||
if not isinstance(cached, dict):
|
||
return False
|
||
if ci_detail is None:
|
||
return True
|
||
cached_contexts = {
|
||
(j.get("context") or "")
|
||
for j in (cached.get("failing_jobs") or [])
|
||
if isinstance(j, dict)
|
||
}
|
||
current_failing_contexts = {
|
||
(s.get("context") or "")
|
||
for s in ci_detail
|
||
if isinstance(s, dict)
|
||
and _is_failing(s.get("state") or s.get("status"))
|
||
}
|
||
current_failing_contexts.discard("")
|
||
# Cache covers everything currently failing iff every current
|
||
# failing context is already in the cache.
|
||
return current_failing_contexts.issubset(cached_contexts)
|
||
|
||
|
||
def fetch_pr_failure_logs(
|
||
cfg: Any,
|
||
head_sha: str,
|
||
*,
|
||
ci_detail: list[dict[str, Any]] | None = None,
|
||
max_jobs: int | None = None,
|
||
max_chars_per_job: int | None = None,
|
||
) -> tuple[dict[str, Any], bool]:
|
||
"""Top-level: fetch the failing-CI log tails for ``head_sha``
|
||
via the on-disk cache.
|
||
|
||
Returns ``(payload, completed)`` where ``payload`` carries the
|
||
schema documented at module top. ``completed`` is True iff every
|
||
failing-job log was fetched cleanly (or there were no failing
|
||
jobs to fetch).
|
||
|
||
``ci_detail`` may be supplied by callers who already fetched it
|
||
(dispatcher pre-fetch path); otherwise we fetch it ourselves.
|
||
|
||
On a cold cache miss: fetch live, persist, return.
|
||
On a cache hit that's already ``completed=True``: serve cached
|
||
(per-SHA immutability — terminal CI state is frozen).
|
||
On a cache hit that's ``completed=False`` AND inside the backoff
|
||
window: serve cached with ``completed=False`` (no live retry).
|
||
On a cache hit that's ``completed=False`` AND backoff expired:
|
||
attempt a fresh fetch and persist whatever we get.
|
||
"""
|
||
effective_max_jobs = max_jobs if max_jobs is not None else _max_jobs()
|
||
effective_max_chars = (
|
||
max_chars_per_job if max_chars_per_job is not None
|
||
else _max_chars_per_job()
|
||
)
|
||
if not head_sha:
|
||
# No SHA → no cache key → no fetch. Return an empty envelope
|
||
# rather than raising so the caller doesn't have to defend.
|
||
empty: dict[str, Any] = {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"head_sha": "",
|
||
"fetched_at": _now(),
|
||
"failing_jobs": [],
|
||
"completed": True,
|
||
"consecutive_failures": 0,
|
||
"next_attempt_after": None,
|
||
}
|
||
return empty, True
|
||
if is_disabled():
|
||
# Bypass cache entirely (live every call).
|
||
detail = ci_detail
|
||
if detail is None:
|
||
detail, _ok = _review_fetch.fetch_ci_check_detail(cfg, head_sha)
|
||
failing, ok = collect_failing_jobs(
|
||
cfg, detail or [],
|
||
max_jobs=effective_max_jobs,
|
||
max_chars_per_job=effective_max_chars,
|
||
)
|
||
payload = {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"head_sha": head_sha,
|
||
"fetched_at": _now(),
|
||
"failing_jobs": failing,
|
||
"completed": ok,
|
||
"consecutive_failures": 0,
|
||
"next_attempt_after": None,
|
||
}
|
||
return payload, ok
|
||
cached = _read_cache(head_sha)
|
||
now_dt = _dt.datetime.now(_dt.timezone.utc)
|
||
if cached is not None and bool(cached.get("completed")):
|
||
# Per-SHA immutability claim: a terminally-fetched cache is
|
||
# good forever — once the CI on a SHA reaches terminal state,
|
||
# nothing changes. BUT — the original implementation marked
|
||
# ``completed=True`` as soon as the CURRENTLY-failing jobs'
|
||
# logs were fetched cleanly, without checking whether checks
|
||
# still-pending at fetch time might later transition to
|
||
# failure. PR #40 run-7 (2026-05-17): cache was sealed at
|
||
# 20:38 with 1 failing job (push-validation); lint and
|
||
# unit_tests transitioned pending→failure later, but the
|
||
# cache was treated as frozen and never re-fetched. The
|
||
# reviewer + implementer both saw only 1 of 3 actual failures.
|
||
#
|
||
# Fix: if the current ``ci_detail`` has more distinct failing
|
||
# contexts than the cached failing_jobs covers, the cache is
|
||
# stale → invalidate and live-fetch. Same-set comparison =
|
||
# cache still valid.
|
||
if _cache_covers_all_current_failures(cached, ci_detail):
|
||
return cached, True
|
||
_logger.info(
|
||
"ci-logs cache for %s has %d failing_jobs but ci_detail "
|
||
"now shows additional failures — re-fetching (some "
|
||
"checks transitioned pending→failure after the cache was "
|
||
"first sealed)",
|
||
head_sha, len(cached.get("failing_jobs") or []),
|
||
)
|
||
# Fall through to the live-fetch path below.
|
||
if cached is not None and _backoff_active(cached, now_dt):
|
||
_logger.info(
|
||
"ci-logs cache for %s in backoff "
|
||
"(failures=%s, next_attempt_after=%s); serving stale",
|
||
head_sha,
|
||
cached.get("consecutive_failures", 0),
|
||
cached.get("next_attempt_after"),
|
||
)
|
||
return cached, False
|
||
# Cache miss OR not-yet-completed with expired backoff → live fetch.
|
||
detail = ci_detail
|
||
if detail is None:
|
||
try:
|
||
detail, _ok = _review_fetch.fetch_ci_check_detail(cfg, head_sha)
|
||
except Exception as exc: # noqa: BLE001
|
||
# Any exception in ci-detail fetch is treated as a
|
||
# backoff-eligible failure so a flaky upstream doesn't
|
||
# burn the dispatcher's per-cycle window.
|
||
return _record_failure(
|
||
head_sha, cached, now_dt,
|
||
error=f"ci-detail:{type(exc).__name__}",
|
||
)
|
||
failing, ok = collect_failing_jobs(
|
||
cfg, detail or [],
|
||
max_jobs=effective_max_jobs,
|
||
max_chars_per_job=effective_max_chars,
|
||
)
|
||
if ok:
|
||
payload = {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"head_sha": head_sha,
|
||
"fetched_at": _now(),
|
||
"failing_jobs": failing,
|
||
"completed": True,
|
||
"consecutive_failures": 0,
|
||
"next_attempt_after": None,
|
||
}
|
||
_write_cache(head_sha, payload)
|
||
return payload, True
|
||
# Partial — at least one job log fetch failed. Persist what we
|
||
# have AND track the failure for backoff.
|
||
prior_failures = int((cached or {}).get("consecutive_failures") or 0)
|
||
next_failures = prior_failures + 1
|
||
payload = {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"head_sha": head_sha,
|
||
"fetched_at": _now(),
|
||
"failing_jobs": failing,
|
||
"completed": False,
|
||
"consecutive_failures": next_failures,
|
||
"next_attempt_after": _compute_next_attempt_after(
|
||
next_failures, now_dt,
|
||
),
|
||
}
|
||
_write_cache(head_sha, payload)
|
||
_logger.warning(
|
||
"ci-logs partial fetch for %s (%s/%s jobs missing logs); "
|
||
"consecutive_failures=%s; next attempt deferred until %s",
|
||
head_sha,
|
||
sum(1 for j in failing if j.get("fetch_error")),
|
||
len(failing),
|
||
next_failures,
|
||
payload["next_attempt_after"],
|
||
)
|
||
return payload, False
|
||
|
||
|
||
def _record_failure(
|
||
head_sha: str,
|
||
prior_cache: dict[str, Any] | None,
|
||
now_dt: _dt.datetime,
|
||
*,
|
||
error: str,
|
||
) -> tuple[dict[str, Any], bool]:
|
||
"""A bookkeeping shortcut for "the live attempt couldn't even
|
||
start" (e.g. ci-detail fetch raised). Persists a minimal
|
||
failure-tracking record + the cached failing_jobs (if any) so the
|
||
backoff loop converges normally."""
|
||
prior_failures = int((prior_cache or {}).get("consecutive_failures") or 0)
|
||
next_failures = prior_failures + 1
|
||
payload = {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"head_sha": head_sha,
|
||
"fetched_at": _now(),
|
||
"failing_jobs": list((prior_cache or {}).get("failing_jobs") or []),
|
||
"completed": False,
|
||
"consecutive_failures": next_failures,
|
||
"next_attempt_after": _compute_next_attempt_after(
|
||
next_failures, now_dt,
|
||
),
|
||
"last_error": error,
|
||
}
|
||
_write_cache(head_sha, payload)
|
||
_logger.warning(
|
||
"ci-logs live fetch failed for %s (%s); "
|
||
"consecutive_failures=%s; next attempt deferred until %s",
|
||
head_sha, error, next_failures, payload["next_attempt_after"],
|
||
)
|
||
return payload, False
|
||
|
||
|
||
def invalidate(head_sha: str) -> None:
|
||
"""Force-remove the cache entry. Idempotent. Use after a CI
|
||
re-run when you want the next fetch to repopulate, or in tests."""
|
||
target = cache_path(head_sha)
|
||
try:
|
||
target.unlink(missing_ok=True)
|
||
except OSError as exc:
|
||
_logger.warning(
|
||
"ci-logs cache invalidate failed for %s at %s: %s",
|
||
head_sha, target, exc,
|
||
)
|
||
|
||
|
||
__all__ = (
|
||
"SCHEMA_VERSION",
|
||
"cache_dir",
|
||
"cache_path",
|
||
"collect_failing_jobs",
|
||
"fetch_pr_failure_logs",
|
||
"invalidate",
|
||
"is_disabled",
|
||
"parse_run_job_ids",
|
||
)
|