"""Persistent on-disk cache for PR/issue comments with delta fetch. What this is ------------ Live test 2026-05-13 showed every implementer cycle on PR #30 re-fetched 1340+ accumulated comments via the ``/repos/{owner}/{repo}/issues/{N}/comments`` endpoint, hit the 20-page pagination cap, truncated the most recent comments off the end, and burned ~30s on the fetch alone. Across 5 attempts in a single PR cycle that's ~150s of identical work plus stale data. This module caches the comment list to disk and supports a delta fetch via Forgejo's ``?since=`` query parameter. The common case becomes: 5-10 NEW comments since the last fetch (or zero) rather than the full 1340. Cache layout ------------ ``/tmp/cleveragents-comment-cache/pr-{N}.json`` — per-PR JSON file. Schema:: { "schema_version": 1, "pr_number": 30, "fetched_at": "2026-05-13T10:00:00+00:00", "since_cursor": "2026-05-13T09:59:00+00:00", "comments": [ {"id": 1, "created_at": "...", "body": "...", ...}, ... ], "any_partial_fetch": false } ``fetched_at`` is the wall-clock time of the last write — it drives the staleness check only. ``since_cursor`` is the ``created_at`` of the newest cached comment; it is what we pass as ``?since=`` on the next delta fetch, so the delta backfills forward from where the cache actually ends. A page-capped seed holds only the OLDEST N comments, so its newest cached comment is NOT the newest on the PR — using a wall-clock ``?since=`` would skip the un-fetched middle forever. The list is append-only by id. ``any_partial_fetch`` is True when the cache was seeded from a page-capped fetch and has not yet been backfilled to completeness by a later delta. Invalidation ------------ Comments edited or deleted upstream are NOT detected in v1. Two defenses: - ``CACHE_MAX_STALENESS_S`` (default 24h) — when the cache is older than this, we discard it and do a full refetch. Bounds the staleness window. - Force-refresh via ``invalidate(pr_number)`` for operator-side manual interventions. Edits and deletions are rare in this codebase's PR workflow; the v1 trade-off (skip detection, do periodic refresh) is intentional. Concurrency ----------- The cache file is written atomically (write-to-tmp + rename). Two dispatchers reading the SAME PR's comments concurrently each compute the same delta and the last writer wins — idempotent. A modest amount of duplicate-fetch work is the cost; the alternative (file locking) would serialise readers, which is worse. The cache is shared between the reviewer and implementer dispatchers because both fetch the same endpoint with the same auth and both benefit equally from the cache. Errors ------ Every code path is best-effort. Cache read failures fall through to a full fetch. Delta fetch failures fall through to a full fetch. Full fetch failures bubble up to the caller as ``(items, completed=False)`` — same contract as ``_review_fetch._api_get_paginated``. """ from __future__ import annotations import datetime as _dt import json import logging import os import sys from pathlib import Path from typing import Any _TOOLS_DIR = str(Path(__file__).resolve().parent) if _TOOLS_DIR not in sys.path: sys.path.insert(0, _TOOLS_DIR) from _loader import ( # noqa: E402 type: ignore[import-not-found] load_sibling as _load_sibling, ) _review_fetch = _load_sibling("_review_fetch", "_review_fetch.py") _backoff = _load_sibling("_backoff", "_backoff.py") _attempt_history = _load_sibling("_attempt_history", "_attempt_history.py") _bot_logins = _load_sibling("_bot_logins", "_bot_logins.py") _logger = logging.getLogger("pr_comments_cache") SCHEMA_VERSION = 1 # Default cache directory. Override via env var for tests / archive # sharing between dispatcher hosts. _DEFAULT_CACHE_DIR = Path("/tmp/cleveragents-comment-cache") _CACHE_DIR_ENV = "IMPLEMENTER_DISPATCHER_COMMENT_CACHE_DIR" # 24 hours — when the cache is older than this we discard it and # do a full refetch. Bounds the edit/delete staleness window # without paying the full-fetch cost every cycle. _DEFAULT_MAX_STALENESS_S = 86400 _STALENESS_ENV = "IMPLEMENTER_DISPATCHER_COMMENT_CACHE_MAX_STALENESS_S" # Kill switch for tests + operators who want to bypass the cache # entirely (e.g. while diagnosing a divergence between cached and # upstream state). _DISABLE_ENV = "IMPLEMENTER_DISPATCHER_COMMENT_CACHE_DISABLE" # Filter bot status / claim / release / sentinel comments BEFORE # writing to cache. Bot ``**Implementation Attempt**`` comments are # KEPT — the prompt's attempt-history digest reads them. Human and # reviewer comments are KEPT. The dropped count + by-author breakdown # is stamped on the cache row as ``bot_filtered`` so the prompt # section can surface a "N bot comments filtered" one-liner. # # Without this, PR #30's cache balloons to ~19,000+ comments (mostly # claim/release/sentinel noise the worker never reads) and the # 50-page paginator cap still clips the middle. After filtering, the # typical cache fits well within any reasonable page budget. # # Disable for rollback / debugging via # ``IMPLEMENTER_DISPATCHER_COMMENT_CACHE_FILTER_BOTS=0``. _FILTER_BOTS_ENV = "IMPLEMENTER_DISPATCHER_COMMENT_CACHE_FILTER_BOTS" def _filter_bots_enabled() -> bool: raw = os.environ.get(_FILTER_BOTS_ENV) if raw is None: return True return raw.strip().lower() in {"1", "true", "yes", "on"} def _filter_for_cache( comments: list[dict[str, Any]], ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Drop bot status/claim/release/sentinel comments; keep human comments and bot ``**Implementation Attempt**`` markers. Returns ``(kept, summary)`` where: - ``kept`` is the filtered comment list (caller stores this). - ``summary`` is ``{"count": N, "by_author": {login: N, ...}}`` for the dropped comments (caller stores this for the prompt section header). Empty dict when filtering is disabled or no comments were dropped. """ if not _filter_bots_enabled() or not comments: return comments, {"count": 0, "by_author": {}} bots = _bot_logins.bot_logins() kept: list[dict[str, Any]] = [] by_author: dict[str, int] = {} for c in comments: if not isinstance(c, dict): continue # Keep ALL non-bot comments + bot attempt-marker comments. if _attempt_history.is_bot_authored( c, bots ) and not _attempt_history.is_attempt_comment(c): user = c.get("user") login = user.get("login") if isinstance(user, dict) else None # Default to ``"unknown"`` so by_author keys are always # strings — JSON serialisation of the cache row would # otherwise emit ``null`` as a key (Python's json.dumps # coerces None → "null"), and downstream prompt rendering # treats the literal string "null" as a login. by_author[login or "unknown"] = by_author.get(login or "unknown", 0) + 1 continue kept.append(c) summary = { "count": sum(by_author.values()), "by_author": by_author, } return kept, summary # Exponential backoff for repeated live-delta failures (R1 + # 2026-05-16). Live test on PR #29 saw the dispatcher hit the # pagination cap ``max_pages=20`` six times in a single hour because # every cycle re-attempted the comment fetch against the same flaky # endpoint, each retry burning ~30s of the worker turn AND polluting # the prompt with a fresh ``data_complete=False`` signal. Counter # resets on the first successful delta — a transient outage doesn't # permanently throttle the PR. Shared shape with the other cache # modules via :class:`_backoff.Backoff`. _BACKOFF = _backoff.Backoff( base_env="IMPLEMENTER_DISPATCHER_COMMENT_CACHE_BACKOFF_BASE_S", max_env="IMPLEMENTER_DISPATCHER_COMMENT_CACHE_BACKOFF_MAX_S", base_default=60, max_default=1800, ) def cache_dir() -> Path: """Resolve the on-disk cache directory.""" return Path(os.environ.get(_CACHE_DIR_ENV) or str(_DEFAULT_CACHE_DIR)) def cache_path(pr_number: int) -> Path: """Per-PR cache file path.""" return cache_dir() / f"pr-{int(pr_number)}.json" def _max_staleness_s() -> int: raw = os.environ.get(_STALENESS_ENV) if raw: try: return max(0, int(raw)) except ValueError: pass return _DEFAULT_MAX_STALENESS_S def is_disabled() -> bool: raw = os.environ.get(_DISABLE_ENV, "").strip().lower() return raw in {"1", "true", "yes", "on"} def _now() -> str: return _dt.datetime.now(_dt.timezone.utc).isoformat() # Thin module-level wrappers preserved so the test suite + any direct # imports continue to work. Both delegate to the shared # :class:`_backoff.Backoff` instance above. def _compute_next_attempt_after( failures: int, now_dt: _dt.datetime, ) -> str | None: return _BACKOFF.next_attempt_after(failures, now_dt) def _backoff_active(cached: dict[str, Any] | None, now_dt: _dt.datetime) -> bool: return _BACKOFF.is_active(cached, now_dt) def _read_cache(pr_number: int) -> dict[str, Any] | None: """Load the cache file for ``pr_number``. Returns ``None`` on any error (missing, malformed, schema-mismatch, stale).""" target = cache_path(pr_number) if not target.exists(): return None try: text = target.read_text(encoding="utf-8") payload = json.loads(text) except (OSError, ValueError) as exc: _logger.warning( "comment cache read failed for PR #%s at %s: %s", pr_number, target, exc, ) return None if not isinstance(payload, dict): return None if payload.get("schema_version") != SCHEMA_VERSION: return None # Staleness check try: fetched_at = _dt.datetime.fromisoformat(payload.get("fetched_at", "")) age_s = (_dt.datetime.now(_dt.timezone.utc) - fetched_at).total_seconds() if age_s > _max_staleness_s(): _logger.info( "comment cache for PR #%s is %.0fs stale (limit %ds); " "discarding for full refetch", pr_number, age_s, _max_staleness_s(), ) return None except (ValueError, TypeError): # Bad timestamp → treat as no-cache return None if not isinstance(payload.get("comments"), list): return None # Lazy migration: legacy cache rows (pre-2026-05-17) sometimes # wrote ``null`` as the by_author key when the comment's user # field was missing. Normalize on read so downstream consumers # never see a non-string key. Cheap (rewrites in-memory only; # next write persists the canonical shape). bf = payload.get("bot_filtered") if isinstance(bf, dict): raw_by_author = bf.get("by_author") if isinstance(raw_by_author, dict) and any( not isinstance(k, str) for k in raw_by_author ): normalized: dict[str, int] = {} for k, v in raw_by_author.items(): key = k if isinstance(k, str) and k else "unknown" try: value = int(v or 0) except (TypeError, ValueError): value = 0 normalized[key] = normalized.get(key, 0) + value bf["by_author"] = normalized return payload def _write_cache(pr_number: int, payload: dict[str, Any]) -> None: """Atomic write of the cache file.""" target = cache_path(pr_number) tmp = target.with_suffix(target.suffix + ".tmp") try: target.parent.mkdir(parents=True, exist_ok=True) tmp.write_text( json.dumps(payload, indent=2, default=str), encoding="utf-8", ) tmp.replace(target) except (OSError, TypeError, ValueError) as exc: _logger.warning( "comment cache write failed for PR #%s at %s: %s", pr_number, target, exc, ) try: tmp.unlink(missing_ok=True) except OSError: pass def _merge_comments( cached: list[dict[str, Any]], delta: list[dict[str, Any]], ) -> list[dict[str, Any]]: """Append ``delta`` to ``cached``, deduplicated by comment ``id``. Forgejo's ``since=`` query is timestamp-based; if a comment was created at the boundary timestamp, the server MAY return it in both the cached snapshot and the delta. Dedup by id is the simplest defence. Preserves the order of ``cached`` (oldest first) and appends delta items (also oldest-first within delta) at the tail. """ seen_ids: set[Any] = set() out: list[dict[str, Any]] = [] for c in cached: cid = c.get("id") if cid is not None and cid in seen_ids: continue if cid is not None: seen_ids.add(cid) out.append(c) for c in delta: cid = c.get("id") if cid is not None and cid in seen_ids: continue if cid is not None: seen_ids.add(cid) out.append(c) return out def _normalize_since_cursor(cursor: str) -> str: """Strip fractional seconds from an ISO-8601 timestamp so the cursor matches the wire format Forgejo accepts in ``?since=``. Why: Forgejo (live-observed 2026-05-17 on PRs #25 + #28) returns HTTP 422 when the ``since=`` parameter carries microsecond precision (e.g. ``2026-05-17T04:48:23.067808+00:00``). The legacy cache code stamped ``fetched_at`` with ``datetime.now(UTC).isoformat()`` — which includes microseconds — and used that as the cursor for the next delta, so any cache row written before the cursor switched to ``created_at`` (whole-second) is poisoned for the lifetime of the cache. This helper sanitizes on the way OUT so even legacy rows recover on the next cycle. Accepts: ``Z`` / lowercase ``z`` (RFC-3339 allows both); ``+HH:MM`` / ``-HH:MM`` offsets including non-zero (``+05:30``); naive timestamps. Preserves the original timezone marker character so a lowercase-``z`` input doesn't silently lose its UTC designator. Conservative: input that doesn't look like an ISO timestamp (no ``T`` separator) is returned untouched — no exception, no truncation that could change semantics.""" if not cursor: return cursor t_pos = cursor.find("T") if t_pos < 0: # Not an ISO-8601 timestamp — leave alone, do NOT strip on # the bare ``.`` (would mangle filenames, URLs, etc.). return cursor # ``+HH:MM`` / ``-HH:MM`` offset form: split at the tz sep, trim # fraction in the date-time half, rejoin verbatim. for tz_sep in ("+", "-"): tz_pos = cursor.find(tz_sep, t_pos) if tz_pos > t_pos: dt_part = cursor[:tz_pos] tz_part = cursor[tz_pos:] dot = dt_part.find(".") if dot >= 0: dt_part = dt_part[:dot] return dt_part + tz_part # ``Z`` / ``z`` form: preserve the original case of the marker so # legal RFC-3339 lowercase ``z`` inputs don't lose their tz suffix # by falling through to the naive-stripper. if cursor and cursor[-1] in ("Z", "z"): marker = cursor[-1] dt_part = cursor[:-1] dot = dt_part.find(".") if dot >= 0: dt_part = dt_part[:dot] return dt_part + marker # Naive ISO form (no tz suffix, but has T): trim the fraction in # the time component only. Bounded by t_pos so a bare "." in the # date portion (impossible in valid ISO but defensive) is left # alone. dot = cursor.find(".", t_pos) if dot >= 0: return cursor[:dot] return cursor def _newest_cursor(comments: list[dict[str, Any]]) -> str: """Return the ``created_at`` of the newest (last) comment, for use as the ``?since=`` delta cursor on the next fetch. The comment list is oldest-first, so the last entry is the newest. Returns "" when the list is empty or the newest entry has no ``created_at`` — the caller falls back to ``fetched_at``. Why this and not ``fetched_at`` (wall-clock): a page-capped seed holds only the OLDEST N comments, so the newest *cached* comment is not the newest on the PR. ``?since=`` backfills forward from where the cache actually ends; ``?since=`` would skip the un-fetched middle forever. """ # Walk from the tail back, skipping non-dict / cursor-less entries. # Why not just trust comments[-1]: ``_filter_for_cache`` skips # non-dict entries silently, so the cached list and the raw # response list can disagree on what "last" means. If Forgejo # returns a malformed entry at the tail of a page (rare but # observed in past Forgejo bugs), trusting the last element # blindly would regress the cursor to "" and the next delta # would re-paginate from page 1. The walk costs O(k) where k is # the count of trailing malformed entries — typically 0 or 1. for entry in reversed(comments): if isinstance(entry, dict): created_at = entry.get("created_at") if created_at: return str(created_at) return "" def get_pr_comments( cfg: Any, pr_number: int, *, owner: str | None = None, repo: str | None = None, ) -> tuple[list[dict[str, Any]], bool]: """Get all comments for the PR, using the on-disk cache for the bulk and fetching only the delta. Returns ``(comments, completed)`` — same contract as :func:`_review_fetch.fetch_pr_comments`. ``completed`` is True when both the cache load AND the delta fetch succeeded cleanly; False if either was partial. ``owner`` / ``repo`` override the values normally read off ``cfg``. The shared comment fetch stack only uses ``cfg.owner`` / ``cfg.repo`` to build the request path; the HTTP transport layer (``_claim_runtime.get``) needs just ``token`` / ``request_timeout_s`` / ``api_retries``. Callers whose ``cfg`` is the narrower ``RuntimeContext`` (e.g. ``_claim_runtime``'s sweep, which uses module-level ``REPO_OWNER`` / ``REPO_NAME`` constants) pass owner / repo explicitly so they can share the cache without growing their cfg shape (spec #6). When the cache is disabled, missing, or stale, falls back to a full fetch (same behaviour as the legacy code path) and seeds the cache for next time. Seed-on-truncation (R8-5): a full fetch that hits the pagination ceiling (``truncated=True``) still seeds the cache — the clipped data is valid, just incomplete — marked ``any_partial_fetch=True``. The next run's delta fetch (``?since=``) backfills forward from the newest cached comment. Before this fix, a heavy PR whose full fetch always truncated NEVER seeded the cache and re-paginated from page 1 every single cycle. A genuinely-failed fetch (network / 5xx / malformed, ``truncated=False``) still does NOT seed — its data is unreliable. """ use_owner = owner or getattr(cfg, "owner", None) use_repo = repo or getattr(cfg, "repo", None) base_path = f"/repos/{use_owner}/{use_repo}/issues/{int(pr_number)}/comments" if is_disabled(): # Path-based fetch (no cache layer). Uses the same paginator the # cache layer uses, so cfg with only RuntimeContext attrs works. items, ok = _review_fetch._api_get_paginated(cfg, base_path) return items, ok cached = _read_cache(pr_number) now_dt = _dt.datetime.now(_dt.timezone.utc) # Backoff short-circuit. If a previous cycle's live-delta failed # and we're inside the deferred window, skip the live attempt # entirely and serve whatever cached bulk we have. The caller # sees ``completed=False`` so the dispatcher's prompt-time # ``data_complete`` aggregate still flips, but we have NOT # burned another ~30s on a guaranteed-to-fail pagination walk. if cached is not None and _backoff_active(cached, now_dt): _logger.info( "comment cache for PR #%s in backoff (failures=%s, " "next_attempt_after=%s); serving stale cache, skipping live " "delta", pr_number, cached.get("consecutive_failures", 0), cached.get("next_attempt_after"), ) return list(cached.get("comments") or []), False if cached is None: # Cache miss → full fetch. Seed on a clean walk OR a page-cap # truncation; skip seeding only on a genuine transient failure # (its data is unreliable). ``since_cursor`` is the newest # comment we actually have, so a truncated seed's next delta # backfills forward instead of from a wall-clock ``?since=``. raw_items, completed, truncated = _review_fetch._api_get_paginated( cfg, base_path, return_truncation=True, ) # Drop bot status/claim/release/sentinel noise before caching. # ``since_cursor`` uses the RAW newest comment (including # filtered ones) so the next delta's ``?since=`` resumes from # the true tip, not from the most-recent kept entry. items, filter_summary = _filter_for_cache(raw_items) if completed or truncated: failures = 0 if completed else 1 _write_cache( pr_number, { "schema_version": SCHEMA_VERSION, "pr_number": int(pr_number), "fetched_at": _now(), "since_cursor": _newest_cursor(raw_items), "comments": items, "any_partial_fetch": not completed, "consecutive_failures": failures, "next_attempt_after": _compute_next_attempt_after( failures, now_dt, ), "bot_filtered": filter_summary, }, ) return items, completed # Cache hit → delta fetch forward from the newest cached comment. # Fall back to ``fetched_at`` for caches written before the # ``since_cursor`` field existed. Both candidates are normalized # to drop microsecond precision because Forgejo returns HTTP 422 # on sub-second ``since=`` values (live-observed on PRs #25 + #28 # 2026-05-17). Legacy cache rows that still carry a microsecond # ``fetched_at`` recover automatically on the next cycle. # # Cursor is URL-encoded via ``urllib.parse.quote`` before # interpolation: the ``+`` in ``+00:00`` decodes to a space on # Forgejo's query-string parser and causes a second 422 class # (live-observed on PR #35 run-1 2026-05-17 — 6 consecutive 422s # on the same clean ``+00:00`` cursor until the cache backed off # for 30 min). ``safe=''`` quotes every non-alphanumeric char so # ``+`` → ``%2B``, ``:`` → ``%3A``, etc. from urllib.parse import quote as _url_quote since = _normalize_since_cursor( cached.get("since_cursor") or cached.get("fetched_at", "") ) path = f"{base_path}?since={_url_quote(since, safe='')}" if since else base_path raw_delta_items, delta_ok = _review_fetch._api_get_paginated(cfg, path) # Filter the delta before merging. Accumulate the filtered count # into the cache's running total so the prompt header reflects # everything skipped across all cycles. delta_items, delta_filter_summary = _filter_for_cache(raw_delta_items) prior_bot_filtered = cached.get("bot_filtered") or { "count": 0, "by_author": {}, } merged_by_author: dict[str, int] = dict(prior_bot_filtered.get("by_author") or {}) for login, n in (delta_filter_summary.get("by_author") or {}).items(): merged_by_author[login] = merged_by_author.get(login, 0) + n merged_bot_filtered = { "count": sum(merged_by_author.values()), "by_author": merged_by_author, } if not delta_ok: # Delta fetch was partial. Bump the failure counter and stamp # the deferred-retry deadline so the NEXT cycle short-circuits # to the stale-serve path above. Persist the merged set so the # short-circuit serves the SAME view this branch returned to # the caller, not the older subset — ``_merge_comments`` dedups # by id, so the merged write is strictly additive (no risk of # discarding cached entries). ``any_partial_fetch`` stays True # because the walk did not reach the PR's end. prior_failures = int(cached.get("consecutive_failures") or 0) next_failures = prior_failures + 1 merged = _merge_comments(cached["comments"], delta_items) _write_cache( pr_number, { **cached, "comments": merged, # since_cursor uses RAW delta tail (not filtered) so the # next ``?since=`` resumes from the true newest comment, # not the most-recent kept entry. "since_cursor": ( _newest_cursor(raw_delta_items) if raw_delta_items else cached.get("since_cursor", "") ), "any_partial_fetch": True, "consecutive_failures": next_failures, "next_attempt_after": _compute_next_attempt_after( next_failures, now_dt, ), "bot_filtered": merged_bot_filtered, }, ) _logger.warning( "comment cache live-delta failed for PR #%s " "(consecutive_failures=%s); next attempt deferred until %s; " "serving %s cached comments to caller", pr_number, next_failures, _compute_next_attempt_after(next_failures, now_dt), len(merged), ) return merged, False # Delta succeeded — merge, persist, return. A clean delta walks to # the PR's end, so even a previously-truncated seed is now whole: # ``any_partial_fetch`` clears AND failure-tracking resets. merged = _merge_comments(cached["comments"], delta_items) cached_payload = { "schema_version": SCHEMA_VERSION, "pr_number": int(pr_number), "fetched_at": _now(), "since_cursor": ( _newest_cursor(raw_delta_items) if raw_delta_items else cached.get("since_cursor", "") ), "comments": merged, "any_partial_fetch": False, "consecutive_failures": 0, "next_attempt_after": None, "bot_filtered": merged_bot_filtered, } _write_cache(pr_number, cached_payload) return merged, True def get_filter_summary(pr_number: int) -> dict[str, Any] | None: """Read the ``bot_filtered`` summary for ``pr_number``'s cache row. Returns ``None`` if the cache is missing, stale, or has no filter summary (older cache file pre-dating the filter feature). Shape: ``{"count": N, "by_author": {login: N, ...}}``. Consumers (prompt builders) call this to surface a one-line "N bot comments filtered" header. The summary is informational only — the actual filtered comments are NOT in the cache, by design (see ``_filter_for_cache`` rationale).""" cached = _read_cache(pr_number) if cached is None: return None summary = cached.get("bot_filtered") if not isinstance(summary, dict): return None return summary def invalidate(pr_number: int) -> None: """Force-remove the cache entry for a PR. Used by: - The dispatcher's post-session cleanup, optionally on hook failures or other "this PR's state moved underneath us" cases. - Operator manual interventions via a future ``tools/clear_comment_cache.py`` script. Idempotent — missing entry is a no-op. """ target = cache_path(pr_number) try: target.unlink(missing_ok=True) except OSError as exc: _logger.warning( "comment cache invalidate failed for PR #%s at %s: %s", pr_number, target, exc, ) __all__ = ( "SCHEMA_VERSION", "cache_dir", "cache_path", "get_filter_summary", "get_pr_comments", "invalidate", "is_disabled", )