Files
cleveragents-core/tools/_implementer_prefetch.py
T
drew 0bc734c020 style: ruff format the controller-state-machine branch (288 files)
Applies `ruff format` to the accumulated formatting debt on this branch.
Formatting-only — no behavioral changes. Required for CI/lint's format
gate (`nox -s format -- --check`), which the branch was failing on 288
tracked files that drifted from ruff's canonical style.

In-progress WIP files are intentionally excluded so this commit stays a
clean formatting-only diff.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-20 00:09:17 -04:00

1056 lines
46 KiB
Python

"""Pre-dispatch Forgejo fetches for the implementer dispatcher.
Mirrors :func:`_review_prompt.fetch_review_context` on the implementer
side. The implementer worker has historically issued every Forgejo
GET inside its LLM session (see the PR #30 post-mortem on 2026-05-08
where the estimator burned 10 min on five unauthed ``webfetch`` calls
before any code was written). This module pulls every read into a
single deterministic pre-dispatch pass so the worker's prompt is
self-contained — no in-session HTTP, no in-session pagination, no
tool-budget creep on ``webfetch``.
Three pre-fetch shapes, one per work group:
- ``pr_fix`` — failing-CI PR. Fetches PR details + diff +
CI combined status + per-check status detail (when overall != success)
+ paginated PR comments + linked-issue bodies + an Epic body when
the PR / linked issues reference one.
- ``request_changes_pr`` — PR with at least one active
``REQUEST_CHANGES`` review the author has not yet addressed.
Same as ``pr_fix`` plus the active reviews list (with their inline
comments) so the worker can read each blocking concern without
paging through the API itself.
- ``new_issue`` — implementing a fresh issue. No PR exists yet,
so we fetch the issue body + paginated issue comments + linked
bodies + Epic body only.
The Forgejo HTTP primitives come from :mod:`_review_fetch` — the
existing fetcher set is fully generic (see the per-function docstrings;
none of them are review-specific in behaviour, only in original
caller). Reusing them here closes the "implementer dispatcher does
zero pre-fetch" gap without forking a parallel ``_implementer_fetch``
module that would need its own ``_api_get_paginated`` retry / partial-
pagination handling.
Each fetcher swallows its own failures and returns ``None`` / ``[]``
so this module never raises. Sections downstream
(:mod:`_implementer_prompt`) handle the missing-data case explicitly
by emitting an "X unavailable" stanza instead of an empty one — the
worker is told what was attempted so it can decide whether to fall
back to its in-session HTTP rules.
The dispatcher's overall ``data_complete`` flag is the AND of every
section's ``completed`` signal. Today's implementer worker does not
short-circuit on incomplete data the way the reviewer dispatcher
defensively downgrades APPROVED → COMMENT, but the flag is still
emitted so a future post-session action (e.g. operator-status
fingerprinting per Phase 5b) can record "this cycle was dispatched
against a partial context" for triage.
"""
from __future__ import annotations
import logging
import os
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
_TOOLS_DIR = str(Path(__file__).resolve().parent)
if _TOOLS_DIR not in sys.path:
sys.path.insert(0, _TOOLS_DIR)
from _loader import ( # noqa: E402 type: ignore[import-not-found]
load_sibling as _load_sibling,
)
_review_fetch = _load_sibling("_review_fetch", "_review_fetch.py")
_ci_logs = _load_sibling("_ci_logs", "_ci_logs.py")
_pr_comments_cache = _load_sibling("_pr_comments_cache", "_pr_comments_cache.py")
_pr_diff = _load_sibling("_pr_diff", "_pr_diff.py")
_attempt_history = _load_sibling("_attempt_history", "_attempt_history.py")
_block_prompt = _load_sibling("_block_prompt", "_block_prompt.py")
_prefetch_section = _load_sibling("_prefetch_section", "_prefetch_section.py")
_bot_logins = _load_sibling("_bot_logins", "_bot_logins.py")
# Loaded at module scope (not deferred per-call) — the comment at
# the original call site warned about a circular-import risk that
# does not exist in practice (``implementer_validate`` does not
# import this module). Per-call ``_load_sibling`` was paying a
# dict lookup + cache hit on every prefetch; module-scope is
# cheaper and matches the load pattern used by ``_review_fetch``
# / ``_pr_diff`` above.
_implementer_validate = _load_sibling("implementer_validate", "implementer_validate.py")
_logger = logging.getLogger("implementer_prefetch")
# ─── Constants ──────────────────────────────────────────────────────────────
# Cap on the linked / Epic issue body characters embedded in the
# prompt. The worker can always paginate to the Forgejo issue itself
# if the body is enormous; the prompt budget exists so a 50 KB issue
# doesn't crowd out the diff.
DEFAULT_BODY_MAX_CHARS = 8000
# Max linked issues we will resolve. Mirrors the reviewer's setting.
DEFAULT_MAX_LINKED_ISSUES = 5
# Cap on per-comment body characters in the rendered prompt. Long
# comments truncate with the standard "[... N more lines truncated
# ...]" marker.
DEFAULT_COMMENT_MAX_CHARS = 4000
# Cap on how many comments are rendered verbatim into the worker
# prompt. The comment list is oldest-first (Forgejo creation order,
# preserved by _pr_comments_cache), so the renderer must take the
# LAST N — the worker needs the most recent attempt history to avoid
# repeating a just-failed approach, not the oldest. The section header
# still reports the true total so the worker knows more exist.
# (run-8 inspection: _build_comments_section took comments[:50] — the
# OLDEST 50 — so on a heavy PR like #30 with 1440 comments the worker
# was shown ancient history and missed every recent attempt.)
DEFAULT_MAX_PROMPT_COMMENTS = 50
# Cap on the CI failure log tail we embed when overall CI != success.
DEFAULT_CI_LOG_TAIL_CHARS = 4000
# Single source of truth for the dataclass's per-section completion
# flag attributes. Consumed by:
#
# - :func:`_init_completion_flags` (loops over the tuple to reset
# every flag rather than enumerating 10 explicit assignments).
# - The sentinel writer
# :mod:`_pr_context_sentinel.\_to_dict` (re-exports the
# tuple at module scope and loops to overlay every flag onto
# the projected dict; see that module's import block).
# - The worker-side reader
# :func:`tools.implementer_pr_context._project_field` (the
# ``metadata`` branch enumerates ``*_completed`` keys present
# in the payload — naturally tracking writer-side additions
# without sharing the tuple directly so the reader stays a
# stand-alone script with no dispatcher imports).
# - The fetcher-binding regression tests in
# ``tests/auto_agents/test_pr_context_sentinel.py`` (import this
# tuple to enumerate every flag a fetcher's preamble must reset
# — including any flag added in the future).
#
# Adding a new ``*_completed`` flag to
# :class:`ImplementerPrefetchResult` MUST also append the
# attribute name to this tuple — the binding tests will fail
# loudly if the new flag isn't initialised by the preamble.
# Conversely, a typo'd / stale entry in the tuple is caught by
# the belt-and-braces guard
# ``test_completion_flag_names_covers_every_dataclass_attribute``
# which asserts the tuple set equals the dataclass set
# (bidirectional drift check).
COMPLETION_FLAG_NAMES: tuple[str, ...] = (
"pr_details_completed",
"diff_completed",
"ci_status_completed",
"ci_detail_completed",
"ci_failure_logs_completed",
"pr_comments_completed",
"request_changes_reviews_completed",
"comment_reviews_completed",
"issue_body_completed",
"issue_comments_completed",
"linked_issues_completed",
"epic_completed",
)
# ─── Result dataclass ───────────────────────────────────────────────────────
@dataclass
class ImplementerPrefetchResult:
"""Aggregate result of one pre-dispatch fetch pass.
Mirrors the reviewer's ``ReviewContext`` carrier role: the
dispatcher stamps an instance onto ``item["_dispatcher_implementer_context"]``
so a future ``post_session_action`` (Phase 5b operator-status
comments) can read the same values without repeating any GETs.
All collection fields default to empty so a caller that only
populates the fields relevant to its work group still produces a
structurally-valid carrier.
Field semantics
---------------
- ``head_sha``: the freshly fetched ``pr_details.head.sha`` for
pr_fix / request_changes_pr; empty string for new_issue. Used
by Phase 3 pre-clone to ensure the diff and the worktree
check out the same commit.
- ``data_complete``: aggregate AND of every section's completion
signal. Diff truncation flips this to False even though the
diff body is still embedded. Linked-issue 404 does NOT flip
this (the placeholder is itself complete information; the
worker's quality-assurance procedure is expected to surface
a broken link as a code review concern).
- ``error_kinds``: free-form list of dispatcher-side fetch
failures (e.g. ``"pr_details:404"``, ``"ci_status:transient"``).
Empty list when every fetch succeeded. Used by the cycle
archive for triage; not embedded in the prompt.
"""
head_sha: str = ""
pr_details: dict[str, Any] | None = None
# True iff ``pr_details`` was fetched successfully. The
# ``description`` and ``title`` fields are derived from
# ``pr_details``; a legitimately-empty PR description with this
# flag True maps to "authoritative empty" (worker skips legacy
# GET) whereas an empty description with this flag False
# (transient fetch failure) maps to "fall through" via the
# worker-side three-case contract.
pr_details_completed: bool = True
diff_text: str = ""
diff_truncated: bool = False
diff_unavailable: bool = False
diff_info: dict[str, int] = field(default_factory=dict)
# True iff the diff was fetched successfully (truncation does
# not flip this — a truncated diff is still authoritative).
# Derives ``diff_completed`` in the sentinel projection.
diff_completed: bool = True
ci_status: dict[str, Any] | None = None
ci_status_completed: bool = True
ci_detail: list[dict[str, Any]] = field(default_factory=list)
ci_detail_completed: bool = True
# Per-job failing-CI log tails, pre-fetched once per head_sha via
# :mod:`_ci_logs` and stamped on the prompt as
# ``## Pre-fetched CI failure logs`` so the worker doesn't have
# to chase the failing assertion through file reads + a
# ~10-minute ``ci_run_local_gate`` invocation. ``None`` when CI
# is success (nothing to fetch).
ci_failure_logs: dict[str, Any] | None = None
ci_failure_logs_completed: bool = True
pr_comments: list[dict[str, Any]] = field(default_factory=list)
pr_comments_completed: bool = True
# Bounded subset of ``pr_comments`` to carry verbatim into the
# worker prompt + sentinel: the most-recent N plus every non-bot
# comment. The bot's older attempt comments are summarised by
# ``pr_comments_digest`` instead — together they keep a heavy PR's
# comment context bounded regardless of total volume (run-8
# findings R8-2 / R8-3). Empty for new_issue work.
pr_comments_view: list[dict[str, Any]] = field(default_factory=list)
# Deterministic attempt-history digest (see _attempt_history) —
# counts by tier / outcome, failing gates, last success. Empty
# dict for new_issue work or a PR with no comments.
pr_comments_digest: dict[str, Any] = field(default_factory=dict)
# Filter-summary stamp from :mod:`_pr_comments_cache` — counts
# bot status/claim/release/sentinel comments dropped at cache
# write time so the prompt section header can surface
# "N bot comments filtered" without storing the noise itself.
# Shape: ``{"count": N, "by_author": {login: N, ...}}`` or
# empty dict when the cache has no summary.
pr_comments_filter_summary: dict[str, Any] = field(default_factory=dict)
request_changes_reviews: list[dict[str, Any]] = field(default_factory=list)
request_changes_reviews_completed: bool = True
# Non-RC reviews (event=COMMENT or APPROVE) — the reviewer's
# advisory feedback that doesn't carry the "blocking" semantic
# but DOES carry substantive review prose + inline comments.
# Pre-R3.4 the implementer never saw these (request_changes_pr
# work group only included active RC; failing_ci_pr included
# no reviews at all) — the 2026-05-17 run-5 observation was
# that the reviewer's ``data_complete=False → COMMENT downgrade``
# path was producing reviews the implementer never saw, leaving
# the two pools working in independent silos on the same PR.
# Now populated for every work group whose ``include_reviews``
# flag is True so the implementer has the full reviewer record.
comment_reviews: list[dict[str, Any]] = field(default_factory=list)
comment_reviews_completed: bool = True
issue_body: str = ""
# True iff the issue body was fetched successfully (only
# meaningful for ``new_issue`` work; the field stays empty for
# ``pr_fix`` / ``request_changes_pr`` where the issue is not
# part of the work shape but the flag is still True because
# "nothing was attempted" is itself a complete answer).
issue_body_completed: bool = True
issue_comments: list[dict[str, Any]] = field(default_factory=list)
issue_comments_completed: bool = True
linked_issues: list[dict[str, Any]] = field(default_factory=list)
linked_issues_completed: bool = True
epic_issue: dict[str, Any] | None = None
epic_completed: bool = True
data_complete: bool = True
error_kinds: list[str] = field(default_factory=list)
# Cross-process block-store references. Populated after the
# per-section fetches have landed; each ref's ``key`` is what
# the worker passes to the ``block_store`` MCP's ``block_fetch``
# tool when it needs the original content of a section that an
# intermediate agent may have summarised. Empty when the block
# store is disabled or every registration failed (the inline
# sections in the prompt are still authoritative).
block_refs: list[Any] = field(default_factory=list)
# Map of carrier-attribute → operator-visible section name. Used by
# :func:`assess_prompt_completeness` so the loud-signal log/telemetry
# names sections in the same nomenclature the worker prompt's
# ``## Pre-fetched …`` headings use, not the Python attribute names.
_COMPLETION_FLAG_LABELS: dict[str, str] = {
"pr_details_completed": "pr_details",
"diff_completed": "diff",
"ci_status_completed": "ci_status",
"ci_detail_completed": "ci_detail",
"ci_failure_logs_completed": "ci_failure_logs",
"pr_comments_completed": "pr_comments",
"request_changes_reviews_completed": "request_changes_reviews",
"comment_reviews_completed": "comment_reviews",
"issue_body_completed": "issue_body",
"issue_comments_completed": "issue_comments",
"linked_issues_completed": "linked_issues",
"epic_completed": "epic_issue",
}
def assess_prompt_completeness(
result: "ImplementerPrefetchResult",
) -> dict[str, Any]:
"""W8 harvest (2026-05-15) — translate a prefetch result's
per-section completion flags into a loud, operator-readable
signal.
Returns ``{"degraded": bool, "missing_sections": list[str],
"error_kinds": list[str]}``. The aggregate ``data_complete`` flag
in :class:`ImplementerPrefetchResult` already encodes the same
AND — but as a single bool it loses the *which sections* signal
an operator needs to triage a degraded cycle. This helper keeps
the AND result AND the list of failing sections together.
Pure function; no I/O. The caller decides whether to log, write
telemetry, or post a status comment based on the result + an
operator-controlled flag.
"""
missing: list[str] = []
for attr, label in _COMPLETION_FLAG_LABELS.items():
if not bool(getattr(result, attr, True)):
missing.append(label)
# Diff truncation isn't a "completed=False" — the bytes are still
# in the carrier — but it IS a degraded signal an operator should
# see (a truncated diff makes some reviews and most full-context
# implementations unreliable). Surface it explicitly so the
# telemetry / log carries the same nuance ``data_complete`` does.
if getattr(result, "diff_truncated", False):
missing.append("diff_truncated")
return {
"degraded": bool(missing) or not bool(getattr(result, "data_complete", True)),
"missing_sections": missing,
"error_kinds": list(getattr(result, "error_kinds", []) or []),
}
# ─── Epic reference parsing ─────────────────────────────────────────────────
def parse_epic_reference(body: str | None) -> int | None:
"""Extract a single Epic / Parent issue reference from ``body``.
Mirrors :data:`implementer_validate.EPIC_REFERENCE_RE` — recognises
``Epic: #123``, ``Epic #123``, ``Parent: #123``, ``Parent #123``
(case-insensitive, on its own line). Returns the issue number or
``None`` if no reference is present.
The parsing is intentionally identical to the post-push validator
so a body that satisfies ``validate-pr-compliance`` is the same
body that resolves to an Epic body in the worker prompt. Adding
a future Epic-shape reference (e.g. ``Tracker:`` or
``Initiative:``) lands in both places by editing the shared
regex in :mod:`implementer_validate`.
"""
if not body:
return None
# Re-uses the regex from ``implementer_validate`` so a single
# edit (e.g. a future Tracker:/Initiative: spelling) propagates
# to both the post-push validator and this dispatcher-side
# prefetch.
match = _implementer_validate.EPIC_REFERENCE_RE.search(body)
if match is None:
return None
try:
return int(match.group("n"))
except (TypeError, ValueError, IndexError):
return None
def _fetch_issue(cfg: Any, number: int) -> tuple[dict[str, Any] | None, bool]:
"""GET a single issue. Returns ``(issue, completed)``. Mirrors
:func:`_review_fetch.fetch_pr_details` shape but for issues, since
:mod:`_review_fetch` exposes no public ``fetch_issue`` (only
linked-issue resolution which adds annotation we don't want here).
"""
if number <= 0:
return None, True
path = f"/repos/{cfg.owner}/{cfg.repo}/issues/{int(number)}"
try:
response = _review_fetch._claim_runtime.get(path, cfg)
except Exception as exc:
_logger.warning("issue-fetch #%s failed: %s", number, exc)
return None, False
status = int(response.get("status") or 0)
if status != 200:
# 4xx is a deterministic answer (issue not retrievable) — not
# a partial fetch. 5xx / transport failures already raised.
if 400 <= status < 500:
return None, True
return None, False
body = response.get("body")
if not isinstance(body, dict):
return None, False
return body, True
def _bounded_comment_view(
comments: list[dict[str, Any]],
max_recent: int,
*,
bot_logins: tuple[str, ...] | None = None,
) -> list[dict[str, Any]]:
"""The subset of ``comments`` to carry verbatim into the worker
prompt / sentinel.
Rule:
- Always keep the most-recent ``max_recent`` comments (gives the
worker the immediate attempt-history context regardless of who
authored them).
- Older than that: keep iff the comment is NOT authored by a known
bot AND is NOT a structured ``**Implementation Attempt**``
marker. The bot's own older claim/status/sentinel/attempt
comments are dropped (attempts are summarised in the digest;
other bot comments are routine noise). Real human / reviewer
comments survive.
Why author-based and not content-based: a content-only classifier
(``is_attempt_comment``) catches the structured attempt marker but
misses claim markers, status comments, sentinel posts, etc. — on
PR #30 those are ~1240 of the 1480 comments and would balloon the
"bounded" view to ~1252 (run-10 inspection). Author-based catches
everything the bots write while keeping every real human comment.
Order preserved (oldest-first); a comment that is both recent and
human is appended exactly once.
"""
bot_set = set(bot_logins) if bot_logins is not None else _bot_logins.bot_logins()
if len(comments) <= max_recent:
return list(comments)
cutoff = len(comments) - max_recent
out: list[dict[str, Any]] = []
for i, comment in enumerate(comments):
if i >= cutoff:
# Always keep the recent window, regardless of author.
out.append(comment)
continue
if not isinstance(comment, dict):
continue
# Older than the recent window: drop if bot-authored OR if it
# is a structured attempt marker (belt-and-braces — should be
# covered by the author check, but ``is_attempt_comment``
# protects against a future bot that posts under an unknown
# login).
if _attempt_history.is_bot_authored(comment, bot_set):
continue
if _attempt_history.is_attempt_comment(comment):
continue
out.append(comment)
return out
# ─── Per-work-group fetchers ────────────────────────────────────────────────
def fetch_pr_fix_context(cfg: Any, item: dict[str, Any]) -> ImplementerPrefetchResult:
"""Pre-dispatch fetch for the ``pr_fix`` work group (failing CI).
Pulls everything the worker needs to:
- Read the PR description and the failing-CI feedback (combined
status + per-check detail when overall != success).
- Read paginated issue-style comments on the PR.
- Read the linked-issue bodies (Closes / Fixes / Refs) so the
worker can confirm the change still aligns with intent.
- Read the parent Epic body when the PR or any linked issue
references one (``Epic: #N`` / ``Parent: #N``).
Does NOT fetch active reviews — the failing-CI bucket is
review-state-agnostic. Use :func:`fetch_request_changes_context`
for the bucket where reviews are load-bearing.
Sections this work_type does NOT attempt (``issue_body``,
``issue_comments``, ``request_changes_reviews``) carry
``*_completed=False`` in the result so the worker-side
three-case contract correctly signals "not attempted; fall
through to legacy GET" instead of "fetched and confirmed
empty" for those fields. See :func:`_init_completion_flags`
for the preamble + success-flip pattern.
"""
# R3.4 (2026-05-17): even failing_ci_pr work benefits from the
# reviewer's feedback. The flag is misnamed (now fetches ALL
# reviews, partitioned into RC + comment), but kept for back-compat.
return _fetch_pr_context(cfg, item, include_active_reviews=True)
def fetch_request_changes_context(
cfg: Any, item: dict[str, Any]
) -> ImplementerPrefetchResult:
"""Pre-dispatch fetch for the ``request_changes_pr`` work group.
Same shape as :func:`fetch_pr_fix_context` plus the list of
active (non-dismissed) ``REQUEST_CHANGES`` reviews with their
inline comments. The worker uses these to drive its fix loop —
every blocking concern must be addressed before the worker
pushes.
Sections this work_type does NOT attempt (``issue_body``,
``issue_comments``) carry ``*_completed=False`` in the result
— same rationale as :func:`fetch_pr_fix_context`.
"""
return _fetch_pr_context(cfg, item, include_active_reviews=True)
def _init_completion_flags(result: ImplementerPrefetchResult) -> None:
"""Reset every ``*_completed`` flag on ``result`` to ``False``
before the fetcher attempts any Forgejo round-trip.
This is the preamble half of the "preamble + success-flip"
pattern used by every per-work-group fetcher. After this runs:
- Sections the work_type **never attempts** (e.g. ``issue_body``
for ``pr_fix``) stay ``False`` through to the on-disk
sentinel. The worker reading ``--field issue_body`` then
gets empty stdout and falls through to its legacy GET —
the correct behaviour for "dispatcher didn't try; you go
ask Forgejo yourself".
- Sections the work_type **does attempt** (e.g. ``pr_details``
for ``pr_fix``) start ``False`` and the fetcher's success
path flips them ``True`` explicitly. This lets early-exits
(e.g. ``pr_details`` 5xx → return before diff / CI / comments
/ reviews) leave the not-yet-attempted flags ``False`` so
the worker correctly falls through for those fields rather
than being told a stale ``[]`` / ``null`` is authoritative.
Why ONE helper for both ``_fetch_pr_context`` and
``fetch_new_issue_context``: the two work-shape families
differ in WHICH flags get flipped ``True`` on success, but
they don't differ in the preamble — every flag starts
``False``. Per-work-type intent is documented on the
individual fetcher docstrings, not duplicated across two
identical helper bodies.
Loops over :data:`COMPLETION_FLAG_NAMES` rather than
enumerating attributes explicitly so adding a new
``*_completed`` flag to the dataclass automatically extends
the preamble — provided the contributor also adds the name
to the tuple. The fetcher-binding regression tests share the
same tuple, so any drift between the dataclass and the
preamble fails a test rather than silently shipping a
half-reset result.
"""
for name in COMPLETION_FLAG_NAMES:
setattr(result, name, False)
def _fetch_pr_context(
cfg: Any,
item: dict[str, Any],
*,
include_active_reviews: bool,
) -> ImplementerPrefetchResult:
"""Shared pr_fix / request_changes_pr fetcher. The
``include_active_reviews`` flag toggles the
:func:`_review_fetch.fetch_existing_reviews` call so the failing-
CI bucket doesn't pay a paginated review fetch it doesn't use.
Every Forgejo round-trip is best-effort — failures are recorded in
:attr:`ImplementerPrefetchResult.error_kinds` and the relevant
section's ``*_completed`` flag flips. Aggregate
:attr:`ImplementerPrefetchResult.data_complete` reflects the
AND of every section's completion signal so a future
post_session_action can short-circuit on partial context.
"""
result = ImplementerPrefetchResult()
# Preamble: every ``*_completed`` flag starts ``False`` BEFORE
# any potential early-exit (invalid PR number, dry-run, fetch
# failure). The flag drives the worker-side three-case
# contract: False → empty stdout → worker falls through to
# legacy GET; True → either ``[]\n`` / ``null\n`` (authoritative
# empty) or content. See :func:`_init_completion_flags`.
#
# ``pr_fix`` work NEVER attempts ``issue_body`` /
# ``issue_comments`` / ``request_changes_reviews``; those
# flags STAY False through to the sentinel.
# ``request_changes_pr`` additionally attempts
# ``request_changes_reviews`` and flips its flag below.
_init_completion_flags(result)
if cfg.dry_run:
return result
pr_number = int(item.get("number") or 0)
if pr_number <= 0:
result.data_complete = False
result.error_kinds.append("invalid-pr-number")
return result
# PR details — head_sha lives here, used by both Phase 2 diff
# fetch AND Phase 3 pre-clone to keep them aligned.
pr_details = _review_fetch.fetch_pr_details(cfg, pr_number)
result.pr_details = pr_details
if pr_details is None:
# ``pr_details_completed`` is already False (set by the
# preamble); leave it False. Any downstream flag (diff,
# ci_*, pr_comments) also stays False because we early-
# exit before reaching those fetches.
result.data_complete = False
result.error_kinds.append("pr_details:fetch-failed")
return result
result.pr_details_completed = True
head = pr_details.get("head") if isinstance(pr_details.get("head"), dict) else {}
result.head_sha = str(head.get("sha") or item.get("head_sha") or "")
# Diff
diff_text, diff_truncated, diff_error, diff_info = _pr_diff.fetch_pr_diff_detailed(
cfg, pr_number
)
result.diff_text = diff_text
result.diff_truncated = diff_truncated
result.diff_info = diff_info
if not diff_text:
result.diff_unavailable = True
# ``diff_completed`` is already False (preamble); leave it.
result.data_complete = False
result.error_kinds.append(f"diff:{diff_error or 'no-diff-returned'}")
else:
# Truncation does NOT flip ``diff_completed`` — a truncated
# diff is still authoritative (just hard-capped). It DOES
# flip ``data_complete`` so the cycle archive flags partial
# context.
result.diff_completed = True
if diff_truncated:
result.data_complete = False
# CI combined status
ci_status = _review_fetch.fetch_ci_status(cfg, result.head_sha)
result.ci_status = ci_status
if ci_status is None:
# ``ci_status_completed`` is already False (preamble).
result.data_complete = False
result.error_kinds.append("ci_status:fetch-failed")
else:
result.ci_status_completed = True
# Per-check detail (only when overall != success — the per-check
# endpoint is verbose and would burn quota for green PRs).
if isinstance(ci_status, dict):
overall_state = ci_status.get("state")
if overall_state == "success":
# Green PR: no failing-check detail to fetch. The empty
# ``ci_detail`` list IS authoritative; flip the flag
# True so the worker reads "0 failing checks" rather
# than "fetch failed; retry".
result.ci_detail_completed = True
elif overall_state:
ci_detail, ci_detail_completed = _review_fetch.fetch_ci_check_detail(
cfg, result.head_sha
)
result.ci_detail = ci_detail
result.ci_detail_completed = ci_detail_completed
if not ci_detail_completed:
result.data_complete = False
result.error_kinds.append("ci_detail:partial")
# Pre-fetch failing-job log tails so the worker sees the
# EXACT failing assertion without burning turns on file
# reads + a 10-minute local gate run. Cached per-head_sha
# so the second cycle on the same SHA is free. Failure
# to fetch must NEVER break prompt assembly — WARN and
# carry on; the worker still has ``ci_detail`` +
# ``target_url`` to follow manually.
try:
ci_failure_logs, ci_failure_logs_completed = (
_ci_logs.fetch_pr_failure_logs(
cfg,
result.head_sha,
ci_detail=ci_detail,
)
)
result.ci_failure_logs = ci_failure_logs
result.ci_failure_logs_completed = ci_failure_logs_completed
if not ci_failure_logs_completed:
result.data_complete = False
result.error_kinds.append("ci_failure_logs:partial")
except (OSError, ValueError) as exc:
# Transport / JSON-parse failure — substrate is
# best-effort. Mark partial + carry on; programmer
# errors propagate to the test suite.
result.ci_failure_logs = None
result.ci_failure_logs_completed = False
result.data_complete = False
result.error_kinds.append(f"ci_failure_logs:{type(exc).__name__}")
# PR comments (issue-style) — use the persistent cache so we
# delta-fetch only new comments instead of paginating from
# page 1 every cycle. Falls back to a full fetch when the
# cache is missing, stale, or disabled.
pr_comments, pr_comments_completed = _pr_comments_cache.get_pr_comments(
cfg, pr_number
)
result.pr_comments = pr_comments
result.pr_comments_completed = pr_comments_completed
# Bounded carry + digest derived from the full list. The full list
# stays on ``result.pr_comments`` (and in the cache); the prompt /
# sentinel consumers read ``pr_comments_view`` + ``pr_comments_digest``.
result.pr_comments_view = _bounded_comment_view(
pr_comments, DEFAULT_MAX_PROMPT_COMMENTS
)
result.pr_comments_digest = _attempt_history.summarize_attempt_history(
pr_comments
).to_dict()
# Filter-summary stamp: how many bot status/claim/release/sentinel
# comments the cache dropped at write time. Empty dict when the
# cache module has no summary (older cache file pre-dating the
# filter feature, OR filter disabled via env). Best-effort —
# surfaces in the prompt section header for operator visibility.
summary = _pr_comments_cache.get_filter_summary(pr_number)
if isinstance(summary, dict):
result.pr_comments_filter_summary = summary
if not pr_comments_completed:
result.data_complete = False
result.error_kinds.append("pr_comments:partial")
# Reviews — fetched once, partitioned into:
# - ``request_changes_reviews``: active REQUEST_CHANGES (blocking,
# must be addressed before push)
# - ``comment_reviews``: every other non-dismissed review
# (COMMENT / APPROVE) — advisory feedback the implementer
# should consider but is not bound to address.
# Both populated whenever ``include_active_reviews`` is True; the
# flag is now misnamed but kept for backward-compat (R3.4,
# 2026-05-17 — pre-this the failing_ci_pr path never fetched
# reviews at all, leaving the implementer blind to the
# reviewer's COMMENT-only feedback the dispatcher's
# ``data_complete=False`` downgrade produces in low-context
# cycles).
if include_active_reviews:
reviews, reviews_completed = _review_fetch.fetch_existing_reviews(
cfg, pr_number
)
active_rc: list[dict[str, Any]] = []
other: list[dict[str, Any]] = []
for r in reviews:
if not isinstance(r, dict):
continue
if _is_active_request_changes_review(r):
active_rc.append(r)
continue
# Drop dismissed reviews; keep COMMENT + APPROVE
# (and any other non-dismissed event) so the implementer
# sees the full reviewer record on this PR. Dismissal
# is signalled via the ``dismissed`` boolean field, not
# via the ``state`` enum — matches the predicate
# ``_is_active_request_changes_review`` uses below.
if bool(r.get("dismissed")):
continue
other.append(r)
result.request_changes_reviews = active_rc
result.request_changes_reviews_completed = reviews_completed
result.comment_reviews = other
result.comment_reviews_completed = reviews_completed
if not reviews_completed:
result.data_complete = False
result.error_kinds.append("reviews:partial")
# Linked issues + Epic
body_for_links = pr_details.get("body") if isinstance(pr_details, dict) else None
_resolve_links_and_epic(cfg, result, body_for_links)
# Register every section as a block in the cross-process block
# store. Best-effort; the prompt's inline content is still the
# primary source if any registration fails.
result.block_refs.extend(
_prefetch_section.register_sections(
_implementer_sections(result, diff_text=result.diff_text),
pr_number=pr_number,
head_sha=result.head_sha,
)
)
return result
def _is_active_request_changes_review(review: dict[str, Any]) -> bool:
"""True iff ``review`` is an active (non-stale, non-dismissed)
REQUEST_CHANGES verdict. Mirrors
:func:`_review_fetch.count_active_request_changes`'s predicate
but exposed here so we can filter the per-review list, not just
count it."""
if not isinstance(review, dict):
return False
state = str(review.get("state") or "").upper()
if state != "REQUEST_CHANGES":
return False
if bool(review.get("dismissed")):
return False
return not bool(review.get("stale"))
def fetch_new_issue_context(
cfg: Any, item: dict[str, Any]
) -> ImplementerPrefetchResult:
"""Pre-dispatch fetch for the ``new_issue`` work group.
No PR exists yet — there is no diff, no CI, no reviews. Pulls:
- The issue body + paginated comments (the worker reads both to
derive the implementation contract).
- Any linked issue bodies referenced from the issue body (a
typical pattern: parent epic links via ``Parent: #N`` and
sibling issues via ``See: #N``).
- The Epic body when the issue references one.
head_sha stays empty — the worker will create the branch itself.
Sections this work_type does NOT attempt (``pr_details`` and
everything derived from it: description, title, diff, CI,
pr_comments, request_changes_reviews) carry ``*_completed=False``
in the result so the worker-side three-case contract correctly
signals "not attempted; fall through to legacy GET" instead of
"fetched and confirmed empty" for those fields. See
:func:`_init_completion_flags` for the preamble + success-flip
pattern.
"""
result = ImplementerPrefetchResult()
# Preamble: every ``*_completed`` flag starts ``False`` BEFORE
# any potential early-exit (invalid issue number, dry-run,
# fetch failure). See :func:`_init_completion_flags`.
#
# ``new_issue`` work NEVER attempts PR-shaped sections
# (``pr_details`` and everything derived from it: ``title``,
# ``description``, ``diff``, ``ci_*``, ``pr_comments``,
# ``request_changes_reviews``); those flags STAY False
# through to the sentinel. The fetcher below flips
# ``issue_body`` / ``issue_comments`` / ``linked_issues`` /
# ``epic`` to True on successful fetch.
_init_completion_flags(result)
if cfg.dry_run:
return result
issue_number = int(item.get("number") or 0)
if issue_number <= 0:
result.data_complete = False
result.error_kinds.append("invalid-issue-number")
return result
issue, issue_completed = _fetch_issue(cfg, issue_number)
if issue is None:
# ``issue_body_completed`` is already False (preamble).
result.data_complete = False
result.error_kinds.append(
"issue:fetch-failed" if not issue_completed else "issue:not-found"
)
return result
body = issue.get("body") if isinstance(issue, dict) else None
result.issue_body = str(body or "")
# Successful fetch — even if ``body`` is empty, the empty value
# is authoritative.
result.issue_body_completed = True
# Issue comments — same paginated path as PR issue-style comments
# because Forgejo treats issues and PRs as one conversation
# endpoint under ``/issues/{N}/comments``. Use the persistent
# cache (C5, 2026-05-13) so we only fetch the delta.
issue_comments, issue_comments_completed = _pr_comments_cache.get_pr_comments(
cfg, issue_number
)
result.issue_comments = issue_comments
result.issue_comments_completed = issue_comments_completed
if not issue_comments_completed:
result.data_complete = False
result.error_kinds.append("issue_comments:partial")
_resolve_links_and_epic(cfg, result, result.issue_body)
# Register every section as a block in the cross-process block
# store. Best-effort; the prompt's inline content is still the
# primary source if any registration fails.
result.block_refs.extend(
_prefetch_section.register_sections(
_implementer_sections(result, diff_text=None),
pr_number=None,
head_sha=result.head_sha,
)
)
return result
def _resolve_links_and_epic(
cfg: Any,
result: ImplementerPrefetchResult,
body_for_links: str | None,
) -> None:
"""Resolve ``Closes #N`` / ``Fixes #N`` / ``Refs #N`` references
in ``body_for_links`` plus the optional ``Epic: #N`` /
``Parent: #N`` reference, mutating ``result`` in-place.
We treat the Epic and the linked-issue list as separate buckets
in the prompt:
- The Epic gets its own section so the worker has the parent
tracking issue's intent (acceptance criteria, parent-Epic
milestone) front-and-center.
- The linked issues get a list section so cross-reference
bodies (e.g. an upstream bug report the PR fixes) are
available without poisoning the Epic's role.
De-duplication: if the same issue number appears as both a
linked-issue reference AND an Epic reference, we render it as
Epic only (the role is more specific). This avoids a confusing
prompt where the same body appears twice.
"""
linked_issues, linked_completed = _review_fetch.fetch_linked_issues(
cfg, body_for_links, max_issues=DEFAULT_MAX_LINKED_ISSUES
)
epic_number = parse_epic_reference(body_for_links)
if epic_number is None:
# No Epic referenced at all — authoritative "no Epic"
# answer. Flip the flag True so the worker's
# ``--field epic`` read emits ``null\n`` (authoritative)
# rather than empty stdout (fall through).
result.epic_completed = True
else:
# The linked-issue resolver may already have fetched the same
# number via a `Closes #N` reference. Rather than refetching,
# promote it from the linked list to the epic slot.
promoted: dict[str, Any] | None = None
remaining: list[dict[str, Any]] = []
for entry in linked_issues:
if isinstance(entry, dict) and int(entry.get("number") or 0) == epic_number:
promoted = entry
else:
remaining.append(entry)
if promoted is not None:
result.epic_issue = promoted
linked_issues = remaining
result.epic_completed = True
else:
epic_body, epic_completed = _fetch_issue(cfg, epic_number)
result.epic_issue = epic_body
if not epic_completed:
# Already False from preamble; leave it.
result.data_complete = False
result.error_kinds.append("epic:fetch-failed")
elif epic_body is None:
# 4xx — Epic referenced but not retrievable. Don't
# poison data_complete; this is a worker-side data-
# quality concern, like the reviewer's broken-link
# path. The downstream prompt section will surface
# the broken reference explicitly. Tag this in
# error_kinds so an operator scanning Phase 4
# telemetry can distinguish "Epic linked but not
# accessible" (likely permissions / deleted) from
# "Epic body fetched fine".
result.epic_completed = True
result.error_kinds.append("epic:not-found")
else:
# Epic body fetched successfully.
result.epic_completed = True
result.linked_issues = linked_issues
result.linked_issues_completed = linked_completed
if not linked_completed:
result.data_complete = False
result.error_kinds.append("linked_issues:partial")
def _implementer_sections(
result: ImplementerPrefetchResult,
*,
diff_text: str | None,
) -> list[Any]:
"""Build the implementer's prefetched-section registry. Empty
entries are filtered by :func:`_prefetch_section.register_sections`.
"""
Section = _prefetch_section.PrefetchSection
json = _block_prompt.serialise_json_block
n_failing = 0
if isinstance(result.ci_failure_logs, dict):
failing = result.ci_failure_logs.get("failing_jobs")
if isinstance(failing, list):
n_failing = len(failing)
return [
Section(
"diff",
diff_text or "",
f"PR diff (head_sha={(result.head_sha or '')[:12]})",
),
Section(
"pr_details",
json(result.pr_details) if isinstance(result.pr_details, dict) else "",
"PR metadata (title, body, head, base)",
),
Section(
"ci_status",
json(result.ci_status) if isinstance(result.ci_status, dict) else "",
"combined CI status",
),
Section(
"ci_detail",
json(result.ci_detail) if result.ci_detail else "",
f"{len(result.ci_detail)} per-check CI status rows",
),
Section(
"ci_failure_logs",
json(result.ci_failure_logs) if result.ci_failure_logs else "",
f"{n_failing} failing-CI job log tails",
),
Section(
"pr_comments",
json(result.pr_comments) if result.pr_comments else "",
f"{len(result.pr_comments)} PR conversation comments (full list)",
),
Section(
"request_changes_reviews",
(
json(result.request_changes_reviews)
if result.request_changes_reviews
else ""
),
f"{len(result.request_changes_reviews)} active REQUEST_CHANGES reviews",
),
Section(
"issue_body",
result.issue_body or "",
"issue body (raw text)",
),
Section(
"issue_comments",
json(result.issue_comments) if result.issue_comments else "",
f"{len(result.issue_comments)} issue comments",
),
Section(
"linked_issues",
json(result.linked_issues) if result.linked_issues else "",
f"{len(result.linked_issues)} linked issues",
),
Section(
"epic",
json(result.epic_issue) if result.epic_issue else "",
"parent Epic issue",
),
]
__all__ = (
"COMPLETION_FLAG_NAMES",
"DEFAULT_BODY_MAX_CHARS",
"DEFAULT_CI_LOG_TAIL_CHARS",
"DEFAULT_COMMENT_MAX_CHARS",
"DEFAULT_MAX_LINKED_ISSUES",
"DEFAULT_MAX_PROMPT_COMMENTS",
"ImplementerPrefetchResult",
"fetch_new_issue_context",
"fetch_pr_fix_context",
"fetch_request_changes_context",
"parse_epic_reference",
)