cleveragents-core/tools/_implementer_prefetch.py

"""Pre-dispatch Forgejo fetches for the implementer dispatcher.

Mirrors :func:`_review_prompt.fetch_review_context` on the implementer
side. The implementer worker has historically issued every Forgejo
GET inside its LLM session (see the PR #30 post-mortem on 2026-05-08
where the estimator burned 10 min on five unauthed ``webfetch`` calls
before any code was written). This module pulls every read into a
single deterministic pre-dispatch pass so the worker's prompt is
self-contained — no in-session HTTP, no in-session pagination, no
tool-budget creep on ``webfetch``.

Three pre-fetch shapes, one per work group:

- ``pr_fix``        — failing-CI PR. Fetches PR details + diff +
  CI combined status + per-check status detail (when overall != success)
  + paginated PR comments + linked-issue bodies + an Epic body when
  the PR / linked issues reference one.
- ``request_changes_pr`` — PR with at least one active
  ``REQUEST_CHANGES`` review the author has not yet addressed.
  Same as ``pr_fix`` plus the active reviews list (with their inline
  comments) so the worker can read each blocking concern without
  paging through the API itself.
- ``new_issue``     — implementing a fresh issue. No PR exists yet,
  so we fetch the issue body + paginated issue comments + linked
  bodies + Epic body only.

The Forgejo HTTP primitives come from :mod:`_review_fetch` — the
existing fetcher set is fully generic (see the per-function docstrings;
none of them are review-specific in behaviour, only in original
caller). Reusing them here closes the "implementer dispatcher does
zero pre-fetch" gap without forking a parallel ``_implementer_fetch``
module that would need its own ``_api_get_paginated`` retry / partial-
pagination handling.

Each fetcher swallows its own failures and returns ``None`` / ``[]``
so this module never raises. Sections downstream
(:mod:`_implementer_prompt`) handle the missing-data case explicitly
by emitting an "X unavailable" stanza instead of an empty one — the
worker is told what was attempted so it can decide whether to fall
back to its in-session HTTP rules.

The dispatcher's overall ``data_complete`` flag is the AND of every
section's ``completed`` signal. Today's implementer worker does not
short-circuit on incomplete data the way the reviewer dispatcher
defensively downgrades APPROVED → COMMENT, but the flag is still
emitted so a future post-session action (e.g. operator-status
fingerprinting per Phase 5b) can record "this cycle was dispatched
against a partial context" for triage.
"""
from __future__ import annotations

import logging
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

_TOOLS_DIR = str(Path(__file__).resolve().parent)
if _TOOLS_DIR not in sys.path:
    sys.path.insert(0, _TOOLS_DIR)
from _loader import (  # noqa: E402  type: ignore[import-not-found]
    load_sibling as _load_sibling,
)

_review_fetch = _load_sibling("_review_fetch", "_review_fetch.py")
_pr_diff = _load_sibling("_pr_diff", "_pr_diff.py")
# Loaded at module scope (not deferred per-call) — the comment at
# the original call site warned about a circular-import risk that
# does not exist in practice (``implementer_validate`` does not
# import this module). Per-call ``_load_sibling`` was paying a
# dict lookup + cache hit on every prefetch; module-scope is
# cheaper and matches the load pattern used by ``_review_fetch``
# / ``_pr_diff`` above.
_implementer_validate = _load_sibling(
    "implementer_validate", "implementer_validate.py"
)


_logger = logging.getLogger("implementer_prefetch")


# ─── Constants ──────────────────────────────────────────────────────────────


# Cap on the linked / Epic issue body characters embedded in the
# prompt. The worker can always paginate to the Forgejo issue itself
# if the body is enormous; the prompt budget exists so a 50 KB issue
# doesn't crowd out the diff.
DEFAULT_BODY_MAX_CHARS = 8000

# Max linked issues we will resolve. Mirrors the reviewer's setting.
DEFAULT_MAX_LINKED_ISSUES = 5

# Cap on per-comment body characters in the rendered prompt. Long
# comments truncate with the standard "[... N more lines truncated
# ...]" marker.
DEFAULT_COMMENT_MAX_CHARS = 4000

# Cap on the CI failure log tail we embed when overall CI != success.
DEFAULT_CI_LOG_TAIL_CHARS = 4000


# ─── Result dataclass ───────────────────────────────────────────────────────


@dataclass
class ImplementerPrefetchResult:
    """Aggregate result of one pre-dispatch fetch pass.

    Mirrors the reviewer's ``ReviewContext`` carrier role: the
    dispatcher stamps an instance onto ``item["_dispatcher_implementer_context"]``
    so a future ``post_session_action`` (Phase 5b operator-status
    comments) can read the same values without repeating any GETs.

    All collection fields default to empty so a caller that only
    populates the fields relevant to its work group still produces a
    structurally-valid carrier.

    Field semantics
    ---------------

    - ``head_sha``: the freshly fetched ``pr_details.head.sha`` for
      pr_fix / request_changes_pr; empty string for new_issue. Used
      by Phase 3 pre-clone to ensure the diff and the worktree
      check out the same commit.
    - ``data_complete``: aggregate AND of every section's completion
      signal. Diff truncation flips this to False even though the
      diff body is still embedded. Linked-issue 404 does NOT flip
      this (the placeholder is itself complete information; the
      worker's quality-assurance procedure is expected to surface
      a broken link as a code review concern).
    - ``error_kinds``: free-form list of dispatcher-side fetch
      failures (e.g. ``"pr_details:404"``, ``"ci_status:transient"``).
      Empty list when every fetch succeeded. Used by the cycle
      archive for triage; not embedded in the prompt.
    """

    head_sha: str = ""
    pr_details: dict[str, Any] | None = None
    diff_text: str = ""
    diff_truncated: bool = False
    diff_unavailable: bool = False
    diff_info: dict[str, int] = field(default_factory=dict)
    ci_status: dict[str, Any] | None = None
    ci_status_completed: bool = True
    ci_detail: list[dict[str, Any]] = field(default_factory=list)
    ci_detail_completed: bool = True
    pr_comments: list[dict[str, Any]] = field(default_factory=list)
    pr_comments_completed: bool = True
    request_changes_reviews: list[dict[str, Any]] = field(default_factory=list)
    request_changes_reviews_completed: bool = True
    issue_body: str = ""
    issue_comments: list[dict[str, Any]] = field(default_factory=list)
    issue_comments_completed: bool = True
    linked_issues: list[dict[str, Any]] = field(default_factory=list)
    linked_issues_completed: bool = True
    epic_issue: dict[str, Any] | None = None
    epic_completed: bool = True
    data_complete: bool = True
    error_kinds: list[str] = field(default_factory=list)


# ─── Epic reference parsing ─────────────────────────────────────────────────


def parse_epic_reference(body: str | None) -> int | None:
    """Extract a single Epic / Parent issue reference from ``body``.

    Mirrors :data:`implementer_validate.EPIC_REFERENCE_RE` — recognises
    ``Epic: #123``, ``Epic #123``, ``Parent: #123``, ``Parent #123``
    (case-insensitive, on its own line). Returns the issue number or
    ``None`` if no reference is present.

    The parsing is intentionally identical to the post-push validator
    so a body that satisfies ``validate-pr-compliance`` is the same
    body that resolves to an Epic body in the worker prompt. Adding
    a future Epic-shape reference (e.g. ``Tracker:`` or
    ``Initiative:``) lands in both places by editing the shared
    regex in :mod:`implementer_validate`.
    """
    if not body:
        return None
    # Re-uses the regex from ``implementer_validate`` so a single
    # edit (e.g. a future Tracker:/Initiative: spelling) propagates
    # to both the post-push validator and this dispatcher-side
    # prefetch.
    match = _implementer_validate.EPIC_REFERENCE_RE.search(body)
    if match is None:
        return None
    try:
        return int(match.group("n"))
    except (TypeError, ValueError, IndexError):
        return None


def _fetch_issue(cfg: Any, number: int) -> tuple[dict[str, Any] | None, bool]:
    """GET a single issue. Returns ``(issue, completed)``. Mirrors
    :func:`_review_fetch.fetch_pr_details` shape but for issues, since
    :mod:`_review_fetch` exposes no public ``fetch_issue`` (only
    linked-issue resolution which adds annotation we don't want here).
    """
    if number <= 0:
        return None, True
    path = f"/repos/{cfg.owner}/{cfg.repo}/issues/{int(number)}"
    try:
        response = _review_fetch._claim_runtime.get(path, cfg)
    except Exception as exc:
        _logger.warning("issue-fetch #%s failed: %s", number, exc)
        return None, False
    status = int(response.get("status") or 0)
    if status != 200:
        # 4xx is a deterministic answer (issue not retrievable) — not
        # a partial fetch. 5xx / transport failures already raised.
        if 400 <= status < 500:
            return None, True
        return None, False
    body = response.get("body")
    if not isinstance(body, dict):
        return None, False
    return body, True


# ─── Per-work-group fetchers ────────────────────────────────────────────────


def fetch_pr_fix_context(
    cfg: Any, item: dict[str, Any]
) -> ImplementerPrefetchResult:
    """Pre-dispatch fetch for the ``pr_fix`` work group (failing CI).

    Pulls everything the worker needs to:

    - Read the PR description and the failing-CI feedback (combined
      status + per-check detail when overall != success).
    - Read paginated issue-style comments on the PR.
    - Read the linked-issue bodies (Closes / Fixes / Refs) so the
      worker can confirm the change still aligns with intent.
    - Read the parent Epic body when the PR or any linked issue
      references one (``Epic: #N`` / ``Parent: #N``).

    Does NOT fetch active reviews — the failing-CI bucket is
    review-state-agnostic. Use :func:`fetch_request_changes_context`
    for the bucket where reviews are load-bearing.
    """
    return _fetch_pr_context(cfg, item, include_active_reviews=False)


def fetch_request_changes_context(
    cfg: Any, item: dict[str, Any]
) -> ImplementerPrefetchResult:
    """Pre-dispatch fetch for the ``request_changes_pr`` work group.

    Same shape as :func:`fetch_pr_fix_context` plus the list of
    active (non-dismissed) ``REQUEST_CHANGES`` reviews with their
    inline comments. The worker uses these to drive its fix loop —
    every blocking concern must be addressed before the worker
    pushes.
    """
    return _fetch_pr_context(cfg, item, include_active_reviews=True)


def _fetch_pr_context(
    cfg: Any,
    item: dict[str, Any],
    *,
    include_active_reviews: bool,
) -> ImplementerPrefetchResult:
    """Shared pr_fix / request_changes_pr fetcher. The
    ``include_active_reviews`` flag toggles the
    :func:`_review_fetch.fetch_existing_reviews` call so the failing-
    CI bucket doesn't pay a paginated review fetch it doesn't use.

    Every Forgejo round-trip is best-effort — failures are recorded in
    :attr:`ImplementerPrefetchResult.error_kinds` and the relevant
    section's ``*_completed`` flag flips. Aggregate
    :attr:`ImplementerPrefetchResult.data_complete` reflects the
    AND of every section's completion signal so a future
    post_session_action can short-circuit on partial context.
    """
    result = ImplementerPrefetchResult()
    if cfg.dry_run:
        return result
    pr_number = int(item.get("number") or 0)
    if pr_number <= 0:
        result.data_complete = False
        result.error_kinds.append("invalid-pr-number")
        return result

    # PR details — head_sha lives here, used by both Phase 2 diff
    # fetch AND Phase 3 pre-clone to keep them aligned.
    pr_details = _review_fetch.fetch_pr_details(cfg, pr_number)
    result.pr_details = pr_details
    if pr_details is None:
        result.data_complete = False
        result.error_kinds.append("pr_details:fetch-failed")
        return result
    head = pr_details.get("head") if isinstance(pr_details.get("head"), dict) else {}
    result.head_sha = str(head.get("sha") or item.get("head_sha") or "")

    # Diff
    diff_text, diff_truncated, diff_error, diff_info = (
        _pr_diff.fetch_pr_diff_detailed(cfg, pr_number)
    )
    result.diff_text = diff_text
    result.diff_truncated = diff_truncated
    result.diff_info = diff_info
    if not diff_text:
        result.diff_unavailable = True
        result.data_complete = False
        result.error_kinds.append(
            f"diff:{diff_error or 'no-diff-returned'}"
        )
    elif diff_truncated:
        result.data_complete = False

    # CI combined status
    ci_status = _review_fetch.fetch_ci_status(cfg, result.head_sha)
    result.ci_status = ci_status
    if ci_status is None:
        result.ci_status_completed = False
        result.data_complete = False
        result.error_kinds.append("ci_status:fetch-failed")

    # Per-check detail (only when overall != success — the per-check
    # endpoint is verbose and would burn quota for green PRs).
    if isinstance(ci_status, dict):
        overall_state = ci_status.get("state")
        if overall_state and overall_state != "success":
            ci_detail, ci_detail_completed = _review_fetch.fetch_ci_check_detail(
                cfg, result.head_sha
            )
            result.ci_detail = ci_detail
            result.ci_detail_completed = ci_detail_completed
            if not ci_detail_completed:
                result.data_complete = False
                result.error_kinds.append("ci_detail:partial")

    # PR comments (issue-style)
    pr_comments, pr_comments_completed = _review_fetch.fetch_pr_comments(
        cfg, pr_number
    )
    result.pr_comments = pr_comments
    result.pr_comments_completed = pr_comments_completed
    if not pr_comments_completed:
        result.data_complete = False
        result.error_kinds.append("pr_comments:partial")

    # Active REQUEST_CHANGES reviews (request_changes_pr only)
    if include_active_reviews:
        reviews, reviews_completed = _review_fetch.fetch_existing_reviews(
            cfg, pr_number
        )
        # Filter to active REQUEST_CHANGES — those are the blocking
        # ones the worker must address. Approved / commented / dismissed
        # reviews don't drive the fix loop.
        active = [
            r for r in reviews if _is_active_request_changes_review(r)
        ]
        result.request_changes_reviews = active
        result.request_changes_reviews_completed = reviews_completed
        if not reviews_completed:
            result.data_complete = False
            result.error_kinds.append("reviews:partial")

    # Linked issues + Epic
    body_for_links = pr_details.get("body") if isinstance(pr_details, dict) else None
    _resolve_links_and_epic(cfg, result, body_for_links)

    return result


def _is_active_request_changes_review(review: dict[str, Any]) -> bool:
    """True iff ``review`` is an active (non-stale, non-dismissed)
    REQUEST_CHANGES verdict. Mirrors
    :func:`_review_fetch.count_active_request_changes`'s predicate
    but exposed here so we can filter the per-review list, not just
    count it."""
    if not isinstance(review, dict):
        return False
    state = str(review.get("state") or "").upper()
    if state != "REQUEST_CHANGES":
        return False
    if bool(review.get("dismissed")):
        return False
    return not bool(review.get("stale"))


def fetch_new_issue_context(
    cfg: Any, item: dict[str, Any]
) -> ImplementerPrefetchResult:
    """Pre-dispatch fetch for the ``new_issue`` work group.

    No PR exists yet — there is no diff, no CI, no reviews. Pulls:

    - The issue body + paginated comments (the worker reads both to
      derive the implementation contract).
    - Any linked issue bodies referenced from the issue body (a
      typical pattern: parent epic links via ``Parent: #N`` and
      sibling issues via ``See: #N``).
    - The Epic body when the issue references one.

    head_sha stays empty — the worker will create the branch itself.
    """
    result = ImplementerPrefetchResult()
    if cfg.dry_run:
        return result
    issue_number = int(item.get("number") or 0)
    if issue_number <= 0:
        result.data_complete = False
        result.error_kinds.append("invalid-issue-number")
        return result

    issue, issue_completed = _fetch_issue(cfg, issue_number)
    if issue is None:
        result.data_complete = False
        result.error_kinds.append(
            "issue:fetch-failed" if not issue_completed else "issue:not-found"
        )
        return result
    body = issue.get("body") if isinstance(issue, dict) else None
    result.issue_body = str(body or "")

    # Issue comments — same paginated path as PR issue-style comments
    # because Forgejo treats issues and PRs as one conversation
    # endpoint under ``/issues/{N}/comments``.
    issue_comments, issue_comments_completed = _review_fetch.fetch_pr_comments(
        cfg, issue_number
    )
    result.issue_comments = issue_comments
    result.issue_comments_completed = issue_comments_completed
    if not issue_comments_completed:
        result.data_complete = False
        result.error_kinds.append("issue_comments:partial")

    _resolve_links_and_epic(cfg, result, result.issue_body)
    return result


def _resolve_links_and_epic(
    cfg: Any,
    result: ImplementerPrefetchResult,
    body_for_links: str | None,
) -> None:
    """Resolve ``Closes #N`` / ``Fixes #N`` / ``Refs #N`` references
    in ``body_for_links`` plus the optional ``Epic: #N`` /
    ``Parent: #N`` reference, mutating ``result`` in-place.

    We treat the Epic and the linked-issue list as separate buckets
    in the prompt:

    - The Epic gets its own section so the worker has the parent
      tracking issue's intent (acceptance criteria, parent-Epic
      milestone) front-and-center.
    - The linked issues get a list section so cross-reference
      bodies (e.g. an upstream bug report the PR fixes) are
      available without poisoning the Epic's role.

    De-duplication: if the same issue number appears as both a
    linked-issue reference AND an Epic reference, we render it as
    Epic only (the role is more specific). This avoids a confusing
    prompt where the same body appears twice.
    """
    linked_issues, linked_completed = _review_fetch.fetch_linked_issues(
        cfg, body_for_links, max_issues=DEFAULT_MAX_LINKED_ISSUES
    )
    epic_number = parse_epic_reference(body_for_links)
    if epic_number is not None:
        # The linked-issue resolver may already have fetched the same
        # number via a `Closes #N` reference. Rather than refetching,
        # promote it from the linked list to the epic slot.
        promoted: dict[str, Any] | None = None
        remaining: list[dict[str, Any]] = []
        for entry in linked_issues:
            if isinstance(entry, dict) and int(entry.get("number") or 0) == epic_number:
                promoted = entry
            else:
                remaining.append(entry)
        if promoted is not None:
            result.epic_issue = promoted
            linked_issues = remaining
        else:
            epic_body, epic_completed = _fetch_issue(cfg, epic_number)
            result.epic_issue = epic_body
            if not epic_completed:
                result.epic_completed = False
                result.data_complete = False
                result.error_kinds.append("epic:fetch-failed")
            elif epic_body is None:
                # 4xx — Epic referenced but not retrievable. Don't
                # poison data_complete; this is a worker-side data-
                # quality concern, like the reviewer's broken-link
                # path. The downstream prompt section will surface
                # the broken reference explicitly. Tag this in
                # error_kinds so an operator scanning Phase 4
                # telemetry can distinguish "Epic linked but not
                # accessible" (likely permissions / deleted) from
                # "Epic body fetched fine".
                result.epic_completed = True
                result.error_kinds.append("epic:not-found")
    result.linked_issues = linked_issues
    result.linked_issues_completed = linked_completed
    if not linked_completed:
        result.data_complete = False
        result.error_kinds.append("linked_issues:partial")


__all__ = (
    "DEFAULT_BODY_MAX_CHARS",
    "DEFAULT_CI_LOG_TAIL_CHARS",
    "DEFAULT_COMMENT_MAX_CHARS",
    "DEFAULT_MAX_LINKED_ISSUES",
    "ImplementerPrefetchResult",
    "fetch_new_issue_context",
    "fetch_pr_fix_context",
    "fetch_request_changes_context",
    "parse_epic_reference",
)