cleveragents-core/tools/controller/master/grooming.py

"""Grooming library — helpers consumed by the grooming_stage_b worker
and the ``run_grooming_side_effects_tick`` master-side post-processor.

The Phase 1 corrected dispatch (worker-queue shape, 2026-05-25) puts
the substantive grooming logic — deterministic conclusive checks,
Stage A suspicion-score pre-filter, Stage B LLM judgment + semantic
validation — INSIDE the grooming_stage_b worker. The worker emits a
single ``GroomingOutputV1`` payload that the controller's state machine
consumes via ``_map_grooming_outcome``.

What lives here
---------------
This module is the pure-function library the worker (and the master
side-effect tick) imports for the deterministic primitives that
DON'T need the LLM:

- ``tokenize_title`` / ``tokenize_body`` — stopword-filtered token sets
  fed into the Stage A Jaccard similarity scoring.
- ``suspicion_score`` — the weighted-Jaccard pre-filter the worker uses
  to pick which candidates to actually evaluate at Stage B.
- ``classify_action_to_verdict`` — the per-LLM-action mapper
  (full_duplicate / unnecessary / needs_evaluation) → controller verdict
  (close / defer) with the ``CLOSE_ENABLED`` policy applied.
- ``DuplicateAction`` + ``Confidence`` — shared StrEnum types.

These helpers have NO Forgejo dependencies and NO DB dependencies; they
are pure functions over the prefetched input dicts.

What does NOT live here
-----------------------
- LLM call / agent invocation — that's the OpenCode-driven worker
  side at ``.opencode/agents/grooming-stage-b.md``.
- Forgejo writes — those happen in
  ``run_grooming_side_effects_tick`` via the decomposed
  ``forgejo_writes.close_act`` / ``defer_act``.
- State-machine transitions — those happen in tick.py via
  ``outcomes._map_grooming_outcome`` + ``state_machine.apply_event``.
"""

from __future__ import annotations

import enum
import re
from typing import Iterable


# ─── shared enums ────────────────────────────────────────────────────


class DuplicateAction(str, enum.Enum):
    """The agent's per-duplicate classification.

    Subclasses ``str`` so values serialize directly as text into
    ``grooming_decisions`` audit rows / output_payload JSON.
    """

    FULL_DUPLICATE = "full_duplicate"
    UNNECESSARY = "unnecessary"
    NEEDS_EVALUATION = "needs_evaluation"


class Confidence(str, enum.Enum):
    """Agent's confidence level on the overall duplicate-detection verdict."""

    HIGH = "high"
    MEDIUM = "medium"
    LOW = "low"


class GroomingVerdict(str, enum.Enum):
    """Top-level grooming verdict — what the controller does with the workflow."""

    PROCEED = "proceed"
    DEFER = "defer"
    CLOSE = "close"


# ─── tokenization ────────────────────────────────────────────────────


# Title/body stopwords — keeps Jaccard scoring from over-matching on
# function words. Sourced from the grooming-worker prompt convention on
# the agents/final-working branch + standard English stoplist.
_STOPWORDS: frozenset[str] = frozenset(
    {
        "a", "an", "and", "are", "as", "at", "be", "but", "by", "for",
        "from", "have", "has", "in", "into", "is", "it", "its", "of", "on",
        "or", "that", "the", "this", "to", "was", "were", "will", "with",
        # PR-jargon stopwords
        "pr", "wip", "draft", "fix", "fixes", "update", "updates",
        "add", "adds", "use", "make", "makes",
    }
)


_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_-]{1,}")


def _tokenize_text(text: str, *, max_tokens: int = 200) -> frozenset[str]:
    """Lowercase + word-split + stopword-strip. Returns a frozenset of
    the first ``max_tokens`` distinct tokens after filtering."""
    if not text:
        return frozenset()
    found: list[str] = []
    seen: set[str] = set()
    for match in _TOKEN_RE.finditer(text.lower()):
        tok = match.group(0)
        if tok in _STOPWORDS or len(tok) < 3 or tok in seen:
            continue
        seen.add(tok)
        found.append(tok)
        if len(found) >= max_tokens:
            break
    return frozenset(found)


def tokenize_title(title: str | None) -> frozenset[str]:
    """Tokenize a PR title for Stage A title-jaccard scoring."""
    return _tokenize_text(title or "", max_tokens=50)


def tokenize_body(body: str | None, *, head_bytes: int = 2048) -> frozenset[str]:
    """Tokenize a PR body. Per the plan: first ~2KB after stripping
    triple-backtick code fences (code blocks dominate token overlap with
    nothing useful for duplicate detection)."""
    if not body:
        return frozenset()
    stripped = re.sub(r"```[\s\S]*?```", "", body)
    return _tokenize_text(stripped[:head_bytes], max_tokens=200)


def jaccard(a: Iterable[str], b: Iterable[str]) -> float:
    """Jaccard similarity over two token sets. Empty-on-empty → 0.0."""
    sa = a if isinstance(a, (set, frozenset)) else frozenset(a)
    sb = b if isinstance(b, (set, frozenset)) else frozenset(b)
    if not sa or not sb:
        return 0.0
    inter = len(sa & sb)
    union = len(sa | sb)
    return inter / union if union else 0.0


# ─── Stage A suspicion scoring ───────────────────────────────────────


# Default weights from the plan (sum to 1.0).
DEFAULT_WEIGHTS: dict[str, float] = {
    "title": 0.25,
    "body": 0.25,
    "files": 0.30,
    "branch": 0.10,
    "closes": 0.10,
}


_CLOSES_RE = re.compile(
    r"\b(?:closes|fixes|resolves)\s+#(\d+)\b",
    re.IGNORECASE,
)


def extract_closes_refs(body: str | None) -> frozenset[int]:
    """Extract issue numbers referenced via ``Closes #N`` / ``Fixes #N`` /
    ``Resolves #N`` in the PR body."""
    if not body:
        return frozenset()
    return frozenset(int(m.group(1)) for m in _CLOSES_RE.finditer(body))


def _branch_slug_similarity(a: str | None, b: str | None) -> float:
    """Rough branch-name similarity. 1.0 on exact match, otherwise
    Jaccard on hyphen-split slug tokens. (Stand-in for the >0.7 cutoff
    in the suspicion formula.)"""
    if not a or not b:
        return 0.0
    if a == b:
        return 1.0
    aset = frozenset(t for t in re.split(r"[-_/]+", a.lower()) if t)
    bset = frozenset(t for t in re.split(r"[-_/]+", b.lower()) if t)
    return jaccard(aset, bset)


def suspicion_score(
    anchor_pr: dict,
    other_pr: dict,
    *,
    weights: dict[str, float] | None = None,
    branch_similarity_threshold: float = 0.7,
) -> float:
    """Compute the Stage A suspicion score between an anchor PR dict
    and another open-PR dict.

    Per the plan:
        suspicion = w_title  * jaccard(title_tokens(anchor),  title_tokens(other))
                  + w_body   * jaccard(body_tokens(anchor),   body_tokens(other))
                  + w_files  * jaccard(touched_files(anchor), touched_files(other))
                  + w_branch * indicator(head_branch_slug_similarity > 0.7)
                  + w_closes * indicator(same_Closes_keyword)

    The dicts are Forgejo PR-detail shapes plus an optional
    ``touched_files`` field the prefetch cache provides. Missing data
    contributes 0.0 to its term (so a partial signal still scores
    meaningfully).
    """
    w = weights or DEFAULT_WEIGHTS

    title_a = tokenize_title(anchor_pr.get("title"))
    title_b = tokenize_title(other_pr.get("title"))
    body_a = tokenize_body(anchor_pr.get("body"))
    body_b = tokenize_body(other_pr.get("body"))
    files_a = frozenset(anchor_pr.get("touched_files") or [])
    files_b = frozenset(other_pr.get("touched_files") or [])

    head_a = (
        anchor_pr.get("head", {}).get("ref")
        if isinstance(anchor_pr.get("head"), dict)
        else None
    )
    head_b = (
        other_pr.get("head", {}).get("ref")
        if isinstance(other_pr.get("head"), dict)
        else None
    )
    branch_indicator = (
        1.0
        if _branch_slug_similarity(head_a, head_b) > branch_similarity_threshold
        else 0.0
    )

    closes_a = extract_closes_refs(anchor_pr.get("body"))
    closes_b = extract_closes_refs(other_pr.get("body"))
    closes_indicator = 1.0 if (closes_a and closes_a & closes_b) else 0.0

    return (
        w["title"] * jaccard(title_a, title_b)
        + w["body"] * jaccard(body_a, body_b)
        + w["files"] * jaccard(files_a, files_b)
        + w["branch"] * branch_indicator
        + w["closes"] * closes_indicator
    )


# ─── deterministic conclusive checks ─────────────────────────────────


# Each check inspects the prefetched data and returns either None
# (check did NOT fire) or a (reason_category, target_workflow_id_or_None)
# tuple that the worker turns into a deterministic verdict. Pure
# functions over dicts; no I/O.


def check_linked_issue_closed(
    anchor_pr: dict,
    closed_issue_numbers: frozenset[int],
) -> str | None:
    """If the anchor body has ``Closes #N`` AND issue N is already
    closed, return the reason_category 'linked_issue_closed'."""
    refs = extract_closes_refs(anchor_pr.get("body"))
    if refs and refs & closed_issue_numbers:
        return "linked_issue_closed"
    return None


# ─── verdict / action mapping ────────────────────────────────────────


def classify_action_to_verdict(
    action: DuplicateAction | str,
    *,
    close_enabled: bool,
) -> GroomingVerdict:
    """Map a per-LLM-duplicate action to the controller verdict, applying
    the CLOSE_ENABLED policy from the plan:

    | LLM action       | close_enabled=false | close_enabled=true |
    |------------------|---------------------|--------------------|
    | full_duplicate   | DEFER               | CLOSE              |
    | unnecessary      | DEFER               | CLOSE              |
    | needs_evaluation | DEFER               | DEFER              |  (always)
    """
    if isinstance(action, str):
        action = DuplicateAction(action)
    if action == DuplicateAction.NEEDS_EVALUATION:
        return GroomingVerdict.DEFER
    return GroomingVerdict.CLOSE if close_enabled else GroomingVerdict.DEFER


__all__ = [
    "Confidence",
    "DEFAULT_WEIGHTS",
    "DuplicateAction",
    "GroomingVerdict",
    "check_linked_issue_closed",
    "classify_action_to_verdict",
    "extract_closes_refs",
    "jaccard",
    "suspicion_score",
    "tokenize_body",
    "tokenize_title",
]