cleveragents-core/tools/_verify_common.py

"""Shared substrate for the open-PR pipeline verifiers.

Both ``verify_review_invariant.py`` and
``verify_implementer_invariant.py`` audit open PRs against the
deterministic invariants of their respective pipelines and open
``auto/invariant-violation`` issues on breach. The Forgejo API
plumbing, idempotent issue creation, label lookup, and PR iteration
are identical between them — extracted here to avoid drift.

The original ``tools/verify_invariant.py`` (the merge invariant
verifier) duplicates most of this logic in-place; that module is the
working template referenced in the auto-agents Tier 2/3 plan. A
future refactor can migrate it to consume this substrate, but is out
of scope for the parity work the plan explicitly authorizes
("strictly additive ... a single Python file under 500 lines per
new verifier").

The substrate is intentionally narrow: it does not own the per-
invariant predicates (those live in each verifier's main module),
and it does not own the cron / argparse plumbing (each verifier
owns its own ``main`` for clarity at the call site).
"""

from __future__ import annotations

import json
import logging
import os
import re
import time
import urllib.error
import urllib.request
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

_logger = logging.getLogger("verify_common")


# Forgejo connection knobs. Same defaults as
# ``tools/verify_invariant.py`` so the two verifiers share canonical
# environment overrides; an operator who configures one configures
# both.
REPO_OWNER = os.environ.get("FORGEJO_OWNER", "cleveragents")
REPO_NAME = os.environ.get("FORGEJO_REPO", "cleveragents-core")
ORG_NAME = os.environ.get("FORGEJO_ORG", "cleveragents")
DEFAULT_BRANCH = os.environ.get("FORGEJO_DEFAULT_BRANCH", "master")
API_BASE = os.environ.get(
    "FORGEJO_API_BASE", "https://git.cleverthis.com/api/v1"
).rstrip("/")
REQUEST_TIMEOUT_SEC = 30
REQUEST_RETRIES = 3
RETRY_BACKOFF_SEC = 2.0


INVARIANT_LABEL = "auto/invariant-violation"


def load_token() -> str:
    """Resolve the Forgejo PAT used for API calls.

    Reads ``GITEA_TOKEN`` from the environment first; falls back to
    ``GITEA_TOKEN=...`` in either ``.devcontainer/.env`` or ``.env`` at
    the repo root. Raises :class:`RuntimeError` if no token is
    available — the verifier cron job exits non-zero in that case so
    the operator notices.

    Setting ``VERIFIER_DOTENV_DISABLED=1`` (or any truthy value)
    skips the dotenv fallback entirely. Used by the hermetic
    subprocess smoke tests to force the no-token branch even on
    devcontainers where ``.devcontainer/.env`` carries a real token.
    """
    token = os.environ.get("GITEA_TOKEN")
    if token:
        return token
    if os.environ.get("VERIFIER_DOTENV_DISABLED", "").strip().lower() in (
        "1",
        "true",
        "yes",
        "on",
    ):
        raise RuntimeError("GITEA_TOKEN not found (dotenv disabled)")
    repo_root = Path(__file__).resolve().parent.parent
    for candidate in (repo_root / ".devcontainer/.env", repo_root / ".env"):
        if candidate.exists():
            for line in candidate.read_text().splitlines():
                m = re.match(r'^\s*GITEA_TOKEN\s*=\s*"?([^"#\s]+)"?', line)
                if m:
                    return m.group(1)
    raise RuntimeError("GITEA_TOKEN not found")


def api(
    method: str,
    path: str,
    token: str,
    body: Any | None = None,
) -> dict[str, Any]:
    """Issue one Forgejo API request with retry on transport errors.

    Mirrors the behaviour of ``tools/verify_invariant.py:api`` so the
    two verifiers behave identically against the same server. Returns
    ``{"status": <int>, "body": <parsed-or-bytes-or-text>}``; non-2xx
    HTTP responses are returned as a dict (NOT raised) so callers can
    branch on ``res["status"]``.

    Retry semantics — important for callers reasoning about
    idempotence:

    - **Transport errors** (``URLError``, ``TimeoutError``, ``OSError``)
      are retried up to ``REQUEST_RETRIES`` times with linear backoff.
      These represent "request never reached the server" or "no
      response received", so a retry is safe even for non-idempotent
      methods (``POST``).
    - **HTTP errors** (``HTTPError`` — i.e. the server responded with
      a non-2xx status) are NOT retried. They surface immediately as
      ``{"status": <code>, "body": <error-text>}`` and the caller
      decides whether to retry based on the code. This is intentional:
      a ``500`` from the server is in principle retryable, but
      retrying a non-idempotent ``POST`` that the server may have
      processed before crashing risks duplicate work (e.g. opening
      duplicate issues). Callers that know a request is idempotent
      (``GET``) and want server-side error retries must implement
      that loop themselves; the current verifiers do not need it
      because the next cron tick replays everything anyway.
    """
    url = f"{API_BASE}{path}" if path.startswith("/") else path
    data = json.dumps(body).encode() if body is not None else None
    headers = {"Authorization": f"token {token}"}
    if data is not None:
        headers["Content-Type"] = "application/json"
    last_err: Exception | None = None
    for attempt in range(1, REQUEST_RETRIES + 1):
        req = urllib.request.Request(url, data=data, headers=headers, method=method)
        try:
            with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as resp:
                payload = resp.read()
                ct = resp.headers.get("Content-Type", "")
                return {
                    "status": resp.status,
                    "body": (
                        json.loads(payload)
                        if payload and "application/json" in ct
                        else payload
                    ),
                }
        except urllib.error.HTTPError as e:
            return {
                "status": e.code,
                "body": e.read().decode("utf-8", "replace"),
            }
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            last_err = e
            if attempt < REQUEST_RETRIES:
                time.sleep(RETRY_BACKOFF_SEC * attempt)
                continue
    raise RuntimeError(f"network error contacting {url}: {last_err}")


def lookup_label_id(token: str, label_name: str) -> int | None:
    """Resolve a label name to its numeric Forgejo ID.

    Searches both the org-level and repo-level label lists; returns
    the first match. ``None`` if the label does not exist (the
    verifier creates issues without that label rather than failing).
    """
    for path in (
        f"/orgs/{ORG_NAME}/labels",
        f"/repos/{REPO_OWNER}/{REPO_NAME}/labels",
    ):
        page = 1
        while True:
            res = api("GET", f"{path}?limit=50&page={page}", token)
            if res["status"] != 200:
                break
            chunk = res["body"] or []
            for lbl in chunk:
                if lbl.get("name") == label_name:
                    return lbl.get("id")
            if len(chunk) < 50:
                break
            page += 1
    return None


def iterate_open_prs(token: str) -> Iterator[dict[str, Any]]:
    """Yield every open PR in the repo, paginated.

    Each yielded dict is the Forgejo PR object verbatim — the caller
    is responsible for extracting ``head.sha`` / ``user.login`` /
    ``labels[].name`` etc. as needed.
    """
    page = 1
    while True:
        res = api(
            "GET",
            f"/repos/{REPO_OWNER}/{REPO_NAME}/pulls?state=open&limit=50&page={page}",
            token,
        )
        if res["status"] != 200:
            return
        chunk = res["body"] or []
        if not isinstance(chunk, list):
            return
        yield from chunk
        if len(chunk) < 50:
            return
        page += 1


def fetch_commit_statuses(
    token: str,
    sha: str,
    *,
    telemetry: dict[str, int] | None = None,
) -> tuple[dict[str, str], bool]:
    """Return ``({context: state}, success)`` for ``sha``'s combined
    status.

    The dict portion mirrors the body shape of the legacy
    ``verify_invariant.fetch_commit_statuses`` (later rows for the
    same context override earlier ones — Forgejo returns them
    chronologically), but the **return type differs**: the legacy
    helper returns a bare ``dict[str, str]`` and signals partial
    fetches by returning whatever was successfully read up to the
    failure, indistinguishably from a clean empty result. This
    helper returns the dict paired with an explicit success flag
    so verifier callers can short-circuit on transient failures
    without false-negatives. Do not assume call-site
    interchangeability.

    The second tuple element is the **success flag**:

    - ``True`` if every page request returned 200; the dict is the
      full set of statuses observed.
    - ``False`` if any page short-circuited on a non-200 (transient
      Forgejo error). The dict carries whatever statuses were
      successfully fetched up to the point of failure — callers
      MUST check the success flag before drawing conclusions about
      the data being a complete observation.

    When ``telemetry`` is provided, the counter
    ``skipped_due_to_transient_error`` is bumped on a transient
    failure so the verifier's run-level summary can distinguish
    "no successful CI checks" from "CI status fetch transiently
    failed".

    The ``(out, success)`` tuple shape replaces an older
    snapshot-the-counter pattern in ``audit_one_pr`` that worked
    correctly but was fragile against future telemetry-bumping
    callers being inserted between the snapshots.
    """
    out: dict[str, str] = {}
    page = 1
    while True:
        res = api(
            "GET",
            f"/repos/{REPO_OWNER}/{REPO_NAME}/commits/{sha}/statuses"
            f"?limit=50&page={page}",
            token,
        )
        if res["status"] != 200:
            if telemetry is not None:
                telemetry["skipped_due_to_transient_error"] = (
                    telemetry.get("skipped_due_to_transient_error", 0) + 1
                )
            return out, False
        chunk = res["body"] or []
        for s in chunk:
            ctx = s.get("context") or ""
            state = (s.get("status") or s.get("state") or "").lower()
            out[ctx] = state
        if len(chunk) < 50:
            break
        page += 1
    return out, True


def required_checks_passed(
    statuses: dict[str, str], required_patterns: list[str]
) -> bool:
    """Each required pattern (with optional trailing ``*`` wildcard)
    must match at least one ``success`` status. Mirrors
    ``verify_invariant.required_checks_passed``.
    """
    if not required_patterns:
        return False
    for pattern in required_patterns:
        prefix = pattern[:-1] if pattern.endswith("*") else pattern
        matches = [
            (ctx, state)
            for ctx, state in statuses.items()
            if (ctx == pattern or (pattern.endswith("*") and ctx.startswith(prefix)))
        ]
        if not matches:
            return False
        if not any(state == "success" for _, state in matches):
            return False
    return True


def get_required_check_contexts(token: str) -> list[str]:
    """Read ``branch_protections/<DEFAULT_BRANCH>.status_check_contexts``."""
    res = api(
        "GET",
        f"/repos/{REPO_OWNER}/{REPO_NAME}/branch_protections/{DEFAULT_BRANCH}",
        token,
    )
    if res["status"] != 200:
        return []
    return ((res["body"] or {}).get("status_check_contexts")) or []


def existing_violation_issue(
    token: str, marker: str, identifier: str
) -> dict[str, Any] | None:
    """Find an open ``auto/invariant-violation`` issue carrying both
    ``marker`` (the verifier-specific HTML comment in the body) and
    ``identifier`` (the PR number / SHA / unique key the verifier
    chose).

    Both must appear in the body to count as a match, so two verifiers
    can use distinct markers without colliding on the same PR.
    """
    res = api(
        "GET",
        f"/repos/{REPO_OWNER}/{REPO_NAME}/issues"
        f"?state=open&type=issues&labels={INVARIANT_LABEL}&limit=50",
        token,
    )
    if res["status"] != 200:
        return None
    issues = res["body"] or []
    if not isinstance(issues, list):
        return None
    for issue in issues:
        body = issue.get("body") or ""
        if marker in body and identifier in body:
            return issue
    return None


def open_violation_issue(
    token: str,
    *,
    title: str,
    body_meta: dict[str, Any],
    marker: str,
    identifier: str,
    summary: str,
    dry_run: bool,
) -> dict[str, Any]:
    """Open one ``auto/invariant-violation`` issue (idempotent).

    The body always carries the ``marker`` HTML comment + a YAML
    metadata fence (so future verifiers can re-parse the body
    deterministically) + a human-readable summary. Returns
    ``{"created": bool, "issue": <dict|None>, "dry_run": bool, ...}``.

    Idempotent: if an open issue with the same ``(marker, identifier)``
    pair exists, returns it without creating a new one.
    """
    existing = existing_violation_issue(token, marker, identifier)
    if existing is not None:
        return {"created": False, "issue": existing, "dry_run": dry_run}
    label_id = lookup_label_id(token, INVARIANT_LABEL)
    body_meta = {
        "detected_at": datetime.now(UTC).isoformat(),
        **body_meta,
    }
    body = (
        f"{marker}\n\n"
        f"```yaml\n"
        f"{json.dumps(body_meta, indent=2)}\n"
        f"```\n\n"
        f"**identifier**: `{identifier}`\n\n"
        f"{summary}"
    )
    if dry_run:
        return {
            "created": False,
            "dry_run": True,
            "title": title,
            "marker": marker,
            "identifier": identifier,
        }
    res = api(
        "POST",
        f"/repos/{REPO_OWNER}/{REPO_NAME}/issues",
        token,
        body={
            "title": title,
            "body": body,
            "labels": [label_id] if label_id else [],
        },
    )
    return {
        "created": res["status"] in (200, 201),
        "status": res["status"],
        "issue": res.get("body"),
    }


__all__ = (
    "API_BASE",
    "DEFAULT_BRANCH",
    "INVARIANT_LABEL",
    "ORG_NAME",
    "REPO_NAME",
    "REPO_OWNER",
    "REQUEST_RETRIES",
    "REQUEST_TIMEOUT_SEC",
    "RETRY_BACKOFF_SEC",
    "api",
    "existing_violation_issue",
    "fetch_commit_statuses",
    "get_required_check_contexts",
    "iterate_open_prs",
    "load_token",
    "lookup_label_id",
    "open_violation_issue",
    "required_checks_passed",
)