cleveragents-core/tools/controller/master/ci_summarize.py

"""Deterministic CI summarizer — builds CISummary from Forgejo status.

The implementer/reviewer prompt prefetch needs a CISummary V1 dict for
the head_sha. This module takes the Forgejo combined-status response
(``{"state": "...", "statuses": [...]}``) plus a log fetcher, and
returns a fully-populated CISummary covering every gate.

Per-gate flow:
1. Map the Forgejo gate context (e.g., "CI / lint") → nox session
   name. The job names in ``.forgejo/workflows/*.yml`` ARE the nox
   session names by convention, so a simple suffix-after-/ extraction
   works. Unknown contexts pass through as ``UnknownGate`` and skip
   the parser.
2. Look up the nox session in NOX_SESSION_TO_PARSER → one or more
   parser names.
3. For each parser, fetch the log (via the injected ``log_fetcher``),
   run ``parse(log, gate_name)``, and wrap the result in a
   GateResult.failure CIFailure dict.
4. For composite gates (security_scan = bandit + semgrep + vulture),
   the outer CIFailure carries the FIRST parser's structure + each
   sub-parser's result goes into ``composite_findings``.

The summarizer never raises — log fetch failures degrade to a
``CIFailure`` with ``error_class='log-fetch-failed'`` so the
implementer at least sees the failure existed.
"""

from __future__ import annotations

import logging
import re
from collections.abc import Callable
from datetime import datetime, timezone
from typing import Any

from ..ci_summary_parsers import NOX_SESSION_TO_PARSER, NOX_SESSIONS_SKIP_COVERAGE
from ..ci_summary_parsers._base import strip_log_timestamps
from ..ci_summary_parsers._registry import (
    ResolvedParser,
    resolve_for_nox_session,
)
from .ci_run_status import (
    GetActionTasksCallback,
    classify_ci_run,
    terminal_verdict,
)

logger = logging.getLogger(__name__)


# log_fetcher(gate_name) -> str | None; production wires this to a
# Forgejo job-log fetcher. None means the log was unreachable.
LogFetcher = Callable[[str], str | None]

# Cap on the raw-log excerpt carried on a NoParserAvailable gate —
# matches the V1 ``CIFailure.raw_log_excerpt`` max_length budget.
_RAW_EXCERPT_MAX_CHARS = 16_384


# Forgejo / Gitea / GH-mirror status states → GateResult.status.
# Covers states observed in the wild across Forgejo, Gitea, and
# GitHub-mirroring (action_required, neutral, stale, timed_out come
# from the GH side; Forgejo can emit cancelled/queued/in_progress for
# Actions-style runs).
_FORGEJO_STATE_TO_GATE_STATUS = {
    "success": "passed",
    "failure": "failed",
    "error": "error",
    "pending": "pending",
    "queued": "pending",
    "in_progress": "pending",
    "warning": "passed",  # advisory: treat as passed
    "neutral": "passed",  # GH-style "no decision" — informational
    "skipped": "skipped",
    "stale": "error",  # GH-style — treat as failed-equivalent
    "cancelled": "error",  # user/system cancelled mid-run
    "timed_out": "error",
    "action_required": "error",  # GH waits for human approval — treat as failed
    None: "pending",
}


def summarize_ci_status(
    *,
    head_sha: str,
    forgejo_status: dict | None,
    log_fetcher: LogFetcher,
    observed_at: datetime | None = None,
    owner: str = "",
    repo: str = "",
    get_action_tasks: GetActionTasksCallback | None = None,
) -> dict:
    """Build a CISummary-shape dict from the Forgejo combined-status.

    Args:
        head_sha: the commit SHA the status covers.
        forgejo_status: response body from
            ``GET /repos/{owner}/{repo}/commits/{sha}/status``.
            Shape: ``{"state": str, "statuses": [{"context": str,
            "state": str, "target_url": str}]}``. None for
            "couldn't fetch" — produces a unknown-state summary.
        log_fetcher: callable that takes the gate context and returns
            the raw log text (or None if unreachable). Production
            wires this via Forgejo's job-log endpoint.
        observed_at: timestamp the status was fetched. Defaults to
            now(UTC).

    Returns the CISummary V1-shape dict (a CISummary contract parses
    it after the caller validates).
    """
    when = observed_at or datetime.now(timezone.utc)
    if forgejo_status is None:
        return _unknown_summary(head_sha=head_sha, observed_at=when)

    overall = _FORGEJO_OVERALL_STATE_MAPPING.get(
        forgejo_status.get("state"),
        "unknown",
    )
    raw_statuses = forgejo_status.get("statuses") or []
    statuses = [s for s in raw_statuses if isinstance(s, dict)]

    gates: list[dict] = []
    parser_versions: dict[str, str] = {}
    for s in statuses:
        gate, versions_seen = _build_gate(s, log_fetcher)
        gates.append(gate)
        parser_versions.update(versions_seen)

    counts = _count_gates(gates)

    # Zombie-CI handling: a run that has stopped progressing but still
    # shows pending gates must not be reported as pending — the
    # implementer would emit ci-not-ready and wait on a dead run.
    # Reclassify so gates_pending drops the zombies and overall_state
    # carries the verdict from the gates that finished. Mirrors
    # ci_status_poll so the poll and the implementer agree (no
    # ci-not-ready <-> ci_red ping-pong).
    if (
        classify_ci_run(
            statuses,
            now=when,
            get_action_tasks=get_action_tasks,
            owner=owner,
            repo=repo,
            head_sha=head_sha,
        )
        == "stale"
    ):
        verdict = terminal_verdict(statuses)
        overall = verdict if verdict != "pending" else "failure"
        counts = {**counts, "pending": 0}

    return {
        "summary_version": "V1",
        "head_sha": head_sha,
        "observed_at": when,
        "overall_state": overall,
        "gates": gates,
        "gates_total": counts["total"],
        "gates_passed": counts["passed"],
        "gates_failed": counts["failed"],
        "gates_skipped": counts["skipped"],
        "gates_pending": counts["pending"],
        "parser_versions": parser_versions,
    }


# Forgejo combined-status states → CISummary.overall_state.
_FORGEJO_OVERALL_STATE_MAPPING: dict[str | None, str] = {
    "success": "success",
    "failure": "failure",
    "error": "error",
    "pending": "pending",
    None: "unknown",
}


def _unknown_summary(*, head_sha: str, observed_at: datetime) -> dict:
    return {
        "summary_version": "V1",
        "head_sha": head_sha,
        "observed_at": observed_at,
        "overall_state": "unknown",
        "gates": [],
        "gates_total": 0,
        "gates_passed": 0,
        "gates_failed": 0,
        "gates_skipped": 0,
        "gates_pending": 0,
        "parser_versions": {},
    }


def _build_gate(
    status: dict,
    log_fetcher: LogFetcher,
) -> tuple[dict, dict[str, str]]:
    """Build one GateResult dict + return parser versions used."""
    context = status.get("context") or "(unknown)"
    state_raw = status.get("state")
    gate_status = _FORGEJO_STATE_TO_GATE_STATUS.get(state_raw, "pending")
    target_url = status.get("target_url")
    severity = "info" if gate_status in {"passed", "skipped"} else "error"

    gate: dict[str, Any] = {
        "name": context,
        "status": gate_status,
        "severity": severity,
        "target_url": target_url,
        "duration_seconds": None,
        "failure": None,
    }
    parser_versions: dict[str, str] = {}

    # Only parse failures/errors. Passed/pending/skipped don't need a
    # log fetch — they have nothing to report.
    if gate_status not in {"failed", "error"}:
        return gate, parser_versions

    nox_session = _gate_to_nox_session(context)
    if nox_session is None or nox_session in NOX_SESSIONS_SKIP_COVERAGE:
        # No parser for this gate — but still surface the raw log so a
        # worker is not blind on an unmapped failed gate.
        gate["failure"] = _no_parser_failure(
            context, _safe_fetch(log_fetcher, context)
        )
        return gate, parser_versions

    try:
        parsers = resolve_for_nox_session(nox_session)
    except KeyError:
        logger.warning(
            "no parser map for nox session %r (gate=%r)",
            nox_session,
            context,
        )
        gate["failure"] = _no_parser_failure(
            context, _safe_fetch(log_fetcher, context)
        )
        return gate, parser_versions

    log = _safe_fetch(log_fetcher, context)
    if log is None:
        gate["failure"] = _log_fetch_failed_failure(parsers, context)
        for p in parsers:
            parser_versions[p.name] = p.version
        return gate, parser_versions

    gate["failure"] = _build_failure(parsers, log, context, parser_versions)
    return gate, parser_versions


def _gate_to_nox_session(context: str) -> str | None:
    """Forgejo gate context → nox session name.

    Convention: ``<workflow-name> / <job-name>`` (e.g., "CI / lint").
    The job name IS the nox session name by repo convention. If the
    name doesn't match any known session, return None and let the
    summarizer fall through to NoParserAvailable."""
    if not context:
        return None
    tail = context.rsplit("/", 1)[-1].strip()
    # Forgejo appends the triggering event to Actions commit-status
    # contexts — "CI / unit_tests (pull_request)". Strip that trailing
    # "(event)" so the job name matches a nox session; without this
    # every PR gate falls through to NoParserAvailable.
    tail = re.sub(r"\s*\([^)]*\)\s*$", "", tail).strip()
    if tail in NOX_SESSION_TO_PARSER:
        return tail
    # Also support the parameterized form e.g. "unit_tests-3.13".
    base = tail.split("-", 1)[0]
    if base in NOX_SESSION_TO_PARSER:
        return base
    return None


def _safe_fetch(log_fetcher: LogFetcher, context: str) -> str | None:
    try:
        log = log_fetcher(context)
    except Exception as exc:  # noqa: BLE001
        logger.warning("log fetch raised for %s: %s", context, exc)
        return None
    if log is None:
        return None
    # Strip the Actions per-line timestamp prefix once, here — the
    # single chokepoint feeding every parser AND the no-parser raw-log
    # excerpt. Without this the parsers' ^-anchored finding regexes
    # match nothing (summary says N errors, findings=0).
    return strip_log_timestamps(log)


def _no_parser_failure(context: str, raw_log: str | None = None) -> dict:
    """CIFailure for a gate with no structured parser. ``raw_log`` —
    when the caller could fetch it — is carried as the excerpt (tail-
    capped) so a worker still sees the failure text."""
    excerpt = (raw_log or "")[-_RAW_EXCERPT_MAX_CHARS:]
    return {
        "parser_used": "(none)",
        "parser_version": "n/a",
        "error_class": "NoParserAvailable",
        "summary_line": f"No parser available for {context!r}"[:200],
        "findings": [],
        "failing_locations": [],
        "failed_assertions": [],
        "raw_log_excerpt": excerpt,
        "log_excerpt_lines": (excerpt.count("\n") + 1) if excerpt else 0,
        "composite_findings": [],
    }


def _log_fetch_failed_failure(parsers, context: str) -> dict:
    parser_names = ", ".join(p.name for p in parsers)
    return {
        "parser_used": parser_names or "(unknown)",
        "parser_version": "n/a",
        "error_class": "log-fetch-failed",
        "summary_line": f"Log fetch failed for {context!r}"[:200],
        "findings": [],
        "failing_locations": [],
        "failed_assertions": [],
        "raw_log_excerpt": "",
        "log_excerpt_lines": 0,
        "composite_findings": [],
    }


def _build_failure(
    parsers: list[ResolvedParser],
    log: str,
    context: str,
    parser_versions: dict[str, str],
) -> dict:
    """Build the CIFailure dict from one or more parsers' results.

    For a single-parser gate the result becomes the outer CIFailure.
    For a composite gate (multi: prefix), the FIRST parser becomes
    the outer + the rest go into composite_findings.
    """
    results = []
    for p in parsers:
        try:
            res = p.parse(log, context)
            parser_versions[p.name] = p.version
            results.append((p, res))
        except Exception as exc:  # noqa: BLE001
            logger.warning(
                "parser %s raised on gate %r: %s",
                p.name,
                context,
                exc,
            )
            # Fall through with no result for this parser.

    if not results:
        return _log_fetch_failed_failure(parsers, context)

    head_parser, head_result = results[0]
    failure = _result_to_failure(head_parser, head_result)
    if len(results) > 1:
        failure["composite_findings"] = [
            _result_to_failure(p, r) for p, r in results[1:]
        ]
    return failure


def _result_to_failure(parser: ResolvedParser, result) -> dict:
    return {
        "parser_used": parser.name,
        "parser_version": parser.version,
        "error_class": result.error_class,
        "summary_line": result.summary_line,
        "findings": [f.model_dump() for f in result.findings],
        "failing_locations": [loc.model_dump() for loc in result.failing_locations],
        "failed_assertions": [a.model_dump() for a in result.failed_assertions],
        "raw_log_excerpt": result.log_excerpt,
        "log_excerpt_lines": result.log_excerpt_lines,
        "composite_findings": [],
    }


def _count_gates(gates: list[dict]) -> dict[str, int]:
    counts = {"total": len(gates), "passed": 0, "failed": 0, "skipped": 0, "pending": 0}
    for g in gates:
        status = g.get("status")
        if status == "passed":
            counts["passed"] += 1
        elif status in {"failed", "error"}:
            counts["failed"] += 1
        elif status == "skipped":
            counts["skipped"] += 1
        elif status == "pending":
            counts["pending"] += 1
    return counts


__all__ = ["LogFetcher", "summarize_ci_status"]