diff --git a/tests/auto_agents/controller/test_ci_summary_parsers.py b/tests/auto_agents/controller/test_ci_summary_parsers.py new file mode 100644 index 000000000..63b99bfbf --- /dev/null +++ b/tests/auto_agents/controller/test_ci_summary_parsers.py @@ -0,0 +1,299 @@ +"""Tests for the per-tool CI summary parsers (Phase 1j). + +Validates each parser: +- Extracts findings from canonical CLI output +- Populates summary_line, error_class, log_excerpt correctly +- Handles empty/garbage input without raising +""" +from __future__ import annotations + +import pytest + +from tools.controller.ci_summary_parsers import ( + EXPECTED_PARSERS, + NOX_SESSION_TO_PARSER, +) +from tools.controller.ci_summary_parsers import _stub, behave, pyright, ruff +from tools.controller.ci_summary_parsers._base import select_log_excerpt +from tools.controller.ci_summary_parsers._registry import ( + resolve, + resolve_for_nox_session, + validate_parser_coverage, +) + + +# ─── shared base helpers ──────────────────────────────────────────── + + +class TestSelectLogExcerpt: + def test_empty_log(self): + excerpt, lines = select_log_excerpt("") + assert excerpt == "" + assert lines == 0 + + def test_short_log_returned_whole(self): + log = "line 1\nline 2\nline 3\n" + excerpt, lines = select_log_excerpt(log, tail_lines=80) + assert "line 1" in excerpt + assert lines == 3 + + def test_long_log_tail_kept(self): + log = "\n".join(f"line {i}" for i in range(200)) + excerpt, lines = select_log_excerpt(log, tail_lines=10) + assert "line 199" in excerpt + assert "line 100" not in excerpt + assert lines == 10 + + def test_hard_char_cap_respected(self): + log = "x" * 50_000 + excerpt, lines = select_log_excerpt(log, max_chars=1000) + assert len(excerpt) <= 1000 + + +# ─── ruff ─────────────────────────────────────────────────────────── + + +class TestRuffParser: + def test_lint_findings_parsed(self): + log = """\ +tools/x.py:10:5: F401 `os` imported but unused +tools/y.py:20:1: E501 line too long (90 > 88 characters) +Found 2 errors. +""" + result = ruff.parse(log, "CI / lint") + assert len(result.findings) == 2 + codes = [f.error_class for f in result.findings] + assert codes == ["F401", "E501"] + assert "Ruff found 2 error(s)" in result.summary_line + assert result.error_class == "RuffMixed" + + def test_all_findings_same_class(self): + log = """\ +a.py:1:1: F401 unused +b.py:2:1: F401 unused +Found 2 errors. +""" + result = ruff.parse(log, "lint") + assert result.error_class == "F401" + + def test_format_findings_parsed(self): + log = """\ +Would reformat: tools/a.py +Would reformat: tools/b.py +2 files would be reformatted +""" + result = ruff.parse(log, "format") + assert len(result.findings) == 2 + assert all(f.error_class == "format" for f in result.findings) + assert "would be reformatted" in result.findings[0].summary + + def test_empty_log(self): + result = ruff.parse("", "lint") + assert result.findings == [] + assert result.error_class == "RuffEmpty" + + def test_garbage_log_no_crash(self): + result = ruff.parse("totally not ruff output\nnope nope\n", "lint") + assert result.findings == [] + # Falls back to "0 ruff findings" summary + assert "0" in result.summary_line or "ruff" in result.summary_line.lower() + + def test_locations_populated(self): + log = "tools/x.py:42:10: F401 foo\n" + result = ruff.parse(log, "lint") + assert len(result.failing_locations) == 1 + assert result.failing_locations[0].file_path == "tools/x.py" + assert result.failing_locations[0].line_range == "42-42" + + +# ─── pyright ─────────────────────────────────────────────────────── + + +class TestPyrightParser: + def test_basic_diagnostic(self): + log = """\ +/home/runner/work/repo/tools/foo.py:10:5 - error: Cannot find name 'x' (reportUnknownVariableType) +/home/runner/work/repo/tools/bar.py:1:1 - warning: trivial (reportUnusedImport) +1 errors, 1 warnings, 0 informations +""" + result = pyright.parse(log, "typecheck") + assert len(result.findings) == 2 + errs = [f for f in result.findings if f.severity == "error"] + assert len(errs) == 1 + assert errs[0].error_class == "reportUnknownVariableType" + assert "Pyright: 1 errors" in result.summary_line + + def test_path_normalization(self): + log = "/abs/path/work/tools/foo.py:10:5 - error: msg (reportFoo)\n" + result = pyright.parse(log, "typecheck") + assert result.findings[0].location.file_path == "tools/foo.py" + + def test_no_errors_only_warnings(self): + log = """\ +/x/tools/foo.py:1:1 - warning: trivial (reportUnusedImport) +0 errors, 1 warnings, 0 informations +""" + result = pyright.parse(log, "typecheck") + # Errors-only locations → empty + assert result.failing_locations == [] + assert result.error_class == "PyrightClean" + + def test_empty_log(self): + result = pyright.parse("", "typecheck") + assert result.findings == [] + assert result.error_class == "PyrightClean" + + def test_two_errors_diff_classes(self): + log = """\ +/x/tools/a.py:1:1 - error: a (reportFoo) +/x/tools/b.py:2:2 - error: b (reportBar) +2 errors, 0 warnings, 0 informations +""" + result = pyright.parse(log, "typecheck") + assert result.error_class == "PyrightMixed" + + def test_two_errors_same_class(self): + log = """\ +/x/tools/a.py:1:1 - error: a (reportFoo) +/x/tools/b.py:2:2 - error: b (reportFoo) +2 errors, 0 warnings, 0 informations +""" + result = pyright.parse(log, "typecheck") + assert result.error_class == "reportFoo" + + +# ─── behave ──────────────────────────────────────────────────────── + + +class TestBehaveParser: + def test_failing_scenario_extracted(self): + log = """\ +Feature: My feature + Scenario: My scenario + Then baz failed + +Failing scenarios: + tests/features/foo.feature:10 Scenario: My scenario + +0 features passed, 1 failed, 0 skipped +0 scenarios passed, 1 failed, 0 skipped +2 steps passed, 1 failed, 1 skipped + AssertionError: expected 1 == 2 +""" + result = behave.parse(log, "unit_tests") + assert len(result.findings) == 1 + assert result.findings[0].error_class == "ScenarioFailed" + assert "My scenario" in result.findings[0].summary + loc = result.findings[0].location + assert loc.file_path == "tests/features/foo.feature" + assert loc.line_range == "10-10" + assert "Behave: 1 feature(s) failed" in result.summary_line + + def test_no_failing_scenarios(self): + log = """\ +1 features passed, 0 failed, 0 skipped +1 scenarios passed, 0 failed, 0 skipped +""" + result = behave.parse(log, "unit_tests") + assert result.findings == [] + assert result.error_class == "BehaveClean" + + def test_empty_log(self): + result = behave.parse("", "unit_tests") + assert result.findings == [] + + def test_assertion_extracted(self): + log = """\ +Failing scenarios: + tests/x.feature:5 Scenario: foo + +AssertionError: my-message-here +0 features passed, 1 failed, 0 skipped +0 scenarios passed, 1 failed, 0 skipped +""" + result = behave.parse(log, "unit_tests") + assert len(result.failed_assertions) == 1 + assert "my-message-here" in result.failed_assertions[0].assertion_excerpt + + +# ─── stub ────────────────────────────────────────────────────────── + + +class TestStubParser: + def test_stub_returns_pending_marker(self): + log = "any log content\nmore\n" + parse = _stub.make_stub("robot_framework") + result = parse(log, "integration_tests") + assert result.error_class == "parser-pending-robot_framework" + assert "parser pending" in result.summary_line + assert "any log content" in result.log_excerpt + assert len(result.findings) == 1 + assert result.findings[0].severity == "error" + + def test_each_stub_independently_named(self): + a = _stub.make_stub("bandit") + b = _stub.make_stub("vulture") + ra = a("log", "g") + rb = b("log", "g") + assert ra.error_class != rb.error_class + + +# ─── registry ────────────────────────────────────────────────────── + + +class TestRegistry: + def test_resolve_real_ruff(self): + r = resolve("ruff") + assert r.is_stub is False + assert r.name == "ruff" + assert r.version == "v1" + + def test_resolve_real_pyright(self): + r = resolve("pyright") + assert r.is_stub is False + assert r.parse is pyright.parse + + def test_resolve_real_behave(self): + r = resolve("behave") + assert r.is_stub is False + + def test_resolve_pending_stub(self): + r = resolve("robot_framework") + assert r.is_stub is True + assert r.version == "v1-stub" + + def test_resolve_for_nox_session_single(self): + parsers = resolve_for_nox_session("lint") + assert [p.name for p in parsers] == ["ruff"] + + def test_resolve_for_nox_session_composite(self): + parsers = resolve_for_nox_session("security_scan") + names = [p.name for p in parsers] + assert names == ["bandit", "semgrep", "vulture"] + + def test_resolve_for_unknown_session_raises(self): + with pytest.raises(KeyError): + resolve_for_nox_session("nope_not_a_session") + + def test_validate_parser_coverage(self): + cov = validate_parser_coverage() + # Every name in EXPECTED_PARSERS shows up. + assert set(cov.keys()) == EXPECTED_PARSERS + # ruff/pyright/behave are real; rest are stubs. + assert cov["ruff"] is True + assert cov["pyright"] is True + assert cov["behave"] is True + assert cov["robot_framework"] is False + assert cov["bandit"] is False + + +# ─── expected coverage shape ─────────────────────────────────────── + + +class TestExpectedCoverageShape: + def test_nox_session_map_only_references_known_parsers(self): + for parser in EXPECTED_PARSERS: + assert resolve(parser).name == parser + + def test_security_scan_is_composite(self): + assert NOX_SESSION_TO_PARSER["security_scan"].startswith("multi:") diff --git a/tests/auto_agents/controller/test_master_ci_summarize.py b/tests/auto_agents/controller/test_master_ci_summarize.py new file mode 100644 index 000000000..b9c07e558 --- /dev/null +++ b/tests/auto_agents/controller/test_master_ci_summarize.py @@ -0,0 +1,231 @@ +"""Tests for the deterministic CI summarizer (Phase 1j). + +Validates summarize_ci_status orchestration: +- Forgejo combined-status → CISummary dict shape (parses as + CISummary V1 contract). +- Gate→nox-session resolution honors workflow naming convention. +- Composite gates (security_scan) get composite_findings rows. +- Failed log fetches degrade to log-fetch-failed CIFailure rather than + raising. +- Unknown gate contexts fall through to NoParserAvailable cleanly. +""" +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from tools.controller.contracts.v1 import CISummary +from tools.controller.master import summarize_ci_status + + +# ─── fixtures ─────────────────────────────────────────────────────── + + +def _forgejo_status(statuses, *, overall="failure") -> dict: + return {"state": overall, "statuses": statuses} + + +def _gate(*, context: str, state: str, target_url: str | None = None) -> dict: + return {"context": context, "state": state, "target_url": target_url} + + +# ─── happy path ──────────────────────────────────────────────────── + + +class TestHappyPath: + def test_basic_summary(self): + statuses = [ + _gate(context="CI / lint", state="success"), + _gate(context="CI / typecheck", state="success"), + ] + log_fetcher = lambda _: "" + summary = summarize_ci_status( + head_sha="abc", forgejo_status=_forgejo_status(statuses, overall="success"), + log_fetcher=log_fetcher, + ) + assert summary["overall_state"] == "success" + assert summary["gates_total"] == 2 + assert summary["gates_passed"] == 2 + assert summary["gates_failed"] == 0 + # No failures → no parser invocations. + for g in summary["gates"]: + assert g["failure"] is None + + def test_failing_lint_invokes_ruff(self): + statuses = [_gate(context="CI / lint", state="failure")] + log = "tools/x.py:1:1: F401 unused\nFound 1 errors.\n" + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: log, + ) + assert summary["gates_failed"] == 1 + failure = summary["gates"][0]["failure"] + assert failure is not None + assert failure["parser_used"] == "ruff" + assert failure["error_class"] == "F401" + assert len(failure["findings"]) == 1 + + def test_parses_as_v1_contract(self): + statuses = [_gate(context="CI / lint", state="failure")] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: "tools/x.py:1:1: F401 unused\nFound 1 errors.\n", + ) + # The CISummary contract must accept the assembled dict. + parsed = CISummary.model_validate(summary) + assert parsed.gates_failed == 1 + assert parsed.gates[0].failure.parser_used == "ruff" + + def test_observed_at_default_now(self): + before = datetime.now(timezone.utc) + summary = summarize_ci_status( + head_sha="abc", forgejo_status=_forgejo_status([], overall="success"), + log_fetcher=lambda _: "", + ) + after = datetime.now(timezone.utc) + assert before <= summary["observed_at"] <= after + + +# ─── composite gates ─────────────────────────────────────────────── + + +class TestCompositeGates: + def test_security_scan_composite_findings(self): + statuses = [_gate(context="CI / security_scan", state="failure")] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: "fake security log\n", + ) + failure = summary["gates"][0]["failure"] + # bandit is the head parser (first in multi:); semgrep + vulture + # nest as composite_findings. + assert failure["parser_used"] == "bandit" + assert len(failure["composite_findings"]) == 2 + names = [c["parser_used"] for c in failure["composite_findings"]] + assert names == ["semgrep", "vulture"] + + +# ─── error handling ──────────────────────────────────────────────── + + +class TestErrorHandling: + def test_log_fetch_returns_none(self): + statuses = [_gate(context="CI / lint", state="failure")] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: None, + ) + failure = summary["gates"][0]["failure"] + assert failure["error_class"] == "log-fetch-failed" + assert failure["raw_log_excerpt"] == "" + + def test_log_fetch_raises_handled(self): + statuses = [_gate(context="CI / lint", state="failure")] + + def boom(_): + raise RuntimeError("network down") + + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=boom, + ) + # Failure recorded but loop didn't crash. + assert summary["gates"][0]["failure"]["error_class"] == "log-fetch-failed" + + def test_unknown_gate_no_parser(self): + statuses = [_gate(context="CI / mystery_gate", state="failure")] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: "log content", + ) + failure = summary["gates"][0]["failure"] + assert failure["error_class"] == "NoParserAvailable" + + def test_forgejo_status_none(self): + summary = summarize_ci_status( + head_sha="abc", forgejo_status=None, + log_fetcher=lambda _: pytest.fail("should not be called"), + ) + assert summary["overall_state"] == "unknown" + assert summary["gates"] == [] + + def test_parameterized_gate_name_normalized(self): + """Forgejo can emit "unit_tests-3.13" for matrix jobs.""" + statuses = [_gate(context="CI / unit_tests-3.13", state="failure")] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: "Failing scenarios:\n x.feature:5 Scenario: bar\n0 features passed, 1 failed, 0 skipped\n0 scenarios passed, 1 failed, 0 skipped\n", + ) + failure = summary["gates"][0]["failure"] + assert failure["parser_used"] == "behave" + + def test_skip_coverage_session_no_parser(self): + # benchmark is in NOX_SESSIONS_SKIP_COVERAGE; should not parse. + statuses = [_gate(context="CI / benchmark", state="failure")] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: pytest.fail("should not fetch log"), + ) + # benchmark isn't in NOX_SESSION_TO_PARSER, so gate→session + # returns None → NoParserAvailable. + failure = summary["gates"][0]["failure"] + assert failure["error_class"] == "NoParserAvailable" + + +# ─── parser_versions aggregate ───────────────────────────────────── + + +class TestParserVersions: + def test_versions_collected_per_parser(self): + statuses = [ + _gate(context="CI / lint", state="failure"), + _gate(context="CI / typecheck", state="failure"), + ] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: "Found 0 errors.\n", + ) + # Both ruff + pyright versions appear. + assert "ruff" in summary["parser_versions"] + assert "pyright" in summary["parser_versions"] + assert summary["parser_versions"]["ruff"] == "v1" + + def test_stub_version_recorded(self): + statuses = [_gate(context="CI / complexity", state="failure")] + summary = summarize_ci_status( + head_sha="abc", + forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: "radon log", + ) + # complexity → radon (currently a stub). + assert summary["parser_versions"].get("radon") == "v1-stub" + + +# ─── gate count aggregates ───────────────────────────────────────── + + +class TestGateCounts: + def test_mixed_statuses(self): + statuses = [ + _gate(context="CI / lint", state="success"), + _gate(context="CI / typecheck", state="failure"), + _gate(context="CI / unit_tests", state="pending"), + _gate(context="CI / build", state="error"), + ] + summary = summarize_ci_status( + head_sha="abc", forgejo_status=_forgejo_status(statuses), + log_fetcher=lambda _: "", + ) + assert summary["gates_passed"] == 1 + assert summary["gates_failed"] == 2 # failure + error + assert summary["gates_pending"] == 1 diff --git a/tools/controller/ci_summary_parsers/_base.py b/tools/controller/ci_summary_parsers/_base.py new file mode 100644 index 000000000..ce4724c4a --- /dev/null +++ b/tools/controller/ci_summary_parsers/_base.py @@ -0,0 +1,76 @@ +"""Shared types + helpers for per-CI-tool parsers. + +Every parser module exposes: +- ``PARSER_VERSION: str`` — bumped when output semantics change. +- ``parse(log: str, gate_name: str) -> ParserResult`` + +Parsers are deterministic Python — no LLM. They take the gate's raw +log text + the gate name, and return structured findings that fit the +V1 ``CIFailure`` contract. Composite gates (e.g., nox security_scan = +bandit + semgrep + vulture) get parsed by an orchestrator that calls +each sub-parser and wraps the results in ``CIFailure.composite_findings``. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..contracts.v1 import CIFailureFinding, FailedAssertion, FileLocation + + +# Default log-excerpt budget (V1 CIFailure.raw_log_excerpt max_length). +DEFAULT_LOG_EXCERPT_CHARS = 16_384 + + +@dataclass(frozen=True) +class ParserResult: + """One parser's structured output for one gate's log. + + ``log_excerpt`` is the parser's selection of the most-relevant + log lines (smart-around-first-error, or trailing N lines). It MUST + fit within ``DEFAULT_LOG_EXCERPT_CHARS`` — the V1 contract enforces + this as max_length on the CIFailure field. + + ``log_excerpt_lines`` is the count of newline-separated lines in + the excerpt, used by the contract field of the same name. + """ + + error_class: str + summary_line: str + findings: list[CIFailureFinding] = field(default_factory=list) + failing_locations: list[FileLocation] = field(default_factory=list) + failed_assertions: list[FailedAssertion] = field(default_factory=list) + log_excerpt: str = "" + log_excerpt_lines: int = 0 + + +def select_log_excerpt(log: str, *, max_chars: int = DEFAULT_LOG_EXCERPT_CHARS, + tail_lines: int = 80) -> tuple[str, int]: + """Pick the most-relevant excerpt from a log. + + Strategy: prefer the LAST ``tail_lines`` (most test/build tools + emit failure context at the end). If the tail exceeds ``max_chars``, + truncate from the head of the tail (keep the bottom). + + Returns ``(excerpt, line_count)``. An empty log produces ``("", 0)``. + """ + if not log: + return "", 0 + lines = log.splitlines() + if len(lines) <= tail_lines: + excerpt = log + else: + excerpt = "\n".join(lines[-tail_lines:]) + if len(excerpt) > max_chars: + excerpt = excerpt[-max_chars:] + # Trim a partial leading line to keep it clean. + nl = excerpt.find("\n") + if 0 < nl < 200: + excerpt = excerpt[nl + 1:] + return excerpt, excerpt.count("\n") + (1 if excerpt and not excerpt.endswith("\n") else 0) + + +__all__ = [ + "DEFAULT_LOG_EXCERPT_CHARS", + "ParserResult", + "select_log_excerpt", +] diff --git a/tools/controller/ci_summary_parsers/_registry.py b/tools/controller/ci_summary_parsers/_registry.py new file mode 100644 index 000000000..c7503fe82 --- /dev/null +++ b/tools/controller/ci_summary_parsers/_registry.py @@ -0,0 +1,88 @@ +"""Parser registry — resolve a parser name to a (parser_version, +parse_function) pair. + +Real parsers ship their own module under ``ci_summary_parsers/``. For +the not-yet-shipped tools, the registry returns a stub from +``_stub.make_stub(name)``. The startup coverage validator +``validate_parser_coverage()`` flags any name in EXPECTED_PARSERS +that still resolves to a stub so the operator can prioritize. +""" +from __future__ import annotations + +import importlib +import logging +from collections.abc import Callable +from dataclasses import dataclass + +from . import EXPECTED_PARSERS, NOX_SESSION_TO_PARSER +from . import _stub +from ._base import ParserResult + +logger = logging.getLogger(__name__) + + +# Parsers that ship in Phase 1j (real implementations). +_REAL_PARSERS: frozenset[str] = frozenset({"ruff", "pyright", "behave"}) + + +@dataclass(frozen=True) +class ResolvedParser: + name: str + version: str + parse: Callable[[str, str], ParserResult] + is_stub: bool + + +def resolve(parser_name: str) -> ResolvedParser: + """Look up the parser for ``parser_name``. Returns a stub if the + real module isn't shipped yet (logs at WARNING level so operators + notice the parser gap).""" + if parser_name in _REAL_PARSERS: + try: + mod = importlib.import_module( + f"tools.controller.ci_summary_parsers.{parser_name}", + ) + return ResolvedParser( + name=parser_name, + version=getattr(mod, "PARSER_VERSION", "v1"), + parse=mod.parse, + is_stub=False, + ) + except ImportError: + logger.warning( + "real parser %s declared but import failed; using stub", + parser_name, + ) + return ResolvedParser( + name=parser_name, + version=_stub.PARSER_VERSION, + parse=_stub.make_stub(parser_name), + is_stub=True, + ) + + +def resolve_for_nox_session(nox_session: str) -> list[ResolvedParser]: + """Map nox session → ordered list of parsers (most sessions have + one; composite security_scan has three). + + Raises KeyError if the session has no mapping (forces operator to + update NOX_SESSION_TO_PARSER explicitly).""" + entry = NOX_SESSION_TO_PARSER[nox_session] # KeyError is intentional + if entry.startswith("multi:"): + names = entry.split(":", 1)[1].split(",") + return [resolve(n.strip()) for n in names if n.strip()] + return [resolve(entry)] + + +def validate_parser_coverage() -> dict[str, bool]: + """Return a {parser_name: is_real} map for every parser in + EXPECTED_PARSERS. Operators can plot this to see parser gaps.""" + return {p: not resolve(p).is_stub for p in sorted(EXPECTED_PARSERS)} + + +__all__ = [ + "ResolvedParser", + "resolve", + "resolve_for_nox_session", + "validate_parser_coverage", +] diff --git a/tools/controller/ci_summary_parsers/_stub.py b/tools/controller/ci_summary_parsers/_stub.py new file mode 100644 index 000000000..1ca350fa5 --- /dev/null +++ b/tools/controller/ci_summary_parsers/_stub.py @@ -0,0 +1,55 @@ +"""Stub parser for tools whose deterministic parser isn't shipped yet. + +Returns a structured CIFailure-shape result that carries the raw log +excerpt + a ``parser-pending`` marker. The implementer's prompt can +still surface the failing log; the operator-side coverage dashboard +flags these as parser gaps to prioritize. + +Phase 1j ships ruff/pyright/behave; Phase 1j+ ships the rest. Until +then, these tools use this stub: +- robot_framework (integration_tests, e2e_tests) +- slipcover (coverage_report) +- bandit, semgrep, vulture (security_scan composite) +- radon (complexity) +- build (build) +""" +from __future__ import annotations + +from ..contracts.v1 import CIFailureFinding +from ._base import ParserResult, select_log_excerpt + +PARSER_VERSION = "v1-stub" + + +def make_stub(parser_name: str): + """Factory: returns a ``parse(log, gate_name)`` for the given + parser name. Each stub identifies itself in ``error_class``.""" + + def parse(log: str, gate_name: str) -> ParserResult: + excerpt, lines = select_log_excerpt(log) + finding = CIFailureFinding( + error_class=f"parser-pending-{parser_name}", + summary=( + f"Structured parser for '{parser_name}' is not yet " + f"implemented; falling back to raw log excerpt." + )[:512], + location=None, + severity="error", + ) + return ParserResult( + error_class=f"parser-pending-{parser_name}", + summary_line=( + f"{parser_name}: parser pending; see raw_log_excerpt" + )[:200], + findings=[finding], + failing_locations=[], + failed_assertions=[], + log_excerpt=excerpt, + log_excerpt_lines=lines, + ) + + parse.__name__ = f"parse_stub_{parser_name}" + return parse + + +__all__ = ["PARSER_VERSION", "make_stub"] diff --git a/tools/controller/ci_summary_parsers/behave.py b/tools/controller/ci_summary_parsers/behave.py new file mode 100644 index 000000000..9a26bbcc6 --- /dev/null +++ b/tools/controller/ci_summary_parsers/behave.py @@ -0,0 +1,143 @@ +"""Behave parser — handles ``nox -s unit_tests``. + +Behave's CLI emits a tree-style trace per scenario: + + Feature: My feature # tests/features/foo.feature:3 + Scenario: My scenario # tests/features/foo.feature:10 + Given foo # passed + When bar # passed + Then baz # failed: AssertionError: expected 1 == 2 + File "tests/steps/baz_steps.py", line 10, in step_then + assert 1 == 2, "expected 1 == 2" + AssertionError: expected 1 == 2 + + Failing scenarios: + tests/features/foo.feature:10 Scenario: My scenario + + 0 features passed, 1 failed, 0 skipped + 0 scenarios passed, 1 failed, 0 skipped + 2 steps passed, 1 failed, 1 skipped + +The parser extracts each failing scenario as a ``FailedAssertion`` ++ a ``CIFailureFinding`` so the implementer can navigate to the +right feature file + step file. +""" +from __future__ import annotations + +import re + +from ..contracts.v1 import CIFailureFinding, FailedAssertion, FileLocation +from ._base import ParserResult, select_log_excerpt + +PARSER_VERSION = "v1" + +# " tests/features/foo.feature:10 Scenario: My scenario" +_FAILING_SCENARIO_RE = re.compile( + r"^\s+(?P\S+\.feature):(?P\d+)\s+Scenario:\s+(?P.+?)\s*$", + re.MULTILINE, +) +# "1 feature passed, 0 failed, 0 skipped" +_FEATURE_SUMMARY_RE = re.compile( + r"(\d+)\s+features?\s+passed,\s+(\d+)\s+failed,\s+(\d+)\s+skipped", +) +_SCENARIO_SUMMARY_RE = re.compile( + r"(\d+)\s+scenarios?\s+passed,\s+(\d+)\s+failed,\s+(\d+)\s+skipped", +) +# AssertionError line OR Exception line. +_ASSERT_RE = re.compile( + r"^\s*(?PAssertionError|Exception|.+Error):\s*(?P.+?)\s*$", + re.MULTILINE, +) + + +def parse(log: str, gate_name: str) -> ParserResult: + findings: list[CIFailureFinding] = [] + locations: list[FileLocation] = [] + assertions: list[FailedAssertion] = [] + + failing_scenarios = list(_FAILING_SCENARIO_RE.finditer(log)) + + # Map scenario → an excerpt of the surrounding log so we can lift + # the assertion line for the FailedAssertion. + for m in failing_scenarios: + loc = FileLocation( + file_path=m["feature"], + line_range=f"{m['line']}-{m['line']}", + function_or_test=m["name"][:120], + ) + findings.append(CIFailureFinding( + error_class="ScenarioFailed", + summary=f"Scenario failed: {m['name'][:200]}"[:512], + location=loc, + severity="error", + )) + locations.append(loc) + + # Pull the first matching assertion globally — multi-assertion + # extraction would over-fit on rare logs. The raw excerpt carries + # the rest of the context. + for am in _ASSERT_RE.finditer(log): + exc = am["exc"].strip() + if exc.endswith("Error"): + msg = am["msg"].strip()[:512] + # Best-effort context: 10 lines around the match. + excerpt = _excerpt_around(log, am.start(), 10) + assertions.append(FailedAssertion( + test_name=(failing_scenarios[0]["name"][:120] + if failing_scenarios else "(unknown)"), + expected=None, + actual=None, + assertion_excerpt=(f"{exc}: {msg}\n\n{excerpt}")[:4096], + )) + break + + summary_line = _summary_line(log, n_failing=len(failing_scenarios)) + excerpt, lines = select_log_excerpt(log) + error_class = "BehaveFailure" if findings else "BehaveClean" + + return ParserResult( + error_class=error_class, + summary_line=summary_line, + findings=findings, + failing_locations=locations, + failed_assertions=assertions, + log_excerpt=excerpt, + log_excerpt_lines=lines, + ) + + +def _summary_line(log: str, *, n_failing: int) -> str: + feat = _FEATURE_SUMMARY_RE.search(log) + sce = _SCENARIO_SUMMARY_RE.search(log) + if feat and sce: + return ( + f"Behave: {feat.group(2)} feature(s) failed, " + f"{sce.group(2)} scenario(s) failed" + )[:200] + if n_failing: + return f"Behave: {n_failing} scenario(s) failed"[:200] + return "Behave: no failing scenarios parsed" + + +def _excerpt_around(log: str, offset: int, lines_each_side: int) -> str: + # Walk N newlines back, N newlines forward. + before = log.rfind("\n", 0, offset) + for _ in range(lines_each_side): + if before <= 0: + before = 0 + break + before = log.rfind("\n", 0, before) + if before == -1: + before = 0 + break + after = offset + for _ in range(lines_each_side): + nxt = log.find("\n", after + 1) + if nxt == -1: + after = len(log) + break + after = nxt + return log[before:after].strip("\n")[:2048] + + +__all__ = ["PARSER_VERSION", "parse"] diff --git a/tools/controller/ci_summary_parsers/pyright.py b/tools/controller/ci_summary_parsers/pyright.py new file mode 100644 index 000000000..03fb1d701 --- /dev/null +++ b/tools/controller/ci_summary_parsers/pyright.py @@ -0,0 +1,117 @@ +"""Pyright parser — handles ``nox -s typecheck``. + +Pyright's CLI emits one diagnostic per line: + + /abs/path/to/file.py:LINE:COL - error: human description (ruleName) + /abs/path/to/file.py:LINE:COL - warning: ... + 1 error, 0 warnings, 0 informations + +Some Pyright versions use absolute paths; we normalize to the +project-relative tail when a known segment (``tools/``, ``tests/``, +``src/``) is in the path. Otherwise the absolute path is kept. +""" +from __future__ import annotations + +import re + +from ..contracts.v1 import CIFailureFinding, FileLocation +from ._base import ParserResult, select_log_excerpt + +PARSER_VERSION = "v1" + +_DIAG_RE = re.compile( + r"^(?P\S+?):(?P\d+):(?P\d+)\s+-\s+" + r"(?Perror|warning|information)\s*:\s+" + r"(?P.+?)\s*$", + re.MULTILINE, +) +# Some Pyright versions emit findings on TWO lines (path/loc then +# indented message); we don't try to parse those (rare). The summary +# line is canonical: +_SUMMARY_RE = re.compile( + r"(\d+)\s+errors?,\s+(\d+)\s+warnings?,\s+(\d+)\s+informations?", +) + +_PATH_NORMALIZE_SEGMENTS = ("/tools/", "/tests/", "/src/", "/.opencode/") + + +def parse(log: str, gate_name: str) -> ParserResult: + findings: list[CIFailureFinding] = [] + locations: list[FileLocation] = [] + + for m in _DIAG_RE.finditer(log): + sev_raw = m["severity"] + if sev_raw == "error": + severity = "error" + error_class = _extract_rule_name(m["message"]) or "PyrightError" + elif sev_raw == "warning": + severity = "warning" + error_class = _extract_rule_name(m["message"]) or "PyrightWarning" + else: + severity = "info" + error_class = "PyrightInfo" + loc = FileLocation( + file_path=_normalize_path(m["file"]), + line_range=f"{m['line']}-{m['line']}", + ) + findings.append(CIFailureFinding( + error_class=error_class, + summary=m["message"].strip()[:512], + location=loc, + severity=severity, + )) + if severity == "error": + locations.append(loc) + + summary_line = _summary_line(log, n_findings=len(findings)) + excerpt, lines = select_log_excerpt(log) + error_class = _aggregate_error_class(findings) + + return ParserResult( + error_class=error_class, + summary_line=summary_line, + findings=findings, + failing_locations=locations, + failed_assertions=[], + log_excerpt=excerpt, + log_excerpt_lines=lines, + ) + + +def _extract_rule_name(message: str) -> str | None: + """Pyright wraps the rule name in trailing parens. Pull it out.""" + m = re.search(r"\((report[A-Za-z0-9]+)\)\s*$", message) + return m.group(1) if m else None + + +def _normalize_path(path: str) -> str: + """Strip leading container path so the implementer can match the + workspace tree.""" + for seg in _PATH_NORMALIZE_SEGMENTS: + idx = path.find(seg) + if idx >= 0: + return path[idx + 1:] + return path + + +def _summary_line(log: str, *, n_findings: int) -> str: + m = _SUMMARY_RE.search(log) + if m: + return ( + f"Pyright: {m.group(1)} errors, {m.group(2)} warnings, " + f"{m.group(3)} informations" + )[:200] + return f"Pyright: {n_findings} diagnostic(s) parsed"[:200] + + +def _aggregate_error_class(findings: list[CIFailureFinding]) -> str: + errors = [f for f in findings if f.severity == "error"] + if not errors: + return "PyrightClean" + classes = {f.error_class for f in errors} + if len(classes) == 1: + return next(iter(classes)) + return "PyrightMixed" + + +__all__ = ["PARSER_VERSION", "parse"] diff --git a/tools/controller/ci_summary_parsers/ruff.py b/tools/controller/ci_summary_parsers/ruff.py new file mode 100644 index 000000000..44b61fc88 --- /dev/null +++ b/tools/controller/ci_summary_parsers/ruff.py @@ -0,0 +1,109 @@ +"""Ruff parser — handles ``nox -s lint`` and ``nox -s format``. + +Ruff's CLI prints one finding per line in the format: + + path/to/file.py:LINE:COL: RULECODE human description + +Followed by a trailing ``Found N errors.`` summary line. Format mode +(`ruff format --check`) emits a different shape: + + Would reformat: path/to/file.py + 1 file would be reformatted + +Both shapes are absorbed below. Findings are emitted as +``CIFailureFinding`` rows so the implementer's prompt can render +"rule + file:line" with one entry per failure. +""" +from __future__ import annotations + +import re + +from ..contracts.v1 import CIFailureFinding, FileLocation +from ._base import ParserResult, select_log_excerpt + +PARSER_VERSION = "v1" + +# Examples: +# tools/controller/master/loop.py:42:5: F401 `os` imported but unused +# tests/x.py:1:1: E501 line too long (90 > 88 characters) +_LINT_RE = re.compile( + r"^(?P[^\s:][^:]*):(?P\d+):(?P\d+):\s+" + r"(?P[A-Z]+\d+)\s+(?P.+)$", + re.MULTILINE, +) +# Examples: ``Would reformat: tools/foo.py`` +_FORMAT_RE = re.compile(r"^Would reformat:\s+(?P\S+)\s*$", re.MULTILINE) +# Summary lines we extract for summary_line in priority order. +_SUMMARY_PATTERNS = [ + re.compile(r"^Found (\d+) errors?\.?$", re.MULTILINE), + re.compile(r"^(\d+) files? would be reformatted$", re.MULTILINE), +] + + +def parse(log: str, gate_name: str) -> ParserResult: + findings: list[CIFailureFinding] = [] + locations: list[FileLocation] = [] + + for m in _LINT_RE.finditer(log): + loc = FileLocation( + file_path=m["file"], + line_range=f"{m['line']}-{m['line']}", + ) + findings.append(CIFailureFinding( + error_class=m["code"], + summary=m["message"].strip()[:512], + location=loc, + severity="error", + )) + locations.append(loc) + + for m in _FORMAT_RE.finditer(log): + loc = FileLocation(file_path=m["file"]) + findings.append(CIFailureFinding( + error_class="format", + summary="file would be reformatted", + location=loc, + severity="warning", + )) + locations.append(loc) + + summary_line = _pick_summary(log, fallback=f"{len(findings)} ruff findings") + excerpt, lines = select_log_excerpt(log) + error_class = _aggregate_error_class(findings) + + return ParserResult( + error_class=error_class, + summary_line=summary_line[:200], + findings=findings, + failing_locations=locations, + failed_assertions=[], + log_excerpt=excerpt, + log_excerpt_lines=lines, + ) + + +def _pick_summary(log: str, *, fallback: str) -> str: + for pat in _SUMMARY_PATTERNS: + matches = pat.findall(log) + if matches: + # Re-render: the regex captured the count. + n = matches[-1] + if pat is _SUMMARY_PATTERNS[0]: + return f"Ruff found {n} error(s)" + return f"Ruff: {n} file(s) need reformat" + return fallback + + +def _aggregate_error_class(findings: list[CIFailureFinding]) -> str: + """If all findings share one error_class, surface it. Else use a + composite marker so the implementer's prompt shows a sensible + top-level error_class on the CIFailure.""" + if not findings: + return "RuffEmpty" + classes = {f.error_class for f in findings} + if len(classes) == 1: + return next(iter(classes)) + return "RuffMixed" + + +__all__ = ["PARSER_VERSION", "parse"] diff --git a/tools/controller/master/__init__.py b/tools/controller/master/__init__.py index 2fdc3c3fe..5ae2c1b4b 100644 --- a/tools/controller/master/__init__.py +++ b/tools/controller/master/__init__.py @@ -71,6 +71,10 @@ from .backfill import ( has_backfill_run, run_startup_backfill, ) +from .ci_summarize import ( + LogFetcher, + summarize_ci_status, +) from .prefetch import ( GetPRDetailsCallback, GetPRDiffCallback, @@ -166,4 +170,7 @@ __all__ = [ "build_implementer_input", "build_reviewer_input", "make_prefetch_callback", + # CI summarizer (Phase 1j) + "LogFetcher", + "summarize_ci_status", ] diff --git a/tools/controller/master/ci_summarize.py b/tools/controller/master/ci_summarize.py new file mode 100644 index 000000000..24d45dcb9 --- /dev/null +++ b/tools/controller/master/ci_summarize.py @@ -0,0 +1,314 @@ +"""Deterministic CI summarizer — builds CISummary from Forgejo status. + +The implementer/reviewer prompt prefetch needs a CISummary V1 dict for +the head_sha. This module takes the Forgejo combined-status response +(``{"state": "...", "statuses": [...]}``) plus a log fetcher, and +returns a fully-populated CISummary covering every gate. + +Per-gate flow: +1. Map the Forgejo gate context (e.g., "CI / lint") → nox session + name. The job names in ``.forgejo/workflows/*.yml`` ARE the nox + session names by convention, so a simple suffix-after-/ extraction + works. Unknown contexts pass through as ``UnknownGate`` and skip + the parser. +2. Look up the nox session in NOX_SESSION_TO_PARSER → one or more + parser names. +3. For each parser, fetch the log (via the injected ``log_fetcher``), + run ``parse(log, gate_name)``, and wrap the result in a + GateResult.failure CIFailure dict. +4. For composite gates (security_scan = bandit + semgrep + vulture), + the outer CIFailure carries the FIRST parser's structure + each + sub-parser's result goes into ``composite_findings``. + +The summarizer never raises — log fetch failures degrade to a +``CIFailure`` with ``error_class='log-fetch-failed'`` so the +implementer at least sees the failure existed. +""" +from __future__ import annotations + +import logging +from collections.abc import Callable +from datetime import datetime, timezone +from typing import Any + +from ..ci_summary_parsers import NOX_SESSION_TO_PARSER, NOX_SESSIONS_SKIP_COVERAGE +from ..ci_summary_parsers._registry import ( + ResolvedParser, + resolve_for_nox_session, +) + +logger = logging.getLogger(__name__) + + +# log_fetcher(gate_name) -> str | None; production wires this to a +# Forgejo job-log fetcher. None means the log was unreachable. +LogFetcher = Callable[[str], str | None] + + +# Forgejo status state → CISummary GateResult.status mapping. +_FORGEJO_STATE_TO_GATE_STATUS = { + "success": "passed", + "failure": "failed", + "error": "error", + "pending": "pending", + "warning": "passed", # advisory: treat as passed + None: "pending", +} + + +def summarize_ci_status( + *, head_sha: str, forgejo_status: dict | None, log_fetcher: LogFetcher, + observed_at: datetime | None = None, +) -> dict: + """Build a CISummary-shape dict from the Forgejo combined-status. + + Args: + head_sha: the commit SHA the status covers. + forgejo_status: response body from + ``GET /repos/{owner}/{repo}/commits/{sha}/status``. + Shape: ``{"state": str, "statuses": [{"context": str, + "state": str, "target_url": str}]}``. None for + "couldn't fetch" — produces a unknown-state summary. + log_fetcher: callable that takes the gate context and returns + the raw log text (or None if unreachable). Production + wires this via Forgejo's job-log endpoint. + observed_at: timestamp the status was fetched. Defaults to + now(UTC). + + Returns the CISummary V1-shape dict (a CISummary contract parses + it after the caller validates). + """ + when = observed_at or datetime.now(timezone.utc) + if forgejo_status is None: + return _unknown_summary(head_sha=head_sha, observed_at=when) + + overall = _FORGEJO_OVERALL_STATE_MAPPING.get( + forgejo_status.get("state"), "unknown", + ) + raw_statuses = forgejo_status.get("statuses") or [] + statuses = [s for s in raw_statuses if isinstance(s, dict)] + + gates: list[dict] = [] + parser_versions: dict[str, str] = {} + for s in statuses: + gate, versions_seen = _build_gate(s, log_fetcher) + gates.append(gate) + parser_versions.update(versions_seen) + + counts = _count_gates(gates) + return { + "summary_version": "V1", + "head_sha": head_sha, + "observed_at": when, + "overall_state": overall, + "gates": gates, + "gates_total": counts["total"], + "gates_passed": counts["passed"], + "gates_failed": counts["failed"], + "gates_skipped": counts["skipped"], + "gates_pending": counts["pending"], + "parser_versions": parser_versions, + } + + +# Forgejo combined-status states → CISummary.overall_state. +_FORGEJO_OVERALL_STATE_MAPPING: dict[str | None, str] = { + "success": "success", + "failure": "failure", + "error": "error", + "pending": "pending", + None: "unknown", +} + + +def _unknown_summary(*, head_sha: str, observed_at: datetime) -> dict: + return { + "summary_version": "V1", + "head_sha": head_sha, + "observed_at": observed_at, + "overall_state": "unknown", + "gates": [], + "gates_total": 0, + "gates_passed": 0, + "gates_failed": 0, + "gates_skipped": 0, + "gates_pending": 0, + "parser_versions": {}, + } + + +def _build_gate( + status: dict, log_fetcher: LogFetcher, +) -> tuple[dict, dict[str, str]]: + """Build one GateResult dict + return parser versions used.""" + context = status.get("context") or "(unknown)" + state_raw = status.get("state") + gate_status = _FORGEJO_STATE_TO_GATE_STATUS.get(state_raw, "pending") + target_url = status.get("target_url") + severity = "info" if gate_status in {"passed", "skipped"} else "error" + + gate: dict[str, Any] = { + "name": context, + "status": gate_status, + "severity": severity, + "target_url": target_url, + "duration_seconds": None, + "failure": None, + } + parser_versions: dict[str, str] = {} + + # Only parse failures/errors. Passed/pending/skipped don't need a + # log fetch — they have nothing to report. + if gate_status not in {"failed", "error"}: + return gate, parser_versions + + nox_session = _gate_to_nox_session(context) + if nox_session is None or nox_session in NOX_SESSIONS_SKIP_COVERAGE: + gate["failure"] = _no_parser_failure(context) + return gate, parser_versions + + try: + parsers = resolve_for_nox_session(nox_session) + except KeyError: + logger.warning( + "no parser map for nox session %r (gate=%r)", + nox_session, context, + ) + gate["failure"] = _no_parser_failure(context) + return gate, parser_versions + + log = _safe_fetch(log_fetcher, context) + if log is None: + gate["failure"] = _log_fetch_failed_failure(parsers, context) + for p in parsers: + parser_versions[p.name] = p.version + return gate, parser_versions + + gate["failure"] = _build_failure(parsers, log, context, parser_versions) + return gate, parser_versions + + +def _gate_to_nox_session(context: str) -> str | None: + """Forgejo gate context → nox session name. + + Convention: `` / `` (e.g., "CI / lint"). + The job name IS the nox session name by repo convention. If the + name doesn't match any known session, return None and let the + summarizer fall through to NoParserAvailable.""" + if not context: + return None + tail = context.rsplit("/", 1)[-1].strip() + if tail in NOX_SESSION_TO_PARSER: + return tail + # Also support the parameterized form e.g. "unit_tests-3.13". + base = tail.split("-", 1)[0] + if base in NOX_SESSION_TO_PARSER: + return base + return None + + +def _safe_fetch(log_fetcher: LogFetcher, context: str) -> str | None: + try: + return log_fetcher(context) + except Exception as exc: # noqa: BLE001 + logger.warning("log fetch raised for %s: %s", context, exc) + return None + + +def _no_parser_failure(context: str) -> dict: + return { + "parser_used": "(none)", + "parser_version": "n/a", + "error_class": "NoParserAvailable", + "summary_line": f"No parser available for {context!r}"[:200], + "findings": [], + "failing_locations": [], + "failed_assertions": [], + "raw_log_excerpt": "", + "log_excerpt_lines": 0, + "composite_findings": [], + } + + +def _log_fetch_failed_failure(parsers, context: str) -> dict: + parser_names = ", ".join(p.name for p in parsers) + return { + "parser_used": parser_names or "(unknown)", + "parser_version": "n/a", + "error_class": "log-fetch-failed", + "summary_line": f"Log fetch failed for {context!r}"[:200], + "findings": [], + "failing_locations": [], + "failed_assertions": [], + "raw_log_excerpt": "", + "log_excerpt_lines": 0, + "composite_findings": [], + } + + +def _build_failure( + parsers: list[ResolvedParser], log: str, context: str, + parser_versions: dict[str, str], +) -> dict: + """Build the CIFailure dict from one or more parsers' results. + + For a single-parser gate the result becomes the outer CIFailure. + For a composite gate (multi: prefix), the FIRST parser becomes + the outer + the rest go into composite_findings. + """ + results = [] + for p in parsers: + try: + res = p.parse(log, context) + parser_versions[p.name] = p.version + results.append((p, res)) + except Exception as exc: # noqa: BLE001 + logger.warning( + "parser %s raised on gate %r: %s", p.name, context, exc, + ) + # Fall through with no result for this parser. + + if not results: + return _log_fetch_failed_failure(parsers, context) + + head_parser, head_result = results[0] + failure = _result_to_failure(head_parser, head_result) + if len(results) > 1: + failure["composite_findings"] = [ + _result_to_failure(p, r) for p, r in results[1:] + ] + return failure + + +def _result_to_failure(parser: ResolvedParser, result) -> dict: + return { + "parser_used": parser.name, + "parser_version": parser.version, + "error_class": result.error_class, + "summary_line": result.summary_line, + "findings": [f.model_dump() for f in result.findings], + "failing_locations": [loc.model_dump() for loc in result.failing_locations], + "failed_assertions": [a.model_dump() for a in result.failed_assertions], + "raw_log_excerpt": result.log_excerpt, + "log_excerpt_lines": result.log_excerpt_lines, + "composite_findings": [], + } + + +def _count_gates(gates: list[dict]) -> dict[str, int]: + counts = {"total": len(gates), "passed": 0, "failed": 0, + "skipped": 0, "pending": 0} + for g in gates: + status = g.get("status") + if status == "passed": + counts["passed"] += 1 + elif status in {"failed", "error"}: + counts["failed"] += 1 + elif status == "skipped": + counts["skipped"] += 1 + elif status == "pending": + counts["pending"] += 1 + return counts + + +__all__ = ["LogFetcher", "summarize_ci_status"]