feat(controller): Phase 1j — deterministic CI summarizer + priority parsers

Replaces "ci_summary=None / failing_gates=[]" placeholders from Phase
1h with a real summarizer that maps Forgejo combined-status →
CISummary V1 dict by running per-tool deterministic parsers on each
failing gate's log.

Priority parsers shipped (cover lint/format/typecheck/unit_tests, the
4 most-failed gates):
- ruff      — F+E codes from `nox -s lint`; Would-reformat lines from
              `nox -s format`. Aggregates to single error_class when
              all findings share one code, else RuffMixed.
- pyright   — error/warning/information diagnostics; rule name pulled
              from trailing `(reportName)` parens. Abs-path
              normalization strips container prefixes.
- behave    — failing scenarios (file:line + name), AssertionError
              extraction. Feature/scenario summary line aggregation.

Stub parsers for not-yet-shipped tools (robot_framework, slipcover,
bandit, semgrep, vulture, radon, build): return a structured
CIFailure with error_class="parser-pending-{name}" + the raw log
excerpt. Operators see the failure; implementer still has log
context. Phase 1j+ replaces stubs with real parsers without changing
the gate-→-session map.

Components:
- _base.py        — ParserResult dataclass + select_log_excerpt()
                    (tail-N-lines smart selection within 16KB cap)
- _stub.py        — make_stub(name) factory for pending tools
- _registry.py    — resolve(parser_name) + resolve_for_nox_session()
                    + validate_parser_coverage()
- master/ci_summarize.py — summarize_ci_status(head_sha, status,
                    log_fetcher) orchestrator. Handles:
                    - composite multi: gates → CIFailure.composite_findings
                    - log_fetcher returning None → log-fetch-failed
                    - log_fetcher raising → caught + log-fetch-failed
                    - Unknown gate context → NoParserAvailable
                    - Forgejo state=None → unknown summary
                    - Parameterized matrix gates ("unit_tests-3.13")
                      → base session name resolution

Tests (+46 across 2 new files, 0 regressions):
- Per-parser canonical + empty + garbage input
- Registry resolution (real vs stub), coverage validator
- Summarizer V1 contract round-trip
- Composite security_scan composite_findings shape
- Error paths (None status, raising fetcher, unknown gate)
- Parser version aggregation across mixed gates

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-18 14:40:16 -04:00
parent 1b09a2054f
commit 3e269ce011
10 changed files with 1439 additions and 0 deletions
@@ -0,0 +1,299 @@
"""Tests for the per-tool CI summary parsers (Phase 1j).
Validates each parser:
- Extracts findings from canonical CLI output
- Populates summary_line, error_class, log_excerpt correctly
- Handles empty/garbage input without raising
"""
from __future__ import annotations
import pytest
from tools.controller.ci_summary_parsers import (
EXPECTED_PARSERS,
NOX_SESSION_TO_PARSER,
)
from tools.controller.ci_summary_parsers import _stub, behave, pyright, ruff
from tools.controller.ci_summary_parsers._base import select_log_excerpt
from tools.controller.ci_summary_parsers._registry import (
resolve,
resolve_for_nox_session,
validate_parser_coverage,
)
# ─── shared base helpers ────────────────────────────────────────────
class TestSelectLogExcerpt:
def test_empty_log(self):
excerpt, lines = select_log_excerpt("")
assert excerpt == ""
assert lines == 0
def test_short_log_returned_whole(self):
log = "line 1\nline 2\nline 3\n"
excerpt, lines = select_log_excerpt(log, tail_lines=80)
assert "line 1" in excerpt
assert lines == 3
def test_long_log_tail_kept(self):
log = "\n".join(f"line {i}" for i in range(200))
excerpt, lines = select_log_excerpt(log, tail_lines=10)
assert "line 199" in excerpt
assert "line 100" not in excerpt
assert lines == 10
def test_hard_char_cap_respected(self):
log = "x" * 50_000
excerpt, lines = select_log_excerpt(log, max_chars=1000)
assert len(excerpt) <= 1000
# ─── ruff ───────────────────────────────────────────────────────────
class TestRuffParser:
def test_lint_findings_parsed(self):
log = """\
tools/x.py:10:5: F401 `os` imported but unused
tools/y.py:20:1: E501 line too long (90 > 88 characters)
Found 2 errors.
"""
result = ruff.parse(log, "CI / lint")
assert len(result.findings) == 2
codes = [f.error_class for f in result.findings]
assert codes == ["F401", "E501"]
assert "Ruff found 2 error(s)" in result.summary_line
assert result.error_class == "RuffMixed"
def test_all_findings_same_class(self):
log = """\
a.py:1:1: F401 unused
b.py:2:1: F401 unused
Found 2 errors.
"""
result = ruff.parse(log, "lint")
assert result.error_class == "F401"
def test_format_findings_parsed(self):
log = """\
Would reformat: tools/a.py
Would reformat: tools/b.py
2 files would be reformatted
"""
result = ruff.parse(log, "format")
assert len(result.findings) == 2
assert all(f.error_class == "format" for f in result.findings)
assert "would be reformatted" in result.findings[0].summary
def test_empty_log(self):
result = ruff.parse("", "lint")
assert result.findings == []
assert result.error_class == "RuffEmpty"
def test_garbage_log_no_crash(self):
result = ruff.parse("totally not ruff output\nnope nope\n", "lint")
assert result.findings == []
# Falls back to "0 ruff findings" summary
assert "0" in result.summary_line or "ruff" in result.summary_line.lower()
def test_locations_populated(self):
log = "tools/x.py:42:10: F401 foo\n"
result = ruff.parse(log, "lint")
assert len(result.failing_locations) == 1
assert result.failing_locations[0].file_path == "tools/x.py"
assert result.failing_locations[0].line_range == "42-42"
# ─── pyright ───────────────────────────────────────────────────────
class TestPyrightParser:
def test_basic_diagnostic(self):
log = """\
/home/runner/work/repo/tools/foo.py:10:5 - error: Cannot find name 'x' (reportUnknownVariableType)
/home/runner/work/repo/tools/bar.py:1:1 - warning: trivial (reportUnusedImport)
1 errors, 1 warnings, 0 informations
"""
result = pyright.parse(log, "typecheck")
assert len(result.findings) == 2
errs = [f for f in result.findings if f.severity == "error"]
assert len(errs) == 1
assert errs[0].error_class == "reportUnknownVariableType"
assert "Pyright: 1 errors" in result.summary_line
def test_path_normalization(self):
log = "/abs/path/work/tools/foo.py:10:5 - error: msg (reportFoo)\n"
result = pyright.parse(log, "typecheck")
assert result.findings[0].location.file_path == "tools/foo.py"
def test_no_errors_only_warnings(self):
log = """\
/x/tools/foo.py:1:1 - warning: trivial (reportUnusedImport)
0 errors, 1 warnings, 0 informations
"""
result = pyright.parse(log, "typecheck")
# Errors-only locations → empty
assert result.failing_locations == []
assert result.error_class == "PyrightClean"
def test_empty_log(self):
result = pyright.parse("", "typecheck")
assert result.findings == []
assert result.error_class == "PyrightClean"
def test_two_errors_diff_classes(self):
log = """\
/x/tools/a.py:1:1 - error: a (reportFoo)
/x/tools/b.py:2:2 - error: b (reportBar)
2 errors, 0 warnings, 0 informations
"""
result = pyright.parse(log, "typecheck")
assert result.error_class == "PyrightMixed"
def test_two_errors_same_class(self):
log = """\
/x/tools/a.py:1:1 - error: a (reportFoo)
/x/tools/b.py:2:2 - error: b (reportFoo)
2 errors, 0 warnings, 0 informations
"""
result = pyright.parse(log, "typecheck")
assert result.error_class == "reportFoo"
# ─── behave ────────────────────────────────────────────────────────
class TestBehaveParser:
def test_failing_scenario_extracted(self):
log = """\
Feature: My feature
Scenario: My scenario
Then baz failed
Failing scenarios:
tests/features/foo.feature:10 Scenario: My scenario
0 features passed, 1 failed, 0 skipped
0 scenarios passed, 1 failed, 0 skipped
2 steps passed, 1 failed, 1 skipped
AssertionError: expected 1 == 2
"""
result = behave.parse(log, "unit_tests")
assert len(result.findings) == 1
assert result.findings[0].error_class == "ScenarioFailed"
assert "My scenario" in result.findings[0].summary
loc = result.findings[0].location
assert loc.file_path == "tests/features/foo.feature"
assert loc.line_range == "10-10"
assert "Behave: 1 feature(s) failed" in result.summary_line
def test_no_failing_scenarios(self):
log = """\
1 features passed, 0 failed, 0 skipped
1 scenarios passed, 0 failed, 0 skipped
"""
result = behave.parse(log, "unit_tests")
assert result.findings == []
assert result.error_class == "BehaveClean"
def test_empty_log(self):
result = behave.parse("", "unit_tests")
assert result.findings == []
def test_assertion_extracted(self):
log = """\
Failing scenarios:
tests/x.feature:5 Scenario: foo
AssertionError: my-message-here
0 features passed, 1 failed, 0 skipped
0 scenarios passed, 1 failed, 0 skipped
"""
result = behave.parse(log, "unit_tests")
assert len(result.failed_assertions) == 1
assert "my-message-here" in result.failed_assertions[0].assertion_excerpt
# ─── stub ──────────────────────────────────────────────────────────
class TestStubParser:
def test_stub_returns_pending_marker(self):
log = "any log content\nmore\n"
parse = _stub.make_stub("robot_framework")
result = parse(log, "integration_tests")
assert result.error_class == "parser-pending-robot_framework"
assert "parser pending" in result.summary_line
assert "any log content" in result.log_excerpt
assert len(result.findings) == 1
assert result.findings[0].severity == "error"
def test_each_stub_independently_named(self):
a = _stub.make_stub("bandit")
b = _stub.make_stub("vulture")
ra = a("log", "g")
rb = b("log", "g")
assert ra.error_class != rb.error_class
# ─── registry ──────────────────────────────────────────────────────
class TestRegistry:
def test_resolve_real_ruff(self):
r = resolve("ruff")
assert r.is_stub is False
assert r.name == "ruff"
assert r.version == "v1"
def test_resolve_real_pyright(self):
r = resolve("pyright")
assert r.is_stub is False
assert r.parse is pyright.parse
def test_resolve_real_behave(self):
r = resolve("behave")
assert r.is_stub is False
def test_resolve_pending_stub(self):
r = resolve("robot_framework")
assert r.is_stub is True
assert r.version == "v1-stub"
def test_resolve_for_nox_session_single(self):
parsers = resolve_for_nox_session("lint")
assert [p.name for p in parsers] == ["ruff"]
def test_resolve_for_nox_session_composite(self):
parsers = resolve_for_nox_session("security_scan")
names = [p.name for p in parsers]
assert names == ["bandit", "semgrep", "vulture"]
def test_resolve_for_unknown_session_raises(self):
with pytest.raises(KeyError):
resolve_for_nox_session("nope_not_a_session")
def test_validate_parser_coverage(self):
cov = validate_parser_coverage()
# Every name in EXPECTED_PARSERS shows up.
assert set(cov.keys()) == EXPECTED_PARSERS
# ruff/pyright/behave are real; rest are stubs.
assert cov["ruff"] is True
assert cov["pyright"] is True
assert cov["behave"] is True
assert cov["robot_framework"] is False
assert cov["bandit"] is False
# ─── expected coverage shape ───────────────────────────────────────
class TestExpectedCoverageShape:
def test_nox_session_map_only_references_known_parsers(self):
for parser in EXPECTED_PARSERS:
assert resolve(parser).name == parser
def test_security_scan_is_composite(self):
assert NOX_SESSION_TO_PARSER["security_scan"].startswith("multi:")
@@ -0,0 +1,231 @@
"""Tests for the deterministic CI summarizer (Phase 1j).
Validates summarize_ci_status orchestration:
- Forgejo combined-status → CISummary dict shape (parses as
CISummary V1 contract).
- Gate→nox-session resolution honors workflow naming convention.
- Composite gates (security_scan) get composite_findings rows.
- Failed log fetches degrade to log-fetch-failed CIFailure rather than
raising.
- Unknown gate contexts fall through to NoParserAvailable cleanly.
"""
from __future__ import annotations
from datetime import datetime, timezone
import pytest
from tools.controller.contracts.v1 import CISummary
from tools.controller.master import summarize_ci_status
# ─── fixtures ───────────────────────────────────────────────────────
def _forgejo_status(statuses, *, overall="failure") -> dict:
return {"state": overall, "statuses": statuses}
def _gate(*, context: str, state: str, target_url: str | None = None) -> dict:
return {"context": context, "state": state, "target_url": target_url}
# ─── happy path ────────────────────────────────────────────────────
class TestHappyPath:
def test_basic_summary(self):
statuses = [
_gate(context="CI / lint", state="success"),
_gate(context="CI / typecheck", state="success"),
]
log_fetcher = lambda _: ""
summary = summarize_ci_status(
head_sha="abc", forgejo_status=_forgejo_status(statuses, overall="success"),
log_fetcher=log_fetcher,
)
assert summary["overall_state"] == "success"
assert summary["gates_total"] == 2
assert summary["gates_passed"] == 2
assert summary["gates_failed"] == 0
# No failures → no parser invocations.
for g in summary["gates"]:
assert g["failure"] is None
def test_failing_lint_invokes_ruff(self):
statuses = [_gate(context="CI / lint", state="failure")]
log = "tools/x.py:1:1: F401 unused\nFound 1 errors.\n"
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: log,
)
assert summary["gates_failed"] == 1
failure = summary["gates"][0]["failure"]
assert failure is not None
assert failure["parser_used"] == "ruff"
assert failure["error_class"] == "F401"
assert len(failure["findings"]) == 1
def test_parses_as_v1_contract(self):
statuses = [_gate(context="CI / lint", state="failure")]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: "tools/x.py:1:1: F401 unused\nFound 1 errors.\n",
)
# The CISummary contract must accept the assembled dict.
parsed = CISummary.model_validate(summary)
assert parsed.gates_failed == 1
assert parsed.gates[0].failure.parser_used == "ruff"
def test_observed_at_default_now(self):
before = datetime.now(timezone.utc)
summary = summarize_ci_status(
head_sha="abc", forgejo_status=_forgejo_status([], overall="success"),
log_fetcher=lambda _: "",
)
after = datetime.now(timezone.utc)
assert before <= summary["observed_at"] <= after
# ─── composite gates ───────────────────────────────────────────────
class TestCompositeGates:
def test_security_scan_composite_findings(self):
statuses = [_gate(context="CI / security_scan", state="failure")]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: "fake security log\n",
)
failure = summary["gates"][0]["failure"]
# bandit is the head parser (first in multi:); semgrep + vulture
# nest as composite_findings.
assert failure["parser_used"] == "bandit"
assert len(failure["composite_findings"]) == 2
names = [c["parser_used"] for c in failure["composite_findings"]]
assert names == ["semgrep", "vulture"]
# ─── error handling ────────────────────────────────────────────────
class TestErrorHandling:
def test_log_fetch_returns_none(self):
statuses = [_gate(context="CI / lint", state="failure")]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: None,
)
failure = summary["gates"][0]["failure"]
assert failure["error_class"] == "log-fetch-failed"
assert failure["raw_log_excerpt"] == ""
def test_log_fetch_raises_handled(self):
statuses = [_gate(context="CI / lint", state="failure")]
def boom(_):
raise RuntimeError("network down")
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=boom,
)
# Failure recorded but loop didn't crash.
assert summary["gates"][0]["failure"]["error_class"] == "log-fetch-failed"
def test_unknown_gate_no_parser(self):
statuses = [_gate(context="CI / mystery_gate", state="failure")]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: "log content",
)
failure = summary["gates"][0]["failure"]
assert failure["error_class"] == "NoParserAvailable"
def test_forgejo_status_none(self):
summary = summarize_ci_status(
head_sha="abc", forgejo_status=None,
log_fetcher=lambda _: pytest.fail("should not be called"),
)
assert summary["overall_state"] == "unknown"
assert summary["gates"] == []
def test_parameterized_gate_name_normalized(self):
"""Forgejo can emit "unit_tests-3.13" for matrix jobs."""
statuses = [_gate(context="CI / unit_tests-3.13", state="failure")]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: "Failing scenarios:\n x.feature:5 Scenario: bar\n0 features passed, 1 failed, 0 skipped\n0 scenarios passed, 1 failed, 0 skipped\n",
)
failure = summary["gates"][0]["failure"]
assert failure["parser_used"] == "behave"
def test_skip_coverage_session_no_parser(self):
# benchmark is in NOX_SESSIONS_SKIP_COVERAGE; should not parse.
statuses = [_gate(context="CI / benchmark", state="failure")]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: pytest.fail("should not fetch log"),
)
# benchmark isn't in NOX_SESSION_TO_PARSER, so gate→session
# returns None → NoParserAvailable.
failure = summary["gates"][0]["failure"]
assert failure["error_class"] == "NoParserAvailable"
# ─── parser_versions aggregate ─────────────────────────────────────
class TestParserVersions:
def test_versions_collected_per_parser(self):
statuses = [
_gate(context="CI / lint", state="failure"),
_gate(context="CI / typecheck", state="failure"),
]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: "Found 0 errors.\n",
)
# Both ruff + pyright versions appear.
assert "ruff" in summary["parser_versions"]
assert "pyright" in summary["parser_versions"]
assert summary["parser_versions"]["ruff"] == "v1"
def test_stub_version_recorded(self):
statuses = [_gate(context="CI / complexity", state="failure")]
summary = summarize_ci_status(
head_sha="abc",
forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: "radon log",
)
# complexity → radon (currently a stub).
assert summary["parser_versions"].get("radon") == "v1-stub"
# ─── gate count aggregates ─────────────────────────────────────────
class TestGateCounts:
def test_mixed_statuses(self):
statuses = [
_gate(context="CI / lint", state="success"),
_gate(context="CI / typecheck", state="failure"),
_gate(context="CI / unit_tests", state="pending"),
_gate(context="CI / build", state="error"),
]
summary = summarize_ci_status(
head_sha="abc", forgejo_status=_forgejo_status(statuses),
log_fetcher=lambda _: "",
)
assert summary["gates_passed"] == 1
assert summary["gates_failed"] == 2 # failure + error
assert summary["gates_pending"] == 1
@@ -0,0 +1,76 @@
"""Shared types + helpers for per-CI-tool parsers.
Every parser module exposes:
- ``PARSER_VERSION: str`` — bumped when output semantics change.
- ``parse(log: str, gate_name: str) -> ParserResult``
Parsers are deterministic Python — no LLM. They take the gate's raw
log text + the gate name, and return structured findings that fit the
V1 ``CIFailure`` contract. Composite gates (e.g., nox security_scan =
bandit + semgrep + vulture) get parsed by an orchestrator that calls
each sub-parser and wraps the results in ``CIFailure.composite_findings``.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from ..contracts.v1 import CIFailureFinding, FailedAssertion, FileLocation
# Default log-excerpt budget (V1 CIFailure.raw_log_excerpt max_length).
DEFAULT_LOG_EXCERPT_CHARS = 16_384
@dataclass(frozen=True)
class ParserResult:
"""One parser's structured output for one gate's log.
``log_excerpt`` is the parser's selection of the most-relevant
log lines (smart-around-first-error, or trailing N lines). It MUST
fit within ``DEFAULT_LOG_EXCERPT_CHARS`` — the V1 contract enforces
this as max_length on the CIFailure field.
``log_excerpt_lines`` is the count of newline-separated lines in
the excerpt, used by the contract field of the same name.
"""
error_class: str
summary_line: str
findings: list[CIFailureFinding] = field(default_factory=list)
failing_locations: list[FileLocation] = field(default_factory=list)
failed_assertions: list[FailedAssertion] = field(default_factory=list)
log_excerpt: str = ""
log_excerpt_lines: int = 0
def select_log_excerpt(log: str, *, max_chars: int = DEFAULT_LOG_EXCERPT_CHARS,
tail_lines: int = 80) -> tuple[str, int]:
"""Pick the most-relevant excerpt from a log.
Strategy: prefer the LAST ``tail_lines`` (most test/build tools
emit failure context at the end). If the tail exceeds ``max_chars``,
truncate from the head of the tail (keep the bottom).
Returns ``(excerpt, line_count)``. An empty log produces ``("", 0)``.
"""
if not log:
return "", 0
lines = log.splitlines()
if len(lines) <= tail_lines:
excerpt = log
else:
excerpt = "\n".join(lines[-tail_lines:])
if len(excerpt) > max_chars:
excerpt = excerpt[-max_chars:]
# Trim a partial leading line to keep it clean.
nl = excerpt.find("\n")
if 0 < nl < 200:
excerpt = excerpt[nl + 1:]
return excerpt, excerpt.count("\n") + (1 if excerpt and not excerpt.endswith("\n") else 0)
__all__ = [
"DEFAULT_LOG_EXCERPT_CHARS",
"ParserResult",
"select_log_excerpt",
]
@@ -0,0 +1,88 @@
"""Parser registry — resolve a parser name to a (parser_version,
parse_function) pair.
Real parsers ship their own module under ``ci_summary_parsers/``. For
the not-yet-shipped tools, the registry returns a stub from
``_stub.make_stub(name)``. The startup coverage validator
``validate_parser_coverage()`` flags any name in EXPECTED_PARSERS
that still resolves to a stub so the operator can prioritize.
"""
from __future__ import annotations
import importlib
import logging
from collections.abc import Callable
from dataclasses import dataclass
from . import EXPECTED_PARSERS, NOX_SESSION_TO_PARSER
from . import _stub
from ._base import ParserResult
logger = logging.getLogger(__name__)
# Parsers that ship in Phase 1j (real implementations).
_REAL_PARSERS: frozenset[str] = frozenset({"ruff", "pyright", "behave"})
@dataclass(frozen=True)
class ResolvedParser:
name: str
version: str
parse: Callable[[str, str], ParserResult]
is_stub: bool
def resolve(parser_name: str) -> ResolvedParser:
"""Look up the parser for ``parser_name``. Returns a stub if the
real module isn't shipped yet (logs at WARNING level so operators
notice the parser gap)."""
if parser_name in _REAL_PARSERS:
try:
mod = importlib.import_module(
f"tools.controller.ci_summary_parsers.{parser_name}",
)
return ResolvedParser(
name=parser_name,
version=getattr(mod, "PARSER_VERSION", "v1"),
parse=mod.parse,
is_stub=False,
)
except ImportError:
logger.warning(
"real parser %s declared but import failed; using stub",
parser_name,
)
return ResolvedParser(
name=parser_name,
version=_stub.PARSER_VERSION,
parse=_stub.make_stub(parser_name),
is_stub=True,
)
def resolve_for_nox_session(nox_session: str) -> list[ResolvedParser]:
"""Map nox session → ordered list of parsers (most sessions have
one; composite security_scan has three).
Raises KeyError if the session has no mapping (forces operator to
update NOX_SESSION_TO_PARSER explicitly)."""
entry = NOX_SESSION_TO_PARSER[nox_session] # KeyError is intentional
if entry.startswith("multi:"):
names = entry.split(":", 1)[1].split(",")
return [resolve(n.strip()) for n in names if n.strip()]
return [resolve(entry)]
def validate_parser_coverage() -> dict[str, bool]:
"""Return a {parser_name: is_real} map for every parser in
EXPECTED_PARSERS. Operators can plot this to see parser gaps."""
return {p: not resolve(p).is_stub for p in sorted(EXPECTED_PARSERS)}
__all__ = [
"ResolvedParser",
"resolve",
"resolve_for_nox_session",
"validate_parser_coverage",
]
@@ -0,0 +1,55 @@
"""Stub parser for tools whose deterministic parser isn't shipped yet.
Returns a structured CIFailure-shape result that carries the raw log
excerpt + a ``parser-pending`` marker. The implementer's prompt can
still surface the failing log; the operator-side coverage dashboard
flags these as parser gaps to prioritize.
Phase 1j ships ruff/pyright/behave; Phase 1j+ ships the rest. Until
then, these tools use this stub:
- robot_framework (integration_tests, e2e_tests)
- slipcover (coverage_report)
- bandit, semgrep, vulture (security_scan composite)
- radon (complexity)
- build (build)
"""
from __future__ import annotations
from ..contracts.v1 import CIFailureFinding
from ._base import ParserResult, select_log_excerpt
PARSER_VERSION = "v1-stub"
def make_stub(parser_name: str):
"""Factory: returns a ``parse(log, gate_name)`` for the given
parser name. Each stub identifies itself in ``error_class``."""
def parse(log: str, gate_name: str) -> ParserResult:
excerpt, lines = select_log_excerpt(log)
finding = CIFailureFinding(
error_class=f"parser-pending-{parser_name}",
summary=(
f"Structured parser for '{parser_name}' is not yet "
f"implemented; falling back to raw log excerpt."
)[:512],
location=None,
severity="error",
)
return ParserResult(
error_class=f"parser-pending-{parser_name}",
summary_line=(
f"{parser_name}: parser pending; see raw_log_excerpt"
)[:200],
findings=[finding],
failing_locations=[],
failed_assertions=[],
log_excerpt=excerpt,
log_excerpt_lines=lines,
)
parse.__name__ = f"parse_stub_{parser_name}"
return parse
__all__ = ["PARSER_VERSION", "make_stub"]
@@ -0,0 +1,143 @@
"""Behave parser — handles ``nox -s unit_tests``.
Behave's CLI emits a tree-style trace per scenario:
Feature: My feature # tests/features/foo.feature:3
Scenario: My scenario # tests/features/foo.feature:10
Given foo # passed
When bar # passed
Then baz # failed: AssertionError: expected 1 == 2
File "tests/steps/baz_steps.py", line 10, in step_then
assert 1 == 2, "expected 1 == 2"
AssertionError: expected 1 == 2
Failing scenarios:
tests/features/foo.feature:10 Scenario: My scenario
0 features passed, 1 failed, 0 skipped
0 scenarios passed, 1 failed, 0 skipped
2 steps passed, 1 failed, 1 skipped
The parser extracts each failing scenario as a ``FailedAssertion``
+ a ``CIFailureFinding`` so the implementer can navigate to the
right feature file + step file.
"""
from __future__ import annotations
import re
from ..contracts.v1 import CIFailureFinding, FailedAssertion, FileLocation
from ._base import ParserResult, select_log_excerpt
PARSER_VERSION = "v1"
# " tests/features/foo.feature:10 Scenario: My scenario"
_FAILING_SCENARIO_RE = re.compile(
r"^\s+(?P<feature>\S+\.feature):(?P<line>\d+)\s+Scenario:\s+(?P<name>.+?)\s*$",
re.MULTILINE,
)
# "1 feature passed, 0 failed, 0 skipped"
_FEATURE_SUMMARY_RE = re.compile(
r"(\d+)\s+features?\s+passed,\s+(\d+)\s+failed,\s+(\d+)\s+skipped",
)
_SCENARIO_SUMMARY_RE = re.compile(
r"(\d+)\s+scenarios?\s+passed,\s+(\d+)\s+failed,\s+(\d+)\s+skipped",
)
# AssertionError line OR Exception line.
_ASSERT_RE = re.compile(
r"^\s*(?P<exc>AssertionError|Exception|.+Error):\s*(?P<msg>.+?)\s*$",
re.MULTILINE,
)
def parse(log: str, gate_name: str) -> ParserResult:
findings: list[CIFailureFinding] = []
locations: list[FileLocation] = []
assertions: list[FailedAssertion] = []
failing_scenarios = list(_FAILING_SCENARIO_RE.finditer(log))
# Map scenario → an excerpt of the surrounding log so we can lift
# the assertion line for the FailedAssertion.
for m in failing_scenarios:
loc = FileLocation(
file_path=m["feature"],
line_range=f"{m['line']}-{m['line']}",
function_or_test=m["name"][:120],
)
findings.append(CIFailureFinding(
error_class="ScenarioFailed",
summary=f"Scenario failed: {m['name'][:200]}"[:512],
location=loc,
severity="error",
))
locations.append(loc)
# Pull the first matching assertion globally — multi-assertion
# extraction would over-fit on rare logs. The raw excerpt carries
# the rest of the context.
for am in _ASSERT_RE.finditer(log):
exc = am["exc"].strip()
if exc.endswith("Error"):
msg = am["msg"].strip()[:512]
# Best-effort context: 10 lines around the match.
excerpt = _excerpt_around(log, am.start(), 10)
assertions.append(FailedAssertion(
test_name=(failing_scenarios[0]["name"][:120]
if failing_scenarios else "(unknown)"),
expected=None,
actual=None,
assertion_excerpt=(f"{exc}: {msg}\n\n{excerpt}")[:4096],
))
break
summary_line = _summary_line(log, n_failing=len(failing_scenarios))
excerpt, lines = select_log_excerpt(log)
error_class = "BehaveFailure" if findings else "BehaveClean"
return ParserResult(
error_class=error_class,
summary_line=summary_line,
findings=findings,
failing_locations=locations,
failed_assertions=assertions,
log_excerpt=excerpt,
log_excerpt_lines=lines,
)
def _summary_line(log: str, *, n_failing: int) -> str:
feat = _FEATURE_SUMMARY_RE.search(log)
sce = _SCENARIO_SUMMARY_RE.search(log)
if feat and sce:
return (
f"Behave: {feat.group(2)} feature(s) failed, "
f"{sce.group(2)} scenario(s) failed"
)[:200]
if n_failing:
return f"Behave: {n_failing} scenario(s) failed"[:200]
return "Behave: no failing scenarios parsed"
def _excerpt_around(log: str, offset: int, lines_each_side: int) -> str:
# Walk N newlines back, N newlines forward.
before = log.rfind("\n", 0, offset)
for _ in range(lines_each_side):
if before <= 0:
before = 0
break
before = log.rfind("\n", 0, before)
if before == -1:
before = 0
break
after = offset
for _ in range(lines_each_side):
nxt = log.find("\n", after + 1)
if nxt == -1:
after = len(log)
break
after = nxt
return log[before:after].strip("\n")[:2048]
__all__ = ["PARSER_VERSION", "parse"]
@@ -0,0 +1,117 @@
"""Pyright parser — handles ``nox -s typecheck``.
Pyright's CLI emits one diagnostic per line:
/abs/path/to/file.py:LINE:COL - error: human description (ruleName)
/abs/path/to/file.py:LINE:COL - warning: ...
1 error, 0 warnings, 0 informations
Some Pyright versions use absolute paths; we normalize to the
project-relative tail when a known segment (``tools/``, ``tests/``,
``src/``) is in the path. Otherwise the absolute path is kept.
"""
from __future__ import annotations
import re
from ..contracts.v1 import CIFailureFinding, FileLocation
from ._base import ParserResult, select_log_excerpt
PARSER_VERSION = "v1"
_DIAG_RE = re.compile(
r"^(?P<file>\S+?):(?P<line>\d+):(?P<col>\d+)\s+-\s+"
r"(?P<severity>error|warning|information)\s*:\s+"
r"(?P<message>.+?)\s*$",
re.MULTILINE,
)
# Some Pyright versions emit findings on TWO lines (path/loc then
# indented message); we don't try to parse those (rare). The summary
# line is canonical:
_SUMMARY_RE = re.compile(
r"(\d+)\s+errors?,\s+(\d+)\s+warnings?,\s+(\d+)\s+informations?",
)
_PATH_NORMALIZE_SEGMENTS = ("/tools/", "/tests/", "/src/", "/.opencode/")
def parse(log: str, gate_name: str) -> ParserResult:
findings: list[CIFailureFinding] = []
locations: list[FileLocation] = []
for m in _DIAG_RE.finditer(log):
sev_raw = m["severity"]
if sev_raw == "error":
severity = "error"
error_class = _extract_rule_name(m["message"]) or "PyrightError"
elif sev_raw == "warning":
severity = "warning"
error_class = _extract_rule_name(m["message"]) or "PyrightWarning"
else:
severity = "info"
error_class = "PyrightInfo"
loc = FileLocation(
file_path=_normalize_path(m["file"]),
line_range=f"{m['line']}-{m['line']}",
)
findings.append(CIFailureFinding(
error_class=error_class,
summary=m["message"].strip()[:512],
location=loc,
severity=severity,
))
if severity == "error":
locations.append(loc)
summary_line = _summary_line(log, n_findings=len(findings))
excerpt, lines = select_log_excerpt(log)
error_class = _aggregate_error_class(findings)
return ParserResult(
error_class=error_class,
summary_line=summary_line,
findings=findings,
failing_locations=locations,
failed_assertions=[],
log_excerpt=excerpt,
log_excerpt_lines=lines,
)
def _extract_rule_name(message: str) -> str | None:
"""Pyright wraps the rule name in trailing parens. Pull it out."""
m = re.search(r"\((report[A-Za-z0-9]+)\)\s*$", message)
return m.group(1) if m else None
def _normalize_path(path: str) -> str:
"""Strip leading container path so the implementer can match the
workspace tree."""
for seg in _PATH_NORMALIZE_SEGMENTS:
idx = path.find(seg)
if idx >= 0:
return path[idx + 1:]
return path
def _summary_line(log: str, *, n_findings: int) -> str:
m = _SUMMARY_RE.search(log)
if m:
return (
f"Pyright: {m.group(1)} errors, {m.group(2)} warnings, "
f"{m.group(3)} informations"
)[:200]
return f"Pyright: {n_findings} diagnostic(s) parsed"[:200]
def _aggregate_error_class(findings: list[CIFailureFinding]) -> str:
errors = [f for f in findings if f.severity == "error"]
if not errors:
return "PyrightClean"
classes = {f.error_class for f in errors}
if len(classes) == 1:
return next(iter(classes))
return "PyrightMixed"
__all__ = ["PARSER_VERSION", "parse"]
+109
View File
@@ -0,0 +1,109 @@
"""Ruff parser — handles ``nox -s lint`` and ``nox -s format``.
Ruff's CLI prints one finding per line in the format:
path/to/file.py:LINE:COL: RULECODE human description
Followed by a trailing ``Found N errors.`` summary line. Format mode
(`ruff format --check`) emits a different shape:
Would reformat: path/to/file.py
1 file would be reformatted
Both shapes are absorbed below. Findings are emitted as
``CIFailureFinding`` rows so the implementer's prompt can render
"rule + file:line" with one entry per failure.
"""
from __future__ import annotations
import re
from ..contracts.v1 import CIFailureFinding, FileLocation
from ._base import ParserResult, select_log_excerpt
PARSER_VERSION = "v1"
# Examples:
# tools/controller/master/loop.py:42:5: F401 `os` imported but unused
# tests/x.py:1:1: E501 line too long (90 > 88 characters)
_LINT_RE = re.compile(
r"^(?P<file>[^\s:][^:]*):(?P<line>\d+):(?P<col>\d+):\s+"
r"(?P<code>[A-Z]+\d+)\s+(?P<message>.+)$",
re.MULTILINE,
)
# Examples: ``Would reformat: tools/foo.py``
_FORMAT_RE = re.compile(r"^Would reformat:\s+(?P<file>\S+)\s*$", re.MULTILINE)
# Summary lines we extract for summary_line in priority order.
_SUMMARY_PATTERNS = [
re.compile(r"^Found (\d+) errors?\.?$", re.MULTILINE),
re.compile(r"^(\d+) files? would be reformatted$", re.MULTILINE),
]
def parse(log: str, gate_name: str) -> ParserResult:
findings: list[CIFailureFinding] = []
locations: list[FileLocation] = []
for m in _LINT_RE.finditer(log):
loc = FileLocation(
file_path=m["file"],
line_range=f"{m['line']}-{m['line']}",
)
findings.append(CIFailureFinding(
error_class=m["code"],
summary=m["message"].strip()[:512],
location=loc,
severity="error",
))
locations.append(loc)
for m in _FORMAT_RE.finditer(log):
loc = FileLocation(file_path=m["file"])
findings.append(CIFailureFinding(
error_class="format",
summary="file would be reformatted",
location=loc,
severity="warning",
))
locations.append(loc)
summary_line = _pick_summary(log, fallback=f"{len(findings)} ruff findings")
excerpt, lines = select_log_excerpt(log)
error_class = _aggregate_error_class(findings)
return ParserResult(
error_class=error_class,
summary_line=summary_line[:200],
findings=findings,
failing_locations=locations,
failed_assertions=[],
log_excerpt=excerpt,
log_excerpt_lines=lines,
)
def _pick_summary(log: str, *, fallback: str) -> str:
for pat in _SUMMARY_PATTERNS:
matches = pat.findall(log)
if matches:
# Re-render: the regex captured the count.
n = matches[-1]
if pat is _SUMMARY_PATTERNS[0]:
return f"Ruff found {n} error(s)"
return f"Ruff: {n} file(s) need reformat"
return fallback
def _aggregate_error_class(findings: list[CIFailureFinding]) -> str:
"""If all findings share one error_class, surface it. Else use a
composite marker so the implementer's prompt shows a sensible
top-level error_class on the CIFailure."""
if not findings:
return "RuffEmpty"
classes = {f.error_class for f in findings}
if len(classes) == 1:
return next(iter(classes))
return "RuffMixed"
__all__ = ["PARSER_VERSION", "parse"]
+7
View File
@@ -71,6 +71,10 @@ from .backfill import (
has_backfill_run,
run_startup_backfill,
)
from .ci_summarize import (
LogFetcher,
summarize_ci_status,
)
from .prefetch import (
GetPRDetailsCallback,
GetPRDiffCallback,
@@ -166,4 +170,7 @@ __all__ = [
"build_implementer_input",
"build_reviewer_input",
"make_prefetch_callback",
# CI summarizer (Phase 1j)
"LogFetcher",
"summarize_ci_status",
]
+314
View File
@@ -0,0 +1,314 @@
"""Deterministic CI summarizer — builds CISummary from Forgejo status.
The implementer/reviewer prompt prefetch needs a CISummary V1 dict for
the head_sha. This module takes the Forgejo combined-status response
(``{"state": "...", "statuses": [...]}``) plus a log fetcher, and
returns a fully-populated CISummary covering every gate.
Per-gate flow:
1. Map the Forgejo gate context (e.g., "CI / lint") → nox session
name. The job names in ``.forgejo/workflows/*.yml`` ARE the nox
session names by convention, so a simple suffix-after-/ extraction
works. Unknown contexts pass through as ``UnknownGate`` and skip
the parser.
2. Look up the nox session in NOX_SESSION_TO_PARSER → one or more
parser names.
3. For each parser, fetch the log (via the injected ``log_fetcher``),
run ``parse(log, gate_name)``, and wrap the result in a
GateResult.failure CIFailure dict.
4. For composite gates (security_scan = bandit + semgrep + vulture),
the outer CIFailure carries the FIRST parser's structure + each
sub-parser's result goes into ``composite_findings``.
The summarizer never raises — log fetch failures degrade to a
``CIFailure`` with ``error_class='log-fetch-failed'`` so the
implementer at least sees the failure existed.
"""
from __future__ import annotations
import logging
from collections.abc import Callable
from datetime import datetime, timezone
from typing import Any
from ..ci_summary_parsers import NOX_SESSION_TO_PARSER, NOX_SESSIONS_SKIP_COVERAGE
from ..ci_summary_parsers._registry import (
ResolvedParser,
resolve_for_nox_session,
)
logger = logging.getLogger(__name__)
# log_fetcher(gate_name) -> str | None; production wires this to a
# Forgejo job-log fetcher. None means the log was unreachable.
LogFetcher = Callable[[str], str | None]
# Forgejo status state → CISummary GateResult.status mapping.
_FORGEJO_STATE_TO_GATE_STATUS = {
"success": "passed",
"failure": "failed",
"error": "error",
"pending": "pending",
"warning": "passed", # advisory: treat as passed
None: "pending",
}
def summarize_ci_status(
*, head_sha: str, forgejo_status: dict | None, log_fetcher: LogFetcher,
observed_at: datetime | None = None,
) -> dict:
"""Build a CISummary-shape dict from the Forgejo combined-status.
Args:
head_sha: the commit SHA the status covers.
forgejo_status: response body from
``GET /repos/{owner}/{repo}/commits/{sha}/status``.
Shape: ``{"state": str, "statuses": [{"context": str,
"state": str, "target_url": str}]}``. None for
"couldn't fetch" — produces a unknown-state summary.
log_fetcher: callable that takes the gate context and returns
the raw log text (or None if unreachable). Production
wires this via Forgejo's job-log endpoint.
observed_at: timestamp the status was fetched. Defaults to
now(UTC).
Returns the CISummary V1-shape dict (a CISummary contract parses
it after the caller validates).
"""
when = observed_at or datetime.now(timezone.utc)
if forgejo_status is None:
return _unknown_summary(head_sha=head_sha, observed_at=when)
overall = _FORGEJO_OVERALL_STATE_MAPPING.get(
forgejo_status.get("state"), "unknown",
)
raw_statuses = forgejo_status.get("statuses") or []
statuses = [s for s in raw_statuses if isinstance(s, dict)]
gates: list[dict] = []
parser_versions: dict[str, str] = {}
for s in statuses:
gate, versions_seen = _build_gate(s, log_fetcher)
gates.append(gate)
parser_versions.update(versions_seen)
counts = _count_gates(gates)
return {
"summary_version": "V1",
"head_sha": head_sha,
"observed_at": when,
"overall_state": overall,
"gates": gates,
"gates_total": counts["total"],
"gates_passed": counts["passed"],
"gates_failed": counts["failed"],
"gates_skipped": counts["skipped"],
"gates_pending": counts["pending"],
"parser_versions": parser_versions,
}
# Forgejo combined-status states → CISummary.overall_state.
_FORGEJO_OVERALL_STATE_MAPPING: dict[str | None, str] = {
"success": "success",
"failure": "failure",
"error": "error",
"pending": "pending",
None: "unknown",
}
def _unknown_summary(*, head_sha: str, observed_at: datetime) -> dict:
return {
"summary_version": "V1",
"head_sha": head_sha,
"observed_at": observed_at,
"overall_state": "unknown",
"gates": [],
"gates_total": 0,
"gates_passed": 0,
"gates_failed": 0,
"gates_skipped": 0,
"gates_pending": 0,
"parser_versions": {},
}
def _build_gate(
status: dict, log_fetcher: LogFetcher,
) -> tuple[dict, dict[str, str]]:
"""Build one GateResult dict + return parser versions used."""
context = status.get("context") or "(unknown)"
state_raw = status.get("state")
gate_status = _FORGEJO_STATE_TO_GATE_STATUS.get(state_raw, "pending")
target_url = status.get("target_url")
severity = "info" if gate_status in {"passed", "skipped"} else "error"
gate: dict[str, Any] = {
"name": context,
"status": gate_status,
"severity": severity,
"target_url": target_url,
"duration_seconds": None,
"failure": None,
}
parser_versions: dict[str, str] = {}
# Only parse failures/errors. Passed/pending/skipped don't need a
# log fetch — they have nothing to report.
if gate_status not in {"failed", "error"}:
return gate, parser_versions
nox_session = _gate_to_nox_session(context)
if nox_session is None or nox_session in NOX_SESSIONS_SKIP_COVERAGE:
gate["failure"] = _no_parser_failure(context)
return gate, parser_versions
try:
parsers = resolve_for_nox_session(nox_session)
except KeyError:
logger.warning(
"no parser map for nox session %r (gate=%r)",
nox_session, context,
)
gate["failure"] = _no_parser_failure(context)
return gate, parser_versions
log = _safe_fetch(log_fetcher, context)
if log is None:
gate["failure"] = _log_fetch_failed_failure(parsers, context)
for p in parsers:
parser_versions[p.name] = p.version
return gate, parser_versions
gate["failure"] = _build_failure(parsers, log, context, parser_versions)
return gate, parser_versions
def _gate_to_nox_session(context: str) -> str | None:
"""Forgejo gate context → nox session name.
Convention: ``<workflow-name> / <job-name>`` (e.g., "CI / lint").
The job name IS the nox session name by repo convention. If the
name doesn't match any known session, return None and let the
summarizer fall through to NoParserAvailable."""
if not context:
return None
tail = context.rsplit("/", 1)[-1].strip()
if tail in NOX_SESSION_TO_PARSER:
return tail
# Also support the parameterized form e.g. "unit_tests-3.13".
base = tail.split("-", 1)[0]
if base in NOX_SESSION_TO_PARSER:
return base
return None
def _safe_fetch(log_fetcher: LogFetcher, context: str) -> str | None:
try:
return log_fetcher(context)
except Exception as exc: # noqa: BLE001
logger.warning("log fetch raised for %s: %s", context, exc)
return None
def _no_parser_failure(context: str) -> dict:
return {
"parser_used": "(none)",
"parser_version": "n/a",
"error_class": "NoParserAvailable",
"summary_line": f"No parser available for {context!r}"[:200],
"findings": [],
"failing_locations": [],
"failed_assertions": [],
"raw_log_excerpt": "",
"log_excerpt_lines": 0,
"composite_findings": [],
}
def _log_fetch_failed_failure(parsers, context: str) -> dict:
parser_names = ", ".join(p.name for p in parsers)
return {
"parser_used": parser_names or "(unknown)",
"parser_version": "n/a",
"error_class": "log-fetch-failed",
"summary_line": f"Log fetch failed for {context!r}"[:200],
"findings": [],
"failing_locations": [],
"failed_assertions": [],
"raw_log_excerpt": "",
"log_excerpt_lines": 0,
"composite_findings": [],
}
def _build_failure(
parsers: list[ResolvedParser], log: str, context: str,
parser_versions: dict[str, str],
) -> dict:
"""Build the CIFailure dict from one or more parsers' results.
For a single-parser gate the result becomes the outer CIFailure.
For a composite gate (multi: prefix), the FIRST parser becomes
the outer + the rest go into composite_findings.
"""
results = []
for p in parsers:
try:
res = p.parse(log, context)
parser_versions[p.name] = p.version
results.append((p, res))
except Exception as exc: # noqa: BLE001
logger.warning(
"parser %s raised on gate %r: %s", p.name, context, exc,
)
# Fall through with no result for this parser.
if not results:
return _log_fetch_failed_failure(parsers, context)
head_parser, head_result = results[0]
failure = _result_to_failure(head_parser, head_result)
if len(results) > 1:
failure["composite_findings"] = [
_result_to_failure(p, r) for p, r in results[1:]
]
return failure
def _result_to_failure(parser: ResolvedParser, result) -> dict:
return {
"parser_used": parser.name,
"parser_version": parser.version,
"error_class": result.error_class,
"summary_line": result.summary_line,
"findings": [f.model_dump() for f in result.findings],
"failing_locations": [loc.model_dump() for loc in result.failing_locations],
"failed_assertions": [a.model_dump() for a in result.failed_assertions],
"raw_log_excerpt": result.log_excerpt,
"log_excerpt_lines": result.log_excerpt_lines,
"composite_findings": [],
}
def _count_gates(gates: list[dict]) -> dict[str, int]:
counts = {"total": len(gates), "passed": 0, "failed": 0,
"skipped": 0, "pending": 0}
for g in gates:
status = g.get("status")
if status == "passed":
counts["passed"] += 1
elif status in {"failed", "error"}:
counts["failed"] += 1
elif status == "skipped":
counts["skipped"] += 1
elif status == "pending":
counts["pending"] += 1
return counts
__all__ = ["LogFetcher", "summarize_ci_status"]