feat(controller): Phase 1j — deterministic CI summarizer + priority parsers
Replaces "ci_summary=None / failing_gates=[]" placeholders from Phase
1h with a real summarizer that maps Forgejo combined-status →
CISummary V1 dict by running per-tool deterministic parsers on each
failing gate's log.
Priority parsers shipped (cover lint/format/typecheck/unit_tests, the
4 most-failed gates):
- ruff — F+E codes from `nox -s lint`; Would-reformat lines from
`nox -s format`. Aggregates to single error_class when
all findings share one code, else RuffMixed.
- pyright — error/warning/information diagnostics; rule name pulled
from trailing `(reportName)` parens. Abs-path
normalization strips container prefixes.
- behave — failing scenarios (file:line + name), AssertionError
extraction. Feature/scenario summary line aggregation.
Stub parsers for not-yet-shipped tools (robot_framework, slipcover,
bandit, semgrep, vulture, radon, build): return a structured
CIFailure with error_class="parser-pending-{name}" + the raw log
excerpt. Operators see the failure; implementer still has log
context. Phase 1j+ replaces stubs with real parsers without changing
the gate-→-session map.
Components:
- _base.py — ParserResult dataclass + select_log_excerpt()
(tail-N-lines smart selection within 16KB cap)
- _stub.py — make_stub(name) factory for pending tools
- _registry.py — resolve(parser_name) + resolve_for_nox_session()
+ validate_parser_coverage()
- master/ci_summarize.py — summarize_ci_status(head_sha, status,
log_fetcher) orchestrator. Handles:
- composite multi: gates → CIFailure.composite_findings
- log_fetcher returning None → log-fetch-failed
- log_fetcher raising → caught + log-fetch-failed
- Unknown gate context → NoParserAvailable
- Forgejo state=None → unknown summary
- Parameterized matrix gates ("unit_tests-3.13")
→ base session name resolution
Tests (+46 across 2 new files, 0 regressions):
- Per-parser canonical + empty + garbage input
- Registry resolution (real vs stub), coverage validator
- Summarizer V1 contract round-trip
- Composite security_scan composite_findings shape
- Error paths (None status, raising fetcher, unknown gate)
- Parser version aggregation across mixed gates
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,299 @@
|
||||
"""Tests for the per-tool CI summary parsers (Phase 1j).
|
||||
|
||||
Validates each parser:
|
||||
- Extracts findings from canonical CLI output
|
||||
- Populates summary_line, error_class, log_excerpt correctly
|
||||
- Handles empty/garbage input without raising
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.controller.ci_summary_parsers import (
|
||||
EXPECTED_PARSERS,
|
||||
NOX_SESSION_TO_PARSER,
|
||||
)
|
||||
from tools.controller.ci_summary_parsers import _stub, behave, pyright, ruff
|
||||
from tools.controller.ci_summary_parsers._base import select_log_excerpt
|
||||
from tools.controller.ci_summary_parsers._registry import (
|
||||
resolve,
|
||||
resolve_for_nox_session,
|
||||
validate_parser_coverage,
|
||||
)
|
||||
|
||||
|
||||
# ─── shared base helpers ────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSelectLogExcerpt:
|
||||
def test_empty_log(self):
|
||||
excerpt, lines = select_log_excerpt("")
|
||||
assert excerpt == ""
|
||||
assert lines == 0
|
||||
|
||||
def test_short_log_returned_whole(self):
|
||||
log = "line 1\nline 2\nline 3\n"
|
||||
excerpt, lines = select_log_excerpt(log, tail_lines=80)
|
||||
assert "line 1" in excerpt
|
||||
assert lines == 3
|
||||
|
||||
def test_long_log_tail_kept(self):
|
||||
log = "\n".join(f"line {i}" for i in range(200))
|
||||
excerpt, lines = select_log_excerpt(log, tail_lines=10)
|
||||
assert "line 199" in excerpt
|
||||
assert "line 100" not in excerpt
|
||||
assert lines == 10
|
||||
|
||||
def test_hard_char_cap_respected(self):
|
||||
log = "x" * 50_000
|
||||
excerpt, lines = select_log_excerpt(log, max_chars=1000)
|
||||
assert len(excerpt) <= 1000
|
||||
|
||||
|
||||
# ─── ruff ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRuffParser:
|
||||
def test_lint_findings_parsed(self):
|
||||
log = """\
|
||||
tools/x.py:10:5: F401 `os` imported but unused
|
||||
tools/y.py:20:1: E501 line too long (90 > 88 characters)
|
||||
Found 2 errors.
|
||||
"""
|
||||
result = ruff.parse(log, "CI / lint")
|
||||
assert len(result.findings) == 2
|
||||
codes = [f.error_class for f in result.findings]
|
||||
assert codes == ["F401", "E501"]
|
||||
assert "Ruff found 2 error(s)" in result.summary_line
|
||||
assert result.error_class == "RuffMixed"
|
||||
|
||||
def test_all_findings_same_class(self):
|
||||
log = """\
|
||||
a.py:1:1: F401 unused
|
||||
b.py:2:1: F401 unused
|
||||
Found 2 errors.
|
||||
"""
|
||||
result = ruff.parse(log, "lint")
|
||||
assert result.error_class == "F401"
|
||||
|
||||
def test_format_findings_parsed(self):
|
||||
log = """\
|
||||
Would reformat: tools/a.py
|
||||
Would reformat: tools/b.py
|
||||
2 files would be reformatted
|
||||
"""
|
||||
result = ruff.parse(log, "format")
|
||||
assert len(result.findings) == 2
|
||||
assert all(f.error_class == "format" for f in result.findings)
|
||||
assert "would be reformatted" in result.findings[0].summary
|
||||
|
||||
def test_empty_log(self):
|
||||
result = ruff.parse("", "lint")
|
||||
assert result.findings == []
|
||||
assert result.error_class == "RuffEmpty"
|
||||
|
||||
def test_garbage_log_no_crash(self):
|
||||
result = ruff.parse("totally not ruff output\nnope nope\n", "lint")
|
||||
assert result.findings == []
|
||||
# Falls back to "0 ruff findings" summary
|
||||
assert "0" in result.summary_line or "ruff" in result.summary_line.lower()
|
||||
|
||||
def test_locations_populated(self):
|
||||
log = "tools/x.py:42:10: F401 foo\n"
|
||||
result = ruff.parse(log, "lint")
|
||||
assert len(result.failing_locations) == 1
|
||||
assert result.failing_locations[0].file_path == "tools/x.py"
|
||||
assert result.failing_locations[0].line_range == "42-42"
|
||||
|
||||
|
||||
# ─── pyright ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestPyrightParser:
|
||||
def test_basic_diagnostic(self):
|
||||
log = """\
|
||||
/home/runner/work/repo/tools/foo.py:10:5 - error: Cannot find name 'x' (reportUnknownVariableType)
|
||||
/home/runner/work/repo/tools/bar.py:1:1 - warning: trivial (reportUnusedImport)
|
||||
1 errors, 1 warnings, 0 informations
|
||||
"""
|
||||
result = pyright.parse(log, "typecheck")
|
||||
assert len(result.findings) == 2
|
||||
errs = [f for f in result.findings if f.severity == "error"]
|
||||
assert len(errs) == 1
|
||||
assert errs[0].error_class == "reportUnknownVariableType"
|
||||
assert "Pyright: 1 errors" in result.summary_line
|
||||
|
||||
def test_path_normalization(self):
|
||||
log = "/abs/path/work/tools/foo.py:10:5 - error: msg (reportFoo)\n"
|
||||
result = pyright.parse(log, "typecheck")
|
||||
assert result.findings[0].location.file_path == "tools/foo.py"
|
||||
|
||||
def test_no_errors_only_warnings(self):
|
||||
log = """\
|
||||
/x/tools/foo.py:1:1 - warning: trivial (reportUnusedImport)
|
||||
0 errors, 1 warnings, 0 informations
|
||||
"""
|
||||
result = pyright.parse(log, "typecheck")
|
||||
# Errors-only locations → empty
|
||||
assert result.failing_locations == []
|
||||
assert result.error_class == "PyrightClean"
|
||||
|
||||
def test_empty_log(self):
|
||||
result = pyright.parse("", "typecheck")
|
||||
assert result.findings == []
|
||||
assert result.error_class == "PyrightClean"
|
||||
|
||||
def test_two_errors_diff_classes(self):
|
||||
log = """\
|
||||
/x/tools/a.py:1:1 - error: a (reportFoo)
|
||||
/x/tools/b.py:2:2 - error: b (reportBar)
|
||||
2 errors, 0 warnings, 0 informations
|
||||
"""
|
||||
result = pyright.parse(log, "typecheck")
|
||||
assert result.error_class == "PyrightMixed"
|
||||
|
||||
def test_two_errors_same_class(self):
|
||||
log = """\
|
||||
/x/tools/a.py:1:1 - error: a (reportFoo)
|
||||
/x/tools/b.py:2:2 - error: b (reportFoo)
|
||||
2 errors, 0 warnings, 0 informations
|
||||
"""
|
||||
result = pyright.parse(log, "typecheck")
|
||||
assert result.error_class == "reportFoo"
|
||||
|
||||
|
||||
# ─── behave ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestBehaveParser:
|
||||
def test_failing_scenario_extracted(self):
|
||||
log = """\
|
||||
Feature: My feature
|
||||
Scenario: My scenario
|
||||
Then baz failed
|
||||
|
||||
Failing scenarios:
|
||||
tests/features/foo.feature:10 Scenario: My scenario
|
||||
|
||||
0 features passed, 1 failed, 0 skipped
|
||||
0 scenarios passed, 1 failed, 0 skipped
|
||||
2 steps passed, 1 failed, 1 skipped
|
||||
AssertionError: expected 1 == 2
|
||||
"""
|
||||
result = behave.parse(log, "unit_tests")
|
||||
assert len(result.findings) == 1
|
||||
assert result.findings[0].error_class == "ScenarioFailed"
|
||||
assert "My scenario" in result.findings[0].summary
|
||||
loc = result.findings[0].location
|
||||
assert loc.file_path == "tests/features/foo.feature"
|
||||
assert loc.line_range == "10-10"
|
||||
assert "Behave: 1 feature(s) failed" in result.summary_line
|
||||
|
||||
def test_no_failing_scenarios(self):
|
||||
log = """\
|
||||
1 features passed, 0 failed, 0 skipped
|
||||
1 scenarios passed, 0 failed, 0 skipped
|
||||
"""
|
||||
result = behave.parse(log, "unit_tests")
|
||||
assert result.findings == []
|
||||
assert result.error_class == "BehaveClean"
|
||||
|
||||
def test_empty_log(self):
|
||||
result = behave.parse("", "unit_tests")
|
||||
assert result.findings == []
|
||||
|
||||
def test_assertion_extracted(self):
|
||||
log = """\
|
||||
Failing scenarios:
|
||||
tests/x.feature:5 Scenario: foo
|
||||
|
||||
AssertionError: my-message-here
|
||||
0 features passed, 1 failed, 0 skipped
|
||||
0 scenarios passed, 1 failed, 0 skipped
|
||||
"""
|
||||
result = behave.parse(log, "unit_tests")
|
||||
assert len(result.failed_assertions) == 1
|
||||
assert "my-message-here" in result.failed_assertions[0].assertion_excerpt
|
||||
|
||||
|
||||
# ─── stub ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStubParser:
|
||||
def test_stub_returns_pending_marker(self):
|
||||
log = "any log content\nmore\n"
|
||||
parse = _stub.make_stub("robot_framework")
|
||||
result = parse(log, "integration_tests")
|
||||
assert result.error_class == "parser-pending-robot_framework"
|
||||
assert "parser pending" in result.summary_line
|
||||
assert "any log content" in result.log_excerpt
|
||||
assert len(result.findings) == 1
|
||||
assert result.findings[0].severity == "error"
|
||||
|
||||
def test_each_stub_independently_named(self):
|
||||
a = _stub.make_stub("bandit")
|
||||
b = _stub.make_stub("vulture")
|
||||
ra = a("log", "g")
|
||||
rb = b("log", "g")
|
||||
assert ra.error_class != rb.error_class
|
||||
|
||||
|
||||
# ─── registry ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRegistry:
|
||||
def test_resolve_real_ruff(self):
|
||||
r = resolve("ruff")
|
||||
assert r.is_stub is False
|
||||
assert r.name == "ruff"
|
||||
assert r.version == "v1"
|
||||
|
||||
def test_resolve_real_pyright(self):
|
||||
r = resolve("pyright")
|
||||
assert r.is_stub is False
|
||||
assert r.parse is pyright.parse
|
||||
|
||||
def test_resolve_real_behave(self):
|
||||
r = resolve("behave")
|
||||
assert r.is_stub is False
|
||||
|
||||
def test_resolve_pending_stub(self):
|
||||
r = resolve("robot_framework")
|
||||
assert r.is_stub is True
|
||||
assert r.version == "v1-stub"
|
||||
|
||||
def test_resolve_for_nox_session_single(self):
|
||||
parsers = resolve_for_nox_session("lint")
|
||||
assert [p.name for p in parsers] == ["ruff"]
|
||||
|
||||
def test_resolve_for_nox_session_composite(self):
|
||||
parsers = resolve_for_nox_session("security_scan")
|
||||
names = [p.name for p in parsers]
|
||||
assert names == ["bandit", "semgrep", "vulture"]
|
||||
|
||||
def test_resolve_for_unknown_session_raises(self):
|
||||
with pytest.raises(KeyError):
|
||||
resolve_for_nox_session("nope_not_a_session")
|
||||
|
||||
def test_validate_parser_coverage(self):
|
||||
cov = validate_parser_coverage()
|
||||
# Every name in EXPECTED_PARSERS shows up.
|
||||
assert set(cov.keys()) == EXPECTED_PARSERS
|
||||
# ruff/pyright/behave are real; rest are stubs.
|
||||
assert cov["ruff"] is True
|
||||
assert cov["pyright"] is True
|
||||
assert cov["behave"] is True
|
||||
assert cov["robot_framework"] is False
|
||||
assert cov["bandit"] is False
|
||||
|
||||
|
||||
# ─── expected coverage shape ───────────────────────────────────────
|
||||
|
||||
|
||||
class TestExpectedCoverageShape:
|
||||
def test_nox_session_map_only_references_known_parsers(self):
|
||||
for parser in EXPECTED_PARSERS:
|
||||
assert resolve(parser).name == parser
|
||||
|
||||
def test_security_scan_is_composite(self):
|
||||
assert NOX_SESSION_TO_PARSER["security_scan"].startswith("multi:")
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Tests for the deterministic CI summarizer (Phase 1j).
|
||||
|
||||
Validates summarize_ci_status orchestration:
|
||||
- Forgejo combined-status → CISummary dict shape (parses as
|
||||
CISummary V1 contract).
|
||||
- Gate→nox-session resolution honors workflow naming convention.
|
||||
- Composite gates (security_scan) get composite_findings rows.
|
||||
- Failed log fetches degrade to log-fetch-failed CIFailure rather than
|
||||
raising.
|
||||
- Unknown gate contexts fall through to NoParserAvailable cleanly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.controller.contracts.v1 import CISummary
|
||||
from tools.controller.master import summarize_ci_status
|
||||
|
||||
|
||||
# ─── fixtures ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _forgejo_status(statuses, *, overall="failure") -> dict:
|
||||
return {"state": overall, "statuses": statuses}
|
||||
|
||||
|
||||
def _gate(*, context: str, state: str, target_url: str | None = None) -> dict:
|
||||
return {"context": context, "state": state, "target_url": target_url}
|
||||
|
||||
|
||||
# ─── happy path ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestHappyPath:
|
||||
def test_basic_summary(self):
|
||||
statuses = [
|
||||
_gate(context="CI / lint", state="success"),
|
||||
_gate(context="CI / typecheck", state="success"),
|
||||
]
|
||||
log_fetcher = lambda _: ""
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc", forgejo_status=_forgejo_status(statuses, overall="success"),
|
||||
log_fetcher=log_fetcher,
|
||||
)
|
||||
assert summary["overall_state"] == "success"
|
||||
assert summary["gates_total"] == 2
|
||||
assert summary["gates_passed"] == 2
|
||||
assert summary["gates_failed"] == 0
|
||||
# No failures → no parser invocations.
|
||||
for g in summary["gates"]:
|
||||
assert g["failure"] is None
|
||||
|
||||
def test_failing_lint_invokes_ruff(self):
|
||||
statuses = [_gate(context="CI / lint", state="failure")]
|
||||
log = "tools/x.py:1:1: F401 unused\nFound 1 errors.\n"
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: log,
|
||||
)
|
||||
assert summary["gates_failed"] == 1
|
||||
failure = summary["gates"][0]["failure"]
|
||||
assert failure is not None
|
||||
assert failure["parser_used"] == "ruff"
|
||||
assert failure["error_class"] == "F401"
|
||||
assert len(failure["findings"]) == 1
|
||||
|
||||
def test_parses_as_v1_contract(self):
|
||||
statuses = [_gate(context="CI / lint", state="failure")]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: "tools/x.py:1:1: F401 unused\nFound 1 errors.\n",
|
||||
)
|
||||
# The CISummary contract must accept the assembled dict.
|
||||
parsed = CISummary.model_validate(summary)
|
||||
assert parsed.gates_failed == 1
|
||||
assert parsed.gates[0].failure.parser_used == "ruff"
|
||||
|
||||
def test_observed_at_default_now(self):
|
||||
before = datetime.now(timezone.utc)
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc", forgejo_status=_forgejo_status([], overall="success"),
|
||||
log_fetcher=lambda _: "",
|
||||
)
|
||||
after = datetime.now(timezone.utc)
|
||||
assert before <= summary["observed_at"] <= after
|
||||
|
||||
|
||||
# ─── composite gates ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCompositeGates:
|
||||
def test_security_scan_composite_findings(self):
|
||||
statuses = [_gate(context="CI / security_scan", state="failure")]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: "fake security log\n",
|
||||
)
|
||||
failure = summary["gates"][0]["failure"]
|
||||
# bandit is the head parser (first in multi:); semgrep + vulture
|
||||
# nest as composite_findings.
|
||||
assert failure["parser_used"] == "bandit"
|
||||
assert len(failure["composite_findings"]) == 2
|
||||
names = [c["parser_used"] for c in failure["composite_findings"]]
|
||||
assert names == ["semgrep", "vulture"]
|
||||
|
||||
|
||||
# ─── error handling ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestErrorHandling:
|
||||
def test_log_fetch_returns_none(self):
|
||||
statuses = [_gate(context="CI / lint", state="failure")]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: None,
|
||||
)
|
||||
failure = summary["gates"][0]["failure"]
|
||||
assert failure["error_class"] == "log-fetch-failed"
|
||||
assert failure["raw_log_excerpt"] == ""
|
||||
|
||||
def test_log_fetch_raises_handled(self):
|
||||
statuses = [_gate(context="CI / lint", state="failure")]
|
||||
|
||||
def boom(_):
|
||||
raise RuntimeError("network down")
|
||||
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=boom,
|
||||
)
|
||||
# Failure recorded but loop didn't crash.
|
||||
assert summary["gates"][0]["failure"]["error_class"] == "log-fetch-failed"
|
||||
|
||||
def test_unknown_gate_no_parser(self):
|
||||
statuses = [_gate(context="CI / mystery_gate", state="failure")]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: "log content",
|
||||
)
|
||||
failure = summary["gates"][0]["failure"]
|
||||
assert failure["error_class"] == "NoParserAvailable"
|
||||
|
||||
def test_forgejo_status_none(self):
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc", forgejo_status=None,
|
||||
log_fetcher=lambda _: pytest.fail("should not be called"),
|
||||
)
|
||||
assert summary["overall_state"] == "unknown"
|
||||
assert summary["gates"] == []
|
||||
|
||||
def test_parameterized_gate_name_normalized(self):
|
||||
"""Forgejo can emit "unit_tests-3.13" for matrix jobs."""
|
||||
statuses = [_gate(context="CI / unit_tests-3.13", state="failure")]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: "Failing scenarios:\n x.feature:5 Scenario: bar\n0 features passed, 1 failed, 0 skipped\n0 scenarios passed, 1 failed, 0 skipped\n",
|
||||
)
|
||||
failure = summary["gates"][0]["failure"]
|
||||
assert failure["parser_used"] == "behave"
|
||||
|
||||
def test_skip_coverage_session_no_parser(self):
|
||||
# benchmark is in NOX_SESSIONS_SKIP_COVERAGE; should not parse.
|
||||
statuses = [_gate(context="CI / benchmark", state="failure")]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: pytest.fail("should not fetch log"),
|
||||
)
|
||||
# benchmark isn't in NOX_SESSION_TO_PARSER, so gate→session
|
||||
# returns None → NoParserAvailable.
|
||||
failure = summary["gates"][0]["failure"]
|
||||
assert failure["error_class"] == "NoParserAvailable"
|
||||
|
||||
|
||||
# ─── parser_versions aggregate ─────────────────────────────────────
|
||||
|
||||
|
||||
class TestParserVersions:
|
||||
def test_versions_collected_per_parser(self):
|
||||
statuses = [
|
||||
_gate(context="CI / lint", state="failure"),
|
||||
_gate(context="CI / typecheck", state="failure"),
|
||||
]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: "Found 0 errors.\n",
|
||||
)
|
||||
# Both ruff + pyright versions appear.
|
||||
assert "ruff" in summary["parser_versions"]
|
||||
assert "pyright" in summary["parser_versions"]
|
||||
assert summary["parser_versions"]["ruff"] == "v1"
|
||||
|
||||
def test_stub_version_recorded(self):
|
||||
statuses = [_gate(context="CI / complexity", state="failure")]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc",
|
||||
forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: "radon log",
|
||||
)
|
||||
# complexity → radon (currently a stub).
|
||||
assert summary["parser_versions"].get("radon") == "v1-stub"
|
||||
|
||||
|
||||
# ─── gate count aggregates ─────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGateCounts:
|
||||
def test_mixed_statuses(self):
|
||||
statuses = [
|
||||
_gate(context="CI / lint", state="success"),
|
||||
_gate(context="CI / typecheck", state="failure"),
|
||||
_gate(context="CI / unit_tests", state="pending"),
|
||||
_gate(context="CI / build", state="error"),
|
||||
]
|
||||
summary = summarize_ci_status(
|
||||
head_sha="abc", forgejo_status=_forgejo_status(statuses),
|
||||
log_fetcher=lambda _: "",
|
||||
)
|
||||
assert summary["gates_passed"] == 1
|
||||
assert summary["gates_failed"] == 2 # failure + error
|
||||
assert summary["gates_pending"] == 1
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Shared types + helpers for per-CI-tool parsers.
|
||||
|
||||
Every parser module exposes:
|
||||
- ``PARSER_VERSION: str`` — bumped when output semantics change.
|
||||
- ``parse(log: str, gate_name: str) -> ParserResult``
|
||||
|
||||
Parsers are deterministic Python — no LLM. They take the gate's raw
|
||||
log text + the gate name, and return structured findings that fit the
|
||||
V1 ``CIFailure`` contract. Composite gates (e.g., nox security_scan =
|
||||
bandit + semgrep + vulture) get parsed by an orchestrator that calls
|
||||
each sub-parser and wraps the results in ``CIFailure.composite_findings``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..contracts.v1 import CIFailureFinding, FailedAssertion, FileLocation
|
||||
|
||||
|
||||
# Default log-excerpt budget (V1 CIFailure.raw_log_excerpt max_length).
|
||||
DEFAULT_LOG_EXCERPT_CHARS = 16_384
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParserResult:
|
||||
"""One parser's structured output for one gate's log.
|
||||
|
||||
``log_excerpt`` is the parser's selection of the most-relevant
|
||||
log lines (smart-around-first-error, or trailing N lines). It MUST
|
||||
fit within ``DEFAULT_LOG_EXCERPT_CHARS`` — the V1 contract enforces
|
||||
this as max_length on the CIFailure field.
|
||||
|
||||
``log_excerpt_lines`` is the count of newline-separated lines in
|
||||
the excerpt, used by the contract field of the same name.
|
||||
"""
|
||||
|
||||
error_class: str
|
||||
summary_line: str
|
||||
findings: list[CIFailureFinding] = field(default_factory=list)
|
||||
failing_locations: list[FileLocation] = field(default_factory=list)
|
||||
failed_assertions: list[FailedAssertion] = field(default_factory=list)
|
||||
log_excerpt: str = ""
|
||||
log_excerpt_lines: int = 0
|
||||
|
||||
|
||||
def select_log_excerpt(log: str, *, max_chars: int = DEFAULT_LOG_EXCERPT_CHARS,
|
||||
tail_lines: int = 80) -> tuple[str, int]:
|
||||
"""Pick the most-relevant excerpt from a log.
|
||||
|
||||
Strategy: prefer the LAST ``tail_lines`` (most test/build tools
|
||||
emit failure context at the end). If the tail exceeds ``max_chars``,
|
||||
truncate from the head of the tail (keep the bottom).
|
||||
|
||||
Returns ``(excerpt, line_count)``. An empty log produces ``("", 0)``.
|
||||
"""
|
||||
if not log:
|
||||
return "", 0
|
||||
lines = log.splitlines()
|
||||
if len(lines) <= tail_lines:
|
||||
excerpt = log
|
||||
else:
|
||||
excerpt = "\n".join(lines[-tail_lines:])
|
||||
if len(excerpt) > max_chars:
|
||||
excerpt = excerpt[-max_chars:]
|
||||
# Trim a partial leading line to keep it clean.
|
||||
nl = excerpt.find("\n")
|
||||
if 0 < nl < 200:
|
||||
excerpt = excerpt[nl + 1:]
|
||||
return excerpt, excerpt.count("\n") + (1 if excerpt and not excerpt.endswith("\n") else 0)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_LOG_EXCERPT_CHARS",
|
||||
"ParserResult",
|
||||
"select_log_excerpt",
|
||||
]
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Parser registry — resolve a parser name to a (parser_version,
|
||||
parse_function) pair.
|
||||
|
||||
Real parsers ship their own module under ``ci_summary_parsers/``. For
|
||||
the not-yet-shipped tools, the registry returns a stub from
|
||||
``_stub.make_stub(name)``. The startup coverage validator
|
||||
``validate_parser_coverage()`` flags any name in EXPECTED_PARSERS
|
||||
that still resolves to a stub so the operator can prioritize.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from . import EXPECTED_PARSERS, NOX_SESSION_TO_PARSER
|
||||
from . import _stub
|
||||
from ._base import ParserResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Parsers that ship in Phase 1j (real implementations).
|
||||
_REAL_PARSERS: frozenset[str] = frozenset({"ruff", "pyright", "behave"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedParser:
|
||||
name: str
|
||||
version: str
|
||||
parse: Callable[[str, str], ParserResult]
|
||||
is_stub: bool
|
||||
|
||||
|
||||
def resolve(parser_name: str) -> ResolvedParser:
|
||||
"""Look up the parser for ``parser_name``. Returns a stub if the
|
||||
real module isn't shipped yet (logs at WARNING level so operators
|
||||
notice the parser gap)."""
|
||||
if parser_name in _REAL_PARSERS:
|
||||
try:
|
||||
mod = importlib.import_module(
|
||||
f"tools.controller.ci_summary_parsers.{parser_name}",
|
||||
)
|
||||
return ResolvedParser(
|
||||
name=parser_name,
|
||||
version=getattr(mod, "PARSER_VERSION", "v1"),
|
||||
parse=mod.parse,
|
||||
is_stub=False,
|
||||
)
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"real parser %s declared but import failed; using stub",
|
||||
parser_name,
|
||||
)
|
||||
return ResolvedParser(
|
||||
name=parser_name,
|
||||
version=_stub.PARSER_VERSION,
|
||||
parse=_stub.make_stub(parser_name),
|
||||
is_stub=True,
|
||||
)
|
||||
|
||||
|
||||
def resolve_for_nox_session(nox_session: str) -> list[ResolvedParser]:
|
||||
"""Map nox session → ordered list of parsers (most sessions have
|
||||
one; composite security_scan has three).
|
||||
|
||||
Raises KeyError if the session has no mapping (forces operator to
|
||||
update NOX_SESSION_TO_PARSER explicitly)."""
|
||||
entry = NOX_SESSION_TO_PARSER[nox_session] # KeyError is intentional
|
||||
if entry.startswith("multi:"):
|
||||
names = entry.split(":", 1)[1].split(",")
|
||||
return [resolve(n.strip()) for n in names if n.strip()]
|
||||
return [resolve(entry)]
|
||||
|
||||
|
||||
def validate_parser_coverage() -> dict[str, bool]:
|
||||
"""Return a {parser_name: is_real} map for every parser in
|
||||
EXPECTED_PARSERS. Operators can plot this to see parser gaps."""
|
||||
return {p: not resolve(p).is_stub for p in sorted(EXPECTED_PARSERS)}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ResolvedParser",
|
||||
"resolve",
|
||||
"resolve_for_nox_session",
|
||||
"validate_parser_coverage",
|
||||
]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Stub parser for tools whose deterministic parser isn't shipped yet.
|
||||
|
||||
Returns a structured CIFailure-shape result that carries the raw log
|
||||
excerpt + a ``parser-pending`` marker. The implementer's prompt can
|
||||
still surface the failing log; the operator-side coverage dashboard
|
||||
flags these as parser gaps to prioritize.
|
||||
|
||||
Phase 1j ships ruff/pyright/behave; Phase 1j+ ships the rest. Until
|
||||
then, these tools use this stub:
|
||||
- robot_framework (integration_tests, e2e_tests)
|
||||
- slipcover (coverage_report)
|
||||
- bandit, semgrep, vulture (security_scan composite)
|
||||
- radon (complexity)
|
||||
- build (build)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from ..contracts.v1 import CIFailureFinding
|
||||
from ._base import ParserResult, select_log_excerpt
|
||||
|
||||
PARSER_VERSION = "v1-stub"
|
||||
|
||||
|
||||
def make_stub(parser_name: str):
|
||||
"""Factory: returns a ``parse(log, gate_name)`` for the given
|
||||
parser name. Each stub identifies itself in ``error_class``."""
|
||||
|
||||
def parse(log: str, gate_name: str) -> ParserResult:
|
||||
excerpt, lines = select_log_excerpt(log)
|
||||
finding = CIFailureFinding(
|
||||
error_class=f"parser-pending-{parser_name}",
|
||||
summary=(
|
||||
f"Structured parser for '{parser_name}' is not yet "
|
||||
f"implemented; falling back to raw log excerpt."
|
||||
)[:512],
|
||||
location=None,
|
||||
severity="error",
|
||||
)
|
||||
return ParserResult(
|
||||
error_class=f"parser-pending-{parser_name}",
|
||||
summary_line=(
|
||||
f"{parser_name}: parser pending; see raw_log_excerpt"
|
||||
)[:200],
|
||||
findings=[finding],
|
||||
failing_locations=[],
|
||||
failed_assertions=[],
|
||||
log_excerpt=excerpt,
|
||||
log_excerpt_lines=lines,
|
||||
)
|
||||
|
||||
parse.__name__ = f"parse_stub_{parser_name}"
|
||||
return parse
|
||||
|
||||
|
||||
__all__ = ["PARSER_VERSION", "make_stub"]
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Behave parser — handles ``nox -s unit_tests``.
|
||||
|
||||
Behave's CLI emits a tree-style trace per scenario:
|
||||
|
||||
Feature: My feature # tests/features/foo.feature:3
|
||||
Scenario: My scenario # tests/features/foo.feature:10
|
||||
Given foo # passed
|
||||
When bar # passed
|
||||
Then baz # failed: AssertionError: expected 1 == 2
|
||||
File "tests/steps/baz_steps.py", line 10, in step_then
|
||||
assert 1 == 2, "expected 1 == 2"
|
||||
AssertionError: expected 1 == 2
|
||||
|
||||
Failing scenarios:
|
||||
tests/features/foo.feature:10 Scenario: My scenario
|
||||
|
||||
0 features passed, 1 failed, 0 skipped
|
||||
0 scenarios passed, 1 failed, 0 skipped
|
||||
2 steps passed, 1 failed, 1 skipped
|
||||
|
||||
The parser extracts each failing scenario as a ``FailedAssertion``
|
||||
+ a ``CIFailureFinding`` so the implementer can navigate to the
|
||||
right feature file + step file.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..contracts.v1 import CIFailureFinding, FailedAssertion, FileLocation
|
||||
from ._base import ParserResult, select_log_excerpt
|
||||
|
||||
PARSER_VERSION = "v1"
|
||||
|
||||
# " tests/features/foo.feature:10 Scenario: My scenario"
|
||||
_FAILING_SCENARIO_RE = re.compile(
|
||||
r"^\s+(?P<feature>\S+\.feature):(?P<line>\d+)\s+Scenario:\s+(?P<name>.+?)\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
# "1 feature passed, 0 failed, 0 skipped"
|
||||
_FEATURE_SUMMARY_RE = re.compile(
|
||||
r"(\d+)\s+features?\s+passed,\s+(\d+)\s+failed,\s+(\d+)\s+skipped",
|
||||
)
|
||||
_SCENARIO_SUMMARY_RE = re.compile(
|
||||
r"(\d+)\s+scenarios?\s+passed,\s+(\d+)\s+failed,\s+(\d+)\s+skipped",
|
||||
)
|
||||
# AssertionError line OR Exception line.
|
||||
_ASSERT_RE = re.compile(
|
||||
r"^\s*(?P<exc>AssertionError|Exception|.+Error):\s*(?P<msg>.+?)\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def parse(log: str, gate_name: str) -> ParserResult:
|
||||
findings: list[CIFailureFinding] = []
|
||||
locations: list[FileLocation] = []
|
||||
assertions: list[FailedAssertion] = []
|
||||
|
||||
failing_scenarios = list(_FAILING_SCENARIO_RE.finditer(log))
|
||||
|
||||
# Map scenario → an excerpt of the surrounding log so we can lift
|
||||
# the assertion line for the FailedAssertion.
|
||||
for m in failing_scenarios:
|
||||
loc = FileLocation(
|
||||
file_path=m["feature"],
|
||||
line_range=f"{m['line']}-{m['line']}",
|
||||
function_or_test=m["name"][:120],
|
||||
)
|
||||
findings.append(CIFailureFinding(
|
||||
error_class="ScenarioFailed",
|
||||
summary=f"Scenario failed: {m['name'][:200]}"[:512],
|
||||
location=loc,
|
||||
severity="error",
|
||||
))
|
||||
locations.append(loc)
|
||||
|
||||
# Pull the first matching assertion globally — multi-assertion
|
||||
# extraction would over-fit on rare logs. The raw excerpt carries
|
||||
# the rest of the context.
|
||||
for am in _ASSERT_RE.finditer(log):
|
||||
exc = am["exc"].strip()
|
||||
if exc.endswith("Error"):
|
||||
msg = am["msg"].strip()[:512]
|
||||
# Best-effort context: 10 lines around the match.
|
||||
excerpt = _excerpt_around(log, am.start(), 10)
|
||||
assertions.append(FailedAssertion(
|
||||
test_name=(failing_scenarios[0]["name"][:120]
|
||||
if failing_scenarios else "(unknown)"),
|
||||
expected=None,
|
||||
actual=None,
|
||||
assertion_excerpt=(f"{exc}: {msg}\n\n{excerpt}")[:4096],
|
||||
))
|
||||
break
|
||||
|
||||
summary_line = _summary_line(log, n_failing=len(failing_scenarios))
|
||||
excerpt, lines = select_log_excerpt(log)
|
||||
error_class = "BehaveFailure" if findings else "BehaveClean"
|
||||
|
||||
return ParserResult(
|
||||
error_class=error_class,
|
||||
summary_line=summary_line,
|
||||
findings=findings,
|
||||
failing_locations=locations,
|
||||
failed_assertions=assertions,
|
||||
log_excerpt=excerpt,
|
||||
log_excerpt_lines=lines,
|
||||
)
|
||||
|
||||
|
||||
def _summary_line(log: str, *, n_failing: int) -> str:
|
||||
feat = _FEATURE_SUMMARY_RE.search(log)
|
||||
sce = _SCENARIO_SUMMARY_RE.search(log)
|
||||
if feat and sce:
|
||||
return (
|
||||
f"Behave: {feat.group(2)} feature(s) failed, "
|
||||
f"{sce.group(2)} scenario(s) failed"
|
||||
)[:200]
|
||||
if n_failing:
|
||||
return f"Behave: {n_failing} scenario(s) failed"[:200]
|
||||
return "Behave: no failing scenarios parsed"
|
||||
|
||||
|
||||
def _excerpt_around(log: str, offset: int, lines_each_side: int) -> str:
|
||||
# Walk N newlines back, N newlines forward.
|
||||
before = log.rfind("\n", 0, offset)
|
||||
for _ in range(lines_each_side):
|
||||
if before <= 0:
|
||||
before = 0
|
||||
break
|
||||
before = log.rfind("\n", 0, before)
|
||||
if before == -1:
|
||||
before = 0
|
||||
break
|
||||
after = offset
|
||||
for _ in range(lines_each_side):
|
||||
nxt = log.find("\n", after + 1)
|
||||
if nxt == -1:
|
||||
after = len(log)
|
||||
break
|
||||
after = nxt
|
||||
return log[before:after].strip("\n")[:2048]
|
||||
|
||||
|
||||
__all__ = ["PARSER_VERSION", "parse"]
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Pyright parser — handles ``nox -s typecheck``.
|
||||
|
||||
Pyright's CLI emits one diagnostic per line:
|
||||
|
||||
/abs/path/to/file.py:LINE:COL - error: human description (ruleName)
|
||||
/abs/path/to/file.py:LINE:COL - warning: ...
|
||||
1 error, 0 warnings, 0 informations
|
||||
|
||||
Some Pyright versions use absolute paths; we normalize to the
|
||||
project-relative tail when a known segment (``tools/``, ``tests/``,
|
||||
``src/``) is in the path. Otherwise the absolute path is kept.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..contracts.v1 import CIFailureFinding, FileLocation
|
||||
from ._base import ParserResult, select_log_excerpt
|
||||
|
||||
PARSER_VERSION = "v1"
|
||||
|
||||
_DIAG_RE = re.compile(
|
||||
r"^(?P<file>\S+?):(?P<line>\d+):(?P<col>\d+)\s+-\s+"
|
||||
r"(?P<severity>error|warning|information)\s*:\s+"
|
||||
r"(?P<message>.+?)\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
# Some Pyright versions emit findings on TWO lines (path/loc then
|
||||
# indented message); we don't try to parse those (rare). The summary
|
||||
# line is canonical:
|
||||
_SUMMARY_RE = re.compile(
|
||||
r"(\d+)\s+errors?,\s+(\d+)\s+warnings?,\s+(\d+)\s+informations?",
|
||||
)
|
||||
|
||||
_PATH_NORMALIZE_SEGMENTS = ("/tools/", "/tests/", "/src/", "/.opencode/")
|
||||
|
||||
|
||||
def parse(log: str, gate_name: str) -> ParserResult:
|
||||
findings: list[CIFailureFinding] = []
|
||||
locations: list[FileLocation] = []
|
||||
|
||||
for m in _DIAG_RE.finditer(log):
|
||||
sev_raw = m["severity"]
|
||||
if sev_raw == "error":
|
||||
severity = "error"
|
||||
error_class = _extract_rule_name(m["message"]) or "PyrightError"
|
||||
elif sev_raw == "warning":
|
||||
severity = "warning"
|
||||
error_class = _extract_rule_name(m["message"]) or "PyrightWarning"
|
||||
else:
|
||||
severity = "info"
|
||||
error_class = "PyrightInfo"
|
||||
loc = FileLocation(
|
||||
file_path=_normalize_path(m["file"]),
|
||||
line_range=f"{m['line']}-{m['line']}",
|
||||
)
|
||||
findings.append(CIFailureFinding(
|
||||
error_class=error_class,
|
||||
summary=m["message"].strip()[:512],
|
||||
location=loc,
|
||||
severity=severity,
|
||||
))
|
||||
if severity == "error":
|
||||
locations.append(loc)
|
||||
|
||||
summary_line = _summary_line(log, n_findings=len(findings))
|
||||
excerpt, lines = select_log_excerpt(log)
|
||||
error_class = _aggregate_error_class(findings)
|
||||
|
||||
return ParserResult(
|
||||
error_class=error_class,
|
||||
summary_line=summary_line,
|
||||
findings=findings,
|
||||
failing_locations=locations,
|
||||
failed_assertions=[],
|
||||
log_excerpt=excerpt,
|
||||
log_excerpt_lines=lines,
|
||||
)
|
||||
|
||||
|
||||
def _extract_rule_name(message: str) -> str | None:
|
||||
"""Pyright wraps the rule name in trailing parens. Pull it out."""
|
||||
m = re.search(r"\((report[A-Za-z0-9]+)\)\s*$", message)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
"""Strip leading container path so the implementer can match the
|
||||
workspace tree."""
|
||||
for seg in _PATH_NORMALIZE_SEGMENTS:
|
||||
idx = path.find(seg)
|
||||
if idx >= 0:
|
||||
return path[idx + 1:]
|
||||
return path
|
||||
|
||||
|
||||
def _summary_line(log: str, *, n_findings: int) -> str:
|
||||
m = _SUMMARY_RE.search(log)
|
||||
if m:
|
||||
return (
|
||||
f"Pyright: {m.group(1)} errors, {m.group(2)} warnings, "
|
||||
f"{m.group(3)} informations"
|
||||
)[:200]
|
||||
return f"Pyright: {n_findings} diagnostic(s) parsed"[:200]
|
||||
|
||||
|
||||
def _aggregate_error_class(findings: list[CIFailureFinding]) -> str:
|
||||
errors = [f for f in findings if f.severity == "error"]
|
||||
if not errors:
|
||||
return "PyrightClean"
|
||||
classes = {f.error_class for f in errors}
|
||||
if len(classes) == 1:
|
||||
return next(iter(classes))
|
||||
return "PyrightMixed"
|
||||
|
||||
|
||||
__all__ = ["PARSER_VERSION", "parse"]
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Ruff parser — handles ``nox -s lint`` and ``nox -s format``.
|
||||
|
||||
Ruff's CLI prints one finding per line in the format:
|
||||
|
||||
path/to/file.py:LINE:COL: RULECODE human description
|
||||
|
||||
Followed by a trailing ``Found N errors.`` summary line. Format mode
|
||||
(`ruff format --check`) emits a different shape:
|
||||
|
||||
Would reformat: path/to/file.py
|
||||
1 file would be reformatted
|
||||
|
||||
Both shapes are absorbed below. Findings are emitted as
|
||||
``CIFailureFinding`` rows so the implementer's prompt can render
|
||||
"rule + file:line" with one entry per failure.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..contracts.v1 import CIFailureFinding, FileLocation
|
||||
from ._base import ParserResult, select_log_excerpt
|
||||
|
||||
PARSER_VERSION = "v1"
|
||||
|
||||
# Examples:
|
||||
# tools/controller/master/loop.py:42:5: F401 `os` imported but unused
|
||||
# tests/x.py:1:1: E501 line too long (90 > 88 characters)
|
||||
_LINT_RE = re.compile(
|
||||
r"^(?P<file>[^\s:][^:]*):(?P<line>\d+):(?P<col>\d+):\s+"
|
||||
r"(?P<code>[A-Z]+\d+)\s+(?P<message>.+)$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
# Examples: ``Would reformat: tools/foo.py``
|
||||
_FORMAT_RE = re.compile(r"^Would reformat:\s+(?P<file>\S+)\s*$", re.MULTILINE)
|
||||
# Summary lines we extract for summary_line in priority order.
|
||||
_SUMMARY_PATTERNS = [
|
||||
re.compile(r"^Found (\d+) errors?\.?$", re.MULTILINE),
|
||||
re.compile(r"^(\d+) files? would be reformatted$", re.MULTILINE),
|
||||
]
|
||||
|
||||
|
||||
def parse(log: str, gate_name: str) -> ParserResult:
|
||||
findings: list[CIFailureFinding] = []
|
||||
locations: list[FileLocation] = []
|
||||
|
||||
for m in _LINT_RE.finditer(log):
|
||||
loc = FileLocation(
|
||||
file_path=m["file"],
|
||||
line_range=f"{m['line']}-{m['line']}",
|
||||
)
|
||||
findings.append(CIFailureFinding(
|
||||
error_class=m["code"],
|
||||
summary=m["message"].strip()[:512],
|
||||
location=loc,
|
||||
severity="error",
|
||||
))
|
||||
locations.append(loc)
|
||||
|
||||
for m in _FORMAT_RE.finditer(log):
|
||||
loc = FileLocation(file_path=m["file"])
|
||||
findings.append(CIFailureFinding(
|
||||
error_class="format",
|
||||
summary="file would be reformatted",
|
||||
location=loc,
|
||||
severity="warning",
|
||||
))
|
||||
locations.append(loc)
|
||||
|
||||
summary_line = _pick_summary(log, fallback=f"{len(findings)} ruff findings")
|
||||
excerpt, lines = select_log_excerpt(log)
|
||||
error_class = _aggregate_error_class(findings)
|
||||
|
||||
return ParserResult(
|
||||
error_class=error_class,
|
||||
summary_line=summary_line[:200],
|
||||
findings=findings,
|
||||
failing_locations=locations,
|
||||
failed_assertions=[],
|
||||
log_excerpt=excerpt,
|
||||
log_excerpt_lines=lines,
|
||||
)
|
||||
|
||||
|
||||
def _pick_summary(log: str, *, fallback: str) -> str:
|
||||
for pat in _SUMMARY_PATTERNS:
|
||||
matches = pat.findall(log)
|
||||
if matches:
|
||||
# Re-render: the regex captured the count.
|
||||
n = matches[-1]
|
||||
if pat is _SUMMARY_PATTERNS[0]:
|
||||
return f"Ruff found {n} error(s)"
|
||||
return f"Ruff: {n} file(s) need reformat"
|
||||
return fallback
|
||||
|
||||
|
||||
def _aggregate_error_class(findings: list[CIFailureFinding]) -> str:
|
||||
"""If all findings share one error_class, surface it. Else use a
|
||||
composite marker so the implementer's prompt shows a sensible
|
||||
top-level error_class on the CIFailure."""
|
||||
if not findings:
|
||||
return "RuffEmpty"
|
||||
classes = {f.error_class for f in findings}
|
||||
if len(classes) == 1:
|
||||
return next(iter(classes))
|
||||
return "RuffMixed"
|
||||
|
||||
|
||||
__all__ = ["PARSER_VERSION", "parse"]
|
||||
@@ -71,6 +71,10 @@ from .backfill import (
|
||||
has_backfill_run,
|
||||
run_startup_backfill,
|
||||
)
|
||||
from .ci_summarize import (
|
||||
LogFetcher,
|
||||
summarize_ci_status,
|
||||
)
|
||||
from .prefetch import (
|
||||
GetPRDetailsCallback,
|
||||
GetPRDiffCallback,
|
||||
@@ -166,4 +170,7 @@ __all__ = [
|
||||
"build_implementer_input",
|
||||
"build_reviewer_input",
|
||||
"make_prefetch_callback",
|
||||
# CI summarizer (Phase 1j)
|
||||
"LogFetcher",
|
||||
"summarize_ci_status",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
"""Deterministic CI summarizer — builds CISummary from Forgejo status.
|
||||
|
||||
The implementer/reviewer prompt prefetch needs a CISummary V1 dict for
|
||||
the head_sha. This module takes the Forgejo combined-status response
|
||||
(``{"state": "...", "statuses": [...]}``) plus a log fetcher, and
|
||||
returns a fully-populated CISummary covering every gate.
|
||||
|
||||
Per-gate flow:
|
||||
1. Map the Forgejo gate context (e.g., "CI / lint") → nox session
|
||||
name. The job names in ``.forgejo/workflows/*.yml`` ARE the nox
|
||||
session names by convention, so a simple suffix-after-/ extraction
|
||||
works. Unknown contexts pass through as ``UnknownGate`` and skip
|
||||
the parser.
|
||||
2. Look up the nox session in NOX_SESSION_TO_PARSER → one or more
|
||||
parser names.
|
||||
3. For each parser, fetch the log (via the injected ``log_fetcher``),
|
||||
run ``parse(log, gate_name)``, and wrap the result in a
|
||||
GateResult.failure CIFailure dict.
|
||||
4. For composite gates (security_scan = bandit + semgrep + vulture),
|
||||
the outer CIFailure carries the FIRST parser's structure + each
|
||||
sub-parser's result goes into ``composite_findings``.
|
||||
|
||||
The summarizer never raises — log fetch failures degrade to a
|
||||
``CIFailure`` with ``error_class='log-fetch-failed'`` so the
|
||||
implementer at least sees the failure existed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from ..ci_summary_parsers import NOX_SESSION_TO_PARSER, NOX_SESSIONS_SKIP_COVERAGE
|
||||
from ..ci_summary_parsers._registry import (
|
||||
ResolvedParser,
|
||||
resolve_for_nox_session,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# log_fetcher(gate_name) -> str | None; production wires this to a
|
||||
# Forgejo job-log fetcher. None means the log was unreachable.
|
||||
LogFetcher = Callable[[str], str | None]
|
||||
|
||||
|
||||
# Forgejo status state → CISummary GateResult.status mapping.
|
||||
_FORGEJO_STATE_TO_GATE_STATUS = {
|
||||
"success": "passed",
|
||||
"failure": "failed",
|
||||
"error": "error",
|
||||
"pending": "pending",
|
||||
"warning": "passed", # advisory: treat as passed
|
||||
None: "pending",
|
||||
}
|
||||
|
||||
|
||||
def summarize_ci_status(
|
||||
*, head_sha: str, forgejo_status: dict | None, log_fetcher: LogFetcher,
|
||||
observed_at: datetime | None = None,
|
||||
) -> dict:
|
||||
"""Build a CISummary-shape dict from the Forgejo combined-status.
|
||||
|
||||
Args:
|
||||
head_sha: the commit SHA the status covers.
|
||||
forgejo_status: response body from
|
||||
``GET /repos/{owner}/{repo}/commits/{sha}/status``.
|
||||
Shape: ``{"state": str, "statuses": [{"context": str,
|
||||
"state": str, "target_url": str}]}``. None for
|
||||
"couldn't fetch" — produces a unknown-state summary.
|
||||
log_fetcher: callable that takes the gate context and returns
|
||||
the raw log text (or None if unreachable). Production
|
||||
wires this via Forgejo's job-log endpoint.
|
||||
observed_at: timestamp the status was fetched. Defaults to
|
||||
now(UTC).
|
||||
|
||||
Returns the CISummary V1-shape dict (a CISummary contract parses
|
||||
it after the caller validates).
|
||||
"""
|
||||
when = observed_at or datetime.now(timezone.utc)
|
||||
if forgejo_status is None:
|
||||
return _unknown_summary(head_sha=head_sha, observed_at=when)
|
||||
|
||||
overall = _FORGEJO_OVERALL_STATE_MAPPING.get(
|
||||
forgejo_status.get("state"), "unknown",
|
||||
)
|
||||
raw_statuses = forgejo_status.get("statuses") or []
|
||||
statuses = [s for s in raw_statuses if isinstance(s, dict)]
|
||||
|
||||
gates: list[dict] = []
|
||||
parser_versions: dict[str, str] = {}
|
||||
for s in statuses:
|
||||
gate, versions_seen = _build_gate(s, log_fetcher)
|
||||
gates.append(gate)
|
||||
parser_versions.update(versions_seen)
|
||||
|
||||
counts = _count_gates(gates)
|
||||
return {
|
||||
"summary_version": "V1",
|
||||
"head_sha": head_sha,
|
||||
"observed_at": when,
|
||||
"overall_state": overall,
|
||||
"gates": gates,
|
||||
"gates_total": counts["total"],
|
||||
"gates_passed": counts["passed"],
|
||||
"gates_failed": counts["failed"],
|
||||
"gates_skipped": counts["skipped"],
|
||||
"gates_pending": counts["pending"],
|
||||
"parser_versions": parser_versions,
|
||||
}
|
||||
|
||||
|
||||
# Forgejo combined-status states → CISummary.overall_state.
|
||||
_FORGEJO_OVERALL_STATE_MAPPING: dict[str | None, str] = {
|
||||
"success": "success",
|
||||
"failure": "failure",
|
||||
"error": "error",
|
||||
"pending": "pending",
|
||||
None: "unknown",
|
||||
}
|
||||
|
||||
|
||||
def _unknown_summary(*, head_sha: str, observed_at: datetime) -> dict:
|
||||
return {
|
||||
"summary_version": "V1",
|
||||
"head_sha": head_sha,
|
||||
"observed_at": observed_at,
|
||||
"overall_state": "unknown",
|
||||
"gates": [],
|
||||
"gates_total": 0,
|
||||
"gates_passed": 0,
|
||||
"gates_failed": 0,
|
||||
"gates_skipped": 0,
|
||||
"gates_pending": 0,
|
||||
"parser_versions": {},
|
||||
}
|
||||
|
||||
|
||||
def _build_gate(
|
||||
status: dict, log_fetcher: LogFetcher,
|
||||
) -> tuple[dict, dict[str, str]]:
|
||||
"""Build one GateResult dict + return parser versions used."""
|
||||
context = status.get("context") or "(unknown)"
|
||||
state_raw = status.get("state")
|
||||
gate_status = _FORGEJO_STATE_TO_GATE_STATUS.get(state_raw, "pending")
|
||||
target_url = status.get("target_url")
|
||||
severity = "info" if gate_status in {"passed", "skipped"} else "error"
|
||||
|
||||
gate: dict[str, Any] = {
|
||||
"name": context,
|
||||
"status": gate_status,
|
||||
"severity": severity,
|
||||
"target_url": target_url,
|
||||
"duration_seconds": None,
|
||||
"failure": None,
|
||||
}
|
||||
parser_versions: dict[str, str] = {}
|
||||
|
||||
# Only parse failures/errors. Passed/pending/skipped don't need a
|
||||
# log fetch — they have nothing to report.
|
||||
if gate_status not in {"failed", "error"}:
|
||||
return gate, parser_versions
|
||||
|
||||
nox_session = _gate_to_nox_session(context)
|
||||
if nox_session is None or nox_session in NOX_SESSIONS_SKIP_COVERAGE:
|
||||
gate["failure"] = _no_parser_failure(context)
|
||||
return gate, parser_versions
|
||||
|
||||
try:
|
||||
parsers = resolve_for_nox_session(nox_session)
|
||||
except KeyError:
|
||||
logger.warning(
|
||||
"no parser map for nox session %r (gate=%r)",
|
||||
nox_session, context,
|
||||
)
|
||||
gate["failure"] = _no_parser_failure(context)
|
||||
return gate, parser_versions
|
||||
|
||||
log = _safe_fetch(log_fetcher, context)
|
||||
if log is None:
|
||||
gate["failure"] = _log_fetch_failed_failure(parsers, context)
|
||||
for p in parsers:
|
||||
parser_versions[p.name] = p.version
|
||||
return gate, parser_versions
|
||||
|
||||
gate["failure"] = _build_failure(parsers, log, context, parser_versions)
|
||||
return gate, parser_versions
|
||||
|
||||
|
||||
def _gate_to_nox_session(context: str) -> str | None:
|
||||
"""Forgejo gate context → nox session name.
|
||||
|
||||
Convention: ``<workflow-name> / <job-name>`` (e.g., "CI / lint").
|
||||
The job name IS the nox session name by repo convention. If the
|
||||
name doesn't match any known session, return None and let the
|
||||
summarizer fall through to NoParserAvailable."""
|
||||
if not context:
|
||||
return None
|
||||
tail = context.rsplit("/", 1)[-1].strip()
|
||||
if tail in NOX_SESSION_TO_PARSER:
|
||||
return tail
|
||||
# Also support the parameterized form e.g. "unit_tests-3.13".
|
||||
base = tail.split("-", 1)[0]
|
||||
if base in NOX_SESSION_TO_PARSER:
|
||||
return base
|
||||
return None
|
||||
|
||||
|
||||
def _safe_fetch(log_fetcher: LogFetcher, context: str) -> str | None:
|
||||
try:
|
||||
return log_fetcher(context)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("log fetch raised for %s: %s", context, exc)
|
||||
return None
|
||||
|
||||
|
||||
def _no_parser_failure(context: str) -> dict:
|
||||
return {
|
||||
"parser_used": "(none)",
|
||||
"parser_version": "n/a",
|
||||
"error_class": "NoParserAvailable",
|
||||
"summary_line": f"No parser available for {context!r}"[:200],
|
||||
"findings": [],
|
||||
"failing_locations": [],
|
||||
"failed_assertions": [],
|
||||
"raw_log_excerpt": "",
|
||||
"log_excerpt_lines": 0,
|
||||
"composite_findings": [],
|
||||
}
|
||||
|
||||
|
||||
def _log_fetch_failed_failure(parsers, context: str) -> dict:
|
||||
parser_names = ", ".join(p.name for p in parsers)
|
||||
return {
|
||||
"parser_used": parser_names or "(unknown)",
|
||||
"parser_version": "n/a",
|
||||
"error_class": "log-fetch-failed",
|
||||
"summary_line": f"Log fetch failed for {context!r}"[:200],
|
||||
"findings": [],
|
||||
"failing_locations": [],
|
||||
"failed_assertions": [],
|
||||
"raw_log_excerpt": "",
|
||||
"log_excerpt_lines": 0,
|
||||
"composite_findings": [],
|
||||
}
|
||||
|
||||
|
||||
def _build_failure(
|
||||
parsers: list[ResolvedParser], log: str, context: str,
|
||||
parser_versions: dict[str, str],
|
||||
) -> dict:
|
||||
"""Build the CIFailure dict from one or more parsers' results.
|
||||
|
||||
For a single-parser gate the result becomes the outer CIFailure.
|
||||
For a composite gate (multi: prefix), the FIRST parser becomes
|
||||
the outer + the rest go into composite_findings.
|
||||
"""
|
||||
results = []
|
||||
for p in parsers:
|
||||
try:
|
||||
res = p.parse(log, context)
|
||||
parser_versions[p.name] = p.version
|
||||
results.append((p, res))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"parser %s raised on gate %r: %s", p.name, context, exc,
|
||||
)
|
||||
# Fall through with no result for this parser.
|
||||
|
||||
if not results:
|
||||
return _log_fetch_failed_failure(parsers, context)
|
||||
|
||||
head_parser, head_result = results[0]
|
||||
failure = _result_to_failure(head_parser, head_result)
|
||||
if len(results) > 1:
|
||||
failure["composite_findings"] = [
|
||||
_result_to_failure(p, r) for p, r in results[1:]
|
||||
]
|
||||
return failure
|
||||
|
||||
|
||||
def _result_to_failure(parser: ResolvedParser, result) -> dict:
|
||||
return {
|
||||
"parser_used": parser.name,
|
||||
"parser_version": parser.version,
|
||||
"error_class": result.error_class,
|
||||
"summary_line": result.summary_line,
|
||||
"findings": [f.model_dump() for f in result.findings],
|
||||
"failing_locations": [loc.model_dump() for loc in result.failing_locations],
|
||||
"failed_assertions": [a.model_dump() for a in result.failed_assertions],
|
||||
"raw_log_excerpt": result.log_excerpt,
|
||||
"log_excerpt_lines": result.log_excerpt_lines,
|
||||
"composite_findings": [],
|
||||
}
|
||||
|
||||
|
||||
def _count_gates(gates: list[dict]) -> dict[str, int]:
|
||||
counts = {"total": len(gates), "passed": 0, "failed": 0,
|
||||
"skipped": 0, "pending": 0}
|
||||
for g in gates:
|
||||
status = g.get("status")
|
||||
if status == "passed":
|
||||
counts["passed"] += 1
|
||||
elif status in {"failed", "error"}:
|
||||
counts["failed"] += 1
|
||||
elif status == "skipped":
|
||||
counts["skipped"] += 1
|
||||
elif status == "pending":
|
||||
counts["pending"] += 1
|
||||
return counts
|
||||
|
||||
|
||||
__all__ = ["LogFetcher", "summarize_ci_status"]
|
||||
Reference in New Issue
Block a user