diff --git a/tests/auto_agents/controller/test_entry_points.py b/tests/auto_agents/controller/test_entry_points.py index ab7389aa5..1e30bd90d 100644 --- a/tests/auto_agents/controller/test_entry_points.py +++ b/tests/auto_agents/controller/test_entry_points.py @@ -103,6 +103,7 @@ class TestMasterOptInLabelFlag: get_pr_diff: callable = lambda o, r, n: None list_pr_reviews: callable = lambda o, r, n: [] list_pr_comments: callable = lambda o, r, n: [] + get_ci_status: callable = lambda o, r, sha: None return _StubCB() def _stub_cfg(self): diff --git a/tests/auto_agents/controller/test_master_ci_status_poll.py b/tests/auto_agents/controller/test_master_ci_status_poll.py new file mode 100644 index 000000000..3cb78e323 --- /dev/null +++ b/tests/auto_agents/controller/test_master_ci_status_poll.py @@ -0,0 +1,274 @@ +"""Tests for the AWAITING_CI status poll (Phase 1k++++ trial).""" +from __future__ import annotations + +import json + +import pytest +from sqlalchemy import text + +from tools.controller.db import ( + Workflow, + WorkflowAttempt, + build_engine, + create_all, + session_scope, +) +from tools.controller.master.ci_status_poll import ( + CIStatusPollReport, + run_ci_status_poll_tick, +) + + +@pytest.fixture +def engine(): + eng = build_engine("sqlite:///:memory:") + create_all(eng) + yield eng + + +def _seed_awaiting_ci_with_attempt( + engine, *, head_sha_after: str | None = "abc1234567", + entity_number: int = 30, +) -> int: + """Seed an AWAITING_CI workflow with a completed implementer + attempt whose head_sha_after is set (so ci_status_poll has a SHA + to query).""" + with session_scope(engine) as s: + w = Workflow( + kind="pr", owner="o", repo="r", entity_number=entity_number, + current_state="AWAITING_CI", + ) + s.add(w); s.flush() + wf_id = w.workflow_id + a = WorkflowAttempt( + workflow_id=wf_id, attempt_number=1, role="implementer", + tier=0, status="complete", + input_payload={}, input_version="V1", + output_payload={"output_version": "V1", "outcome": "resolved", + "files_touched": ["x.py"], "commit_shas": [], + "confidence": "high", "blockers": [], + "used_tier": 0, "wallclock_seconds": 1.0}, + output_version="V1", outcome="resolved", + head_sha_before="def1234567", + head_sha_after=head_sha_after, + ) + s.add(a) + return wf_id + + +class TestHappyPaths: + def test_empty_db_no_op(self, engine): + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: pytest.fail("no fetch"), + ) + assert report.workflows_scanned == 0 + assert report.workflows_advanced_green == 0 + assert report.workflows_advanced_red == 0 + + def test_ci_success_transitions_to_reviewing(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "success"}, + ) + assert report.workflows_advanced_green == 1 + with session_scope(engine) as s: + w = s.query(Workflow).filter_by(workflow_id=wf_id).one() + assert w.current_state == "REVIEWING" + + def test_ci_failure_transitions_back_to_implementing(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "failure"}, + ) + assert report.workflows_advanced_red == 1 + with session_scope(engine) as s: + w = s.query(Workflow).filter_by(workflow_id=wf_id).one() + assert w.current_state == "IMPLEMENTING" + + def test_ci_pending_waits_no_transition(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "pending"}, + ) + assert report.workflows_waiting == 1 + with session_scope(engine) as s: + w = s.query(Workflow).filter_by(workflow_id=wf_id).one() + assert w.current_state == "AWAITING_CI" + + def test_fetch_failure_no_op(self, engine): + _seed_awaiting_ci_with_attempt(engine) + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: None, + ) + assert report.workflows_fetch_failed == 1 + assert report.workflows_advanced_green == 0 + + +class TestErrorPaths: + def test_callback_raises_handled(self, engine): + _seed_awaiting_ci_with_attempt(engine) + + def boom(o, r, sha): + raise ConnectionError("forgejo down") + + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", get_ci_status=boom, + ) + assert report.workflows_fetch_failed == 1 + + def test_workflow_without_head_sha_skipped(self, engine): + """An AWAITING_CI workflow with no completed implementer + attempt that pushed (head_sha_after is NULL) → skip.""" + _seed_awaiting_ci_with_attempt(engine, head_sha_after=None) + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: pytest.fail("no fetch"), + ) + assert report.workflows_waiting == 1 + + +class TestEventRows: + def test_green_emits_ci_green_event(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "success"}, + ) + with session_scope(engine) as s: + events = s.execute( + text( + "SELECT event_type, from_state, to_state, payload " + "FROM controller_events " + "WHERE event_type = 'ci-green'" + ), + ).all() + assert len(events) == 1 + ev = events[0] + assert ev.from_state == "AWAITING_CI" + assert ev.to_state == "REVIEWING" + payload = json.loads(ev.payload) + assert payload["reason"] == "ci_green" + assert payload["ci_state"] == "success" + + def test_red_emits_ci_red_event(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "failure"}, + ) + with session_scope(engine) as s: + events = s.execute( + text( + "SELECT event_type, payload FROM controller_events " + "WHERE event_type = 'ci-red'" + ), + ).all() + assert len(events) == 1 + payload = json.loads(events[0].payload) + assert payload["reason"] == "ci_red_retry_same_tier" + + +class TestExtendedStateMapping: + def test_cancelled_treated_as_red(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "cancelled"}, + ) + with session_scope(engine) as s: + w = s.query(Workflow).filter_by(workflow_id=wf_id).one() + assert w.current_state == "IMPLEMENTING" + + def test_neutral_treated_as_green(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "neutral"}, + ) + with session_scope(engine) as s: + w = s.query(Workflow).filter_by(workflow_id=wf_id).one() + assert w.current_state == "REVIEWING" + + def test_in_progress_waits(self, engine): + wf_id = _seed_awaiting_ci_with_attempt(engine) + report = run_ci_status_poll_tick( + engine, owner="o", repo="r", + get_ci_status=lambda o, r, sha: {"state": "in_progress"}, + ) + assert report.workflows_waiting == 1 + + +class TestOtherRepoExcluded: + def test_workflows_for_other_repo_not_targeted(self, engine): + _seed_awaiting_ci_with_attempt(engine) + # Poll a different (owner, repo). + report = run_ci_status_poll_tick( + engine, owner="other", repo="other", + get_ci_status=lambda o, r, sha: pytest.fail("no fetch"), + ) + assert report.workflows_scanned == 0 + + +class TestLoopIntegration: + def test_loop_runs_ci_status_poll_on_cadence(self, engine): + """End-to-end through master_main_loop: AWAITING_CI → + ci_status_poll fetches success → REVIEWING. Pins the + autonomous trial path (no operator SQL needed).""" + import threading + from tools.controller.master.loop import ( + MasterConfig, + MasterTickReport, + master_main_loop, + ) + + wf_id = _seed_awaiting_ci_with_attempt(engine) + stop = threading.Event() + seen: list[MasterTickReport] = [] + + def on_iter(r: MasterTickReport): + seen.append(r) + if ( + r.ci_status_poll + and r.ci_status_poll.workflows_advanced_green + ): + stop.set() + + # Safety net so the test fails fast on regression. + safety = threading.Timer(5.0, stop.set) + safety.daemon = True + safety.start() + try: + cfg = MasterConfig( + tick_interval_s=0.05, + reaper_interval_s=10, + reconciliation_interval_s=10, + ci_poll_exhaustion_interval_s=10, + ci_status_poll_interval_s=0.05, + ) + master_main_loop( + engine, config=cfg, stop_event=stop, + on_iteration=on_iter, + ci_status_poll_args=( + "o", "r", lambda o, r, sha: {"state": "success"}, + ), + ) + finally: + safety.cancel() + + advanced = [ + r for r in seen + if r.ci_status_poll + and r.ci_status_poll.workflows_advanced_green + ] + assert advanced, ( + "ci_status_poll never saw a green transition; safety timer " + "likely fired first (test logic regressed)" + ) + with session_scope(engine) as s: + w = s.query(Workflow).filter_by(workflow_id=wf_id).one() + assert w.current_state == "REVIEWING" diff --git a/tools/controller/deploy/RUNBOOK.md b/tools/controller/deploy/RUNBOOK.md index 62fc6b9b6..53f7e6291 100644 --- a/tools/controller/deploy/RUNBOOK.md +++ b/tools/controller/deploy/RUNBOOK.md @@ -291,37 +291,29 @@ the fallback instruction work fine. If an agent ignores the fallback `worker-internal-error`. Watch the log for "did not emit canonical output within Ns" messages. -### CI status polling not wired (AWAITING_CI exits only via timeout) -The state machine transitions to AWAITING_CI after the implementer -pushes. From there, the workflow needs `ci_green` / `ci_red_*` -events to advance to REVIEWING. Those events are NOT emitted by -any production code — only `ci_polling_exhausted` fires (default 2h -timeout → STUCK). +### CI status polling — autonomous (no manual SQL needed) +The master loop runs ``run_ci_status_poll_tick`` every +``CONTROLLER_CI_STATUS_POLL_INTERVAL_S`` (default 60s). For each +workflow in AWAITING_CI, it calls Forgejo's +``/commits/{sha}/status`` combined-status endpoint and applies the +matching state transition: -**Trial impact**: a successful implementation will sit in -AWAITING_CI for 2h then STUCK. To manually advance during the trial: +| Forgejo state | Event fired | Next state | +|---|---|---| +| success / neutral / skipped / warning | ``ci_green`` | REVIEWING | +| failure / error / cancelled / timed_out / stale | ``ci_red_retry_same_tier`` | IMPLEMENTING | +| pending / queued / in_progress / action_required | (no-op) | stays AWAITING_CI | -```sql --- Mark CI as green to advance to REVIEWING: -UPDATE workflows -SET current_state = 'REVIEWING', - last_transition_at = NOW(), - entered_state_at = NOW() -WHERE workflow_id = AND current_state = 'AWAITING_CI'; +Operators no longer need the manual SQL workaround. If CI hangs +permanently (broken integration, runner outage), the +``ci_polling_exhausted`` timeout (default 2h) still fires as the +safety net → STUCK. -INSERT INTO controller_events - (workflow_id, ts, event_type, from_state, to_state, payload, - forgejo_write_pending, replay_attempts) -VALUES (, NOW(), 'manual', 'AWAITING_CI', 'REVIEWING', - '{"reason":"operator-marked-ci-green-during-trial"}', 0, 0); +To tune for trial speed: ``` - -To shorten the timeout for the trial (e.g. 5 min): +CONTROLLER_CI_STATUS_POLL_INTERVAL_S=10 # poll every 10s +CONTROLLER_AWAITING_CI_TIMEOUT_S=300 # 5-min timeout instead of 2h ``` -CONTROLLER_AWAITING_CI_TIMEOUT_S=300 -``` - -A real CI poller is queued as Phase 1n. ### Trial checklist 1. Set FORGEJO_URL + FORGEJO_TOKEN + CLEVERAGENTS_DB_URL on master @@ -339,8 +331,10 @@ A real CI poller is queued as Phase 1n. - Scheduler enqueues an implementer attempt - Worker spawns the implementer agent - Implementer commits + pushes → workflow → AWAITING_CI -7. **Manual step**: once you've confirmed the push, run the SQL above - to mark CI green + advance to REVIEWING. +7. CI runs in Forgejo Actions; the master's CI-status poller fires + every 60s (or whatever you set ``CONTROLLER_CI_STATUS_POLL_INTERVAL_S`` + to) — when CI goes green the workflow advances to REVIEWING + automatically. 8. The reviewer attempt runs; if `verdict='approve'`, the workflow transitions to MERGING and the controller calls Forgejo's merge endpoint. diff --git a/tools/controller/master/__init__.py b/tools/controller/master/__init__.py index db202adff..382b50282 100644 --- a/tools/controller/master/__init__.py +++ b/tools/controller/master/__init__.py @@ -76,6 +76,11 @@ from .ci_poll import ( DEFAULT_AWAITING_CI_TIMEOUT_S, run_ci_poll_exhaustion_tick, ) +from .ci_status_poll import ( + CIStatusPollReport, + GetCIStatusCallback, + run_ci_status_poll_tick, +) from .promote import ( PromoteDiscoveredReport, run_promote_discovered_tick, @@ -193,6 +198,10 @@ __all__ = [ "CIPollExhaustionReport", "DEFAULT_AWAITING_CI_TIMEOUT_S", "run_ci_poll_exhaustion_tick", + # CI status poll (Phase 1k++++) + "CIStatusPollReport", + "GetCIStatusCallback", + "run_ci_status_poll_tick", # Promote DISCOVERED → ANALYZING (Phase 1k+++) "PromoteDiscoveredReport", "run_promote_discovered_tick", diff --git a/tools/controller/master/__main__.py b/tools/controller/master/__main__.py index eb54171a8..b5dd5ed17 100644 --- a/tools/controller/master/__main__.py +++ b/tools/controller/master/__main__.py @@ -217,6 +217,9 @@ def main(argv: list[str] | None = None) -> int: callbacks.list_prs, callbacks.list_issues, {"require_opt_in_label": require_opt_in_label}, ), + ci_status_poll_args=( + args.owner, args.repo, callbacks.get_ci_status, + ), ) return 0 diff --git a/tools/controller/master/ci_status_poll.py b/tools/controller/master/ci_status_poll.py new file mode 100644 index 000000000..afd762f27 --- /dev/null +++ b/tools/controller/master/ci_status_poll.py @@ -0,0 +1,236 @@ +"""AWAITING_CI status poll — autonomous CI exit (Phase 1k++++ trial). + +Closes the last operator-intervention gap for the Phase 2 trial. +Previously, workflows that transitioned to AWAITING_CI sat there +forever (only ``ci_polling_exhausted`` fired, leading to STUCK after +the 2h timeout). The RUNBOOK had a manual SQL workaround. + +This module ships an autonomous tick: + +1. SELECT workflows in AWAITING_CI + their latest implementer + attempt's ``head_sha_after`` (the SHA the implementer pushed). +2. For each: call ``get_ci_status(owner, repo, head_sha)`` — wraps + Forgejo's ``/commits/{sha}/status`` combined-status endpoint. +3. Decide the next event based on the combined ``state`` field: + - ``success`` → ``ci_green`` → REVIEWING + - ``failure`` / ``error`` → ``ci_red_retry_same_tier`` → + IMPLEMENTING (the scheduler will then enqueue a fresh + implementer attempt at the same tier; pickup_guard + + ci_poll_exhaustion catch infinite loops) + - ``pending`` / ``queued`` / ``in_progress`` → no-op (wait for + the next tick) + - None / fetch failure → no-op (transient; reconciliation also + re-checks PR state independently) + +What this module DOES NOT do (yet): +- Per-gate CI summarization. ``ci_summarize.py`` exists but plugging + it in would require fetching every job log per gate — heavy. For + the trial we only need the overall state to advance the workflow. + The next implementer attempt's prefetch picks up the failing + gates via the existing ci_summary path. +- Escalation to ``ci_red_escalate``. Always uses retry_same_tier; + ESCALATING is reached via the regular attempts-per-tier exhaustion + path (pickup_guard / outcome mapper). +- Flake detection. Future ``ci_flake_retry`` integration ships when + the flake classifier lands. +""" +from __future__ import annotations + +import json +import logging +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from sqlalchemy import text +from sqlalchemy.engine import Engine + +from ..db.session import session_scope +from ..state_machine import IllegalTransitionError, apply_event + +logger = logging.getLogger(__name__) + + +# get_ci_status(owner, repo, head_sha) → Forgejo combined-status +# dict ({"state": "...", "statuses": [...]}) or None on transient +# fetch failure. +GetCIStatusCallback = Callable[[str, str, str], dict | None] + + +# Map Forgejo combined-status state → state-machine event. +# None = no-op (wait for next tick). +_STATE_TO_EVENT: dict[str | None, str | None] = { + "success": "ci_green", + "failure": "ci_red_retry_same_tier", + "error": "ci_red_retry_same_tier", + "pending": None, + "queued": None, + "in_progress": None, + "warning": "ci_green", # advisory; treat as passed + "neutral": "ci_green", + "skipped": "ci_green", + "cancelled": "ci_red_retry_same_tier", + "timed_out": "ci_red_retry_same_tier", + "action_required": None, # human intervention needed; wait + "stale": "ci_red_retry_same_tier", + None: None, +} + + +@dataclass +class CIStatusPollReport: + """Per-sweep summary.""" + + workflows_scanned: int = 0 + workflows_advanced_green: int = 0 + workflows_advanced_red: int = 0 + workflows_waiting: int = 0 + workflows_fetch_failed: int = 0 + transitions: list[tuple[int, str, str]] = field(default_factory=list) + # Each entry: (workflow_id, ci_state, event_fired) + + +def run_ci_status_poll_tick( + engine: Engine, *, + owner: str, repo: str, + get_ci_status: GetCIStatusCallback, +) -> CIStatusPollReport: + """One sweep: poll Forgejo CI status for every AWAITING_CI + workflow in this (owner, repo) + apply state-machine transitions. + """ + report = CIStatusPollReport() + now = datetime.now(timezone.utc) + + with session_scope(engine) as session: + # Find AWAITING_CI workflows + their latest implementer + # attempt's head_sha_after. The push SHA is what we need to + # query Forgejo's CI status against. + rows = session.execute( + text( + "SELECT w.workflow_id, w.entity_number, " + " (SELECT a.head_sha_after FROM workflow_attempts a " + " WHERE a.workflow_id = w.workflow_id " + " AND a.role = 'implementer' " + " AND a.status = 'complete' " + " AND a.head_sha_after IS NOT NULL " + " ORDER BY a.attempt_number DESC LIMIT 1) AS head_sha " + " FROM workflows w " + " WHERE w.current_state = 'AWAITING_CI' " + " AND w.owner = :owner AND w.repo = :repo " + " AND w.kind = 'pr'" + ), + {"owner": owner, "repo": repo}, + ).all() + report.workflows_scanned = len(rows) + + for row in rows: + wf_id = row.workflow_id + head_sha = row.head_sha + if not head_sha: + logger.warning( + "ci_status_poll: workflow %s in AWAITING_CI without " + "a head_sha (no completed implementer attempt found); " + "skipping", wf_id, + ) + report.workflows_waiting += 1 + continue + + try: + ci = get_ci_status(owner, repo, head_sha) + except Exception as exc: # noqa: BLE001 — transient + logger.warning( + "ci_status_poll: fetch failed for workflow %s " + "(head=%s): %s", wf_id, head_sha[:12], exc, + ) + report.workflows_fetch_failed += 1 + continue + + if ci is None: + report.workflows_fetch_failed += 1 + continue + + ci_state = ci.get("state") + event = _STATE_TO_EVENT.get(ci_state) + if event is None: + # Pending / queued / in_progress / unknown → wait. + report.workflows_waiting += 1 + continue + + try: + new_state = apply_event("AWAITING_CI", event) + except (IllegalTransitionError, ValueError) as exc: + logger.warning( + "ci_status_poll: apply_event(AWAITING_CI, %s) " + "raised %s for workflow %s; skipping", + event, exc, wf_id, + ) + continue + + # Apply via direct UPDATE filtered on current_state so a + # concurrent reconciliation can't double-transition (TOCTOU + # defense, same pattern as promote.py). + result = session.execute( + text( + "UPDATE workflows SET " + " current_state = :to_state, " + " last_transition_at = :now, " + " entered_state_at = :now " + "WHERE workflow_id = :wf_id " + " AND current_state = 'AWAITING_CI'" + ), + { + "to_state": new_state, "now": now, "wf_id": wf_id, + }, + ) + if (result.rowcount or 0) == 0: + logger.info( + "ci_status_poll: workflow %s no longer in AWAITING_CI " + "(race lost); skipping event row", + wf_id, + ) + continue + session.execute( + text( + "INSERT INTO controller_events " + "(workflow_id, ts, event_type, from_state, to_state, " + " payload, forgejo_write_pending, replay_attempts) " + "VALUES (:wf_id, :ts, :event_type, 'AWAITING_CI', " + " :to_state, :payload, 0, 0)" + ), + { + "wf_id": wf_id, "ts": now, + "event_type": ( + "ci-green" if event == "ci_green" else "ci-red" + ), + "to_state": new_state, + "payload": json.dumps({ + "reason": event, + "ci_state": ci_state, + "head_sha": head_sha, + "source": "ci_status_poll", + }), + }, + ) + + if event == "ci_green": + report.workflows_advanced_green += 1 + else: + report.workflows_advanced_red += 1 + report.transitions.append((wf_id, ci_state, event)) + + if report.workflows_advanced_green or report.workflows_advanced_red: + logger.info( + "ci_status_poll: %d scanned (green=%d, red=%d, waiting=%d, " + "fetch_failed=%d)", + report.workflows_scanned, report.workflows_advanced_green, + report.workflows_advanced_red, report.workflows_waiting, + report.workflows_fetch_failed, + ) + return report + + +__all__ = [ + "CIStatusPollReport", + "GetCIStatusCallback", + "run_ci_status_poll_tick", +] diff --git a/tools/controller/master/forgejo_http.py b/tools/controller/master/forgejo_http.py index 0eb85465f..3b2447961 100644 --- a/tools/controller/master/forgejo_http.py +++ b/tools/controller/master/forgejo_http.py @@ -48,6 +48,12 @@ class _ClaimRuntime(Protocol): def delete(self, path: str, cfg: Any) -> dict[str, Any]: ... +# get_ci_status(owner, repo, head_sha) → Forgejo combined-status dict +# (``{"state": "...", "statuses": [...]}``) or None on fetch failure. +from collections.abc import Callable as _Callable +GetCIStatusCallback = _Callable[[str, str, str], dict | None] + + @dataclass class ForgejoCallbacks: """Bundle of every callback the controller needs. Built by @@ -65,6 +71,8 @@ class ForgejoCallbacks: # Reconciliation callbacks (Phase 1g): get_pr_state: rec.GetPRStateCallback get_issue_state: rec.GetIssueStateCallback + # CI status callback (Phase 1k++++ trial — wired to ci_status_poll): + get_ci_status: "GetCIStatusCallback" # Prefetch callbacks (Phase 1h): get_pr_details: pf.GetPRDetailsCallback get_pr_diff: pf.GetPRDiffCallback @@ -105,6 +113,7 @@ def build_callbacks( get_pr_diff=_make_get_pr_diff(cfg, runtime), list_pr_reviews=_make_list_pr_reviews(cfg, runtime), list_pr_comments=_make_list_pr_comments(cfg, runtime), + get_ci_status=_make_get_ci_status(cfg, runtime), ) @@ -388,4 +397,32 @@ def _make_list_pr_comments(cfg, runtime): return list_pr_comments -__all__ = ["ForgejoCallbacks", "build_callbacks"] +# ─── CI status (Phase 1k++++ trial — wired to ci_status_poll) ─────── + + +def _make_get_ci_status(cfg, runtime): + def get_ci_status(owner: str, repo: str, head_sha: str) -> dict | None: + if not head_sha: + return None + path = f"/repos/{owner}/{repo}/commits/{head_sha}/status" + try: + resp = runtime.get(path, cfg) + except Exception as exc: # noqa: BLE001 — transient + logger.warning( + "get_ci_status: transport error for %s/%s @%s: %s", + owner, repo, head_sha[:12], exc, + ) + return None + status = int(resp.get("status") or 0) + if status != 200: + logger.warning( + "get_ci_status: HTTP %s for %s/%s @%s", + status, owner, repo, head_sha[:12], + ) + return None + body = resp.get("body") + return body if isinstance(body, dict) else None + return get_ci_status + + +__all__ = ["ForgejoCallbacks", "GetCIStatusCallback", "build_callbacks"] diff --git a/tools/controller/master/loop.py b/tools/controller/master/loop.py index 0e2e6e638..9f4b50786 100644 --- a/tools/controller/master/loop.py +++ b/tools/controller/master/loop.py @@ -39,6 +39,11 @@ from ..pickup_guard import ( ) from ..reaper import ReaperReport, reap_stale_attempts from .ci_poll import CIPollExhaustionReport, run_ci_poll_exhaustion_tick +from .ci_status_poll import ( + CIStatusPollReport, + GetCIStatusCallback, + run_ci_status_poll_tick, +) from .discovery import DiscoveryReport, run_discovery from .merging import MergeCallback, MergingHandlerReport, run_merging_tick from .promote import PromoteDiscoveredReport, run_promote_discovered_tick @@ -77,6 +82,12 @@ class MasterConfig: ci_poll_exhaustion_interval_s: float = float( os.environ.get("CONTROLLER_CI_POLL_EXHAUSTION_INTERVAL_S", "300") ) + # CI status polling — how often the master checks Forgejo for CI + # results on AWAITING_CI workflows. Default 60s; faster than + # reconciliation since CI usually finishes in minutes. + ci_status_poll_interval_s: float = float( + os.environ.get("CONTROLLER_CI_STATUS_POLL_INTERVAL_S", "60") + ) # Periodic discovery — how often the master polls Forgejo for # new PRs/issues. Without this, only PRs that existed at master # startup get discovered (backfill is one-shot). Default 30s. @@ -99,6 +110,7 @@ class MasterTickReport: scheduler: SchedulerReport | None = None merging: MergingHandlerReport | None = None discovery: DiscoveryReport | None = None + ci_status_poll: CIStatusPollReport | None = None def run_master_iteration( @@ -157,6 +169,12 @@ def master_main_loop( # this, only PRs that existed at master startup (via backfill) # are ever managed — PRs created after master startup wait until # the master restarts. + ci_status_poll_args: tuple | None = None, + # If set: (owner, repo, get_ci_status). The CI status poller + # fires every ci_status_poll_interval_s. Without this, workflows + # in AWAITING_CI exit only via the polling-exhaustion timeout + # (operator-intervention path); with it, ci_green / ci_red + # transitions fire autonomously. ) -> None: """Run the master loop until ``stop_event`` is set. @@ -172,6 +190,7 @@ def master_main_loop( last_reconcile_at_iteration = 0 last_ci_poll_exhaustion_at_iteration = 0 last_discovery_at_iteration = 0 + last_ci_status_poll_at_iteration = 0 iteration = 0 logger.info( @@ -332,6 +351,31 @@ def master_main_loop( ) last_discovery_at_iteration = iteration + # CI status poll: scan AWAITING_CI workflows + apply + # ci_green / ci_red transitions based on Forgejo's + # combined-status. Without this the workflow only exits + # AWAITING_CI via the ci_poll_exhaustion timeout. Runs + # on ci_status_poll_interval_s cadence. + ci_status_report: CIStatusPollReport | None = None + should_ci_status_poll = ( + ci_status_poll_args is not None + and (iteration - last_ci_status_poll_at_iteration) + * cfg.tick_interval_s + >= cfg.ci_status_poll_interval_s + ) + if should_ci_status_poll and ci_status_poll_args is not None: + try: + csp_owner, csp_repo, csp_get_ci = ci_status_poll_args + ci_status_report = run_ci_status_poll_tick( + engine, owner=csp_owner, repo=csp_repo, + get_ci_status=csp_get_ci, + ) + except Exception: + logger.exception( + "ci_status_poll tick raised; continuing" + ) + last_ci_status_poll_at_iteration = iteration + if on_iteration is not None: try: on_iteration(MasterTickReport( @@ -344,6 +388,7 @@ def master_main_loop( scheduler=scheduler_report, merging=merging_report, discovery=discovery_report, + ci_status_poll=ci_status_report, )) except Exception: logger.exception("on_iteration callback raised")