cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

feat(controller): autonomous CI status polling — closes the last trial gap

Browse Source 
The Phase 2 trial previously required operator-intervention SQL to
advance workflows from AWAITING_CI → REVIEWING (no automated CI
status polling). This commit wires the missing tick so the trial
runs end-to-end without manual help.

Components:
- ``master/forgejo_http.py``: new ``get_ci_status`` callback wraps
  Forgejo's ``/commits/{sha}/status`` combined-status endpoint;
  added to ``ForgejoCallbacks``.
- ``master/ci_status_poll.py`` (NEW): ``run_ci_status_poll_tick``
  scans AWAITING_CI workflows, fetches CI status keyed on the
  latest implementer attempt's ``head_sha_after``, and applies
  state transitions via ``apply_event``. TOCTOU-defended UPDATE
  (``WHERE current_state='AWAITING_CI'``) + per-row exception
  isolation.
- ``master/loop.py``: new ``ci_status_poll_args=(owner, repo,
  get_ci_status)`` kwarg + ``ci_status_poll_interval_s`` config
  (default 60s) + ``MasterTickReport.ci_status_poll`` field.
- ``master/__main__.py``: threads ``callbacks.get_ci_status`` into
  the loop.

State mapping (Forgejo combined-status state → event):
- success / neutral / skipped / warning → ci_green → REVIEWING
- failure / error / cancelled / timed_out / stale →
  ci_red_retry_same_tier → IMPLEMENTING
- pending / queued / in_progress / action_required → no-op (wait)
- None / unknown / fetch failure → no-op (transient)

The ``ci_polling_exhausted`` timeout (default 2h) remains as the
safety net for CI that genuinely never reports.

Tests (+14 in test_master_ci_status_poll.py):
- Happy paths (success→green, failure→red, pending→wait)
- Error paths (callback raises; workflow without head_sha)
- Event row shape (event_type='ci-green'/'ci-red', reason payload)
- Extended state mapping (cancelled, neutral, in_progress)
- Other-repo isolation
- LoopIntegration end-to-end via master_main_loop with safety timer

RUNBOOK updated: removed the manual SQL workaround; added the
autonomous CI poll's tunables.

Total: 726 controller tests pass (+14 net), 0 regressions.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Drew Morris
2026-05-18 17:37:04 -04:00
parent f93f0d1c53
commit 3ca794be75
8 changed files with 628 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/auto_agents/controller/test_entry_points.py
+1
View File
@@ -103,6 +103,7 @@ class TestMasterOptInLabelFlag:
get_pr_diff: callable = lambda o, r, n: None
list_pr_reviews: callable = lambda o, r, n: []
list_pr_comments: callable = lambda o, r, n: []
get_ci_status: callable = lambda o, r, sha: None
return _StubCB()
def _stub_cfg(self):
tests/auto_agents/controller/test_master_ci_status_poll.py
+274
View File
@@ -0,0 +1,274 @@
"""Tests for the AWAITING_CI status poll (Phase 1k++++ trial)."""
from __future__ import annotations
import json
import pytest
from sqlalchemy import text
from tools.controller.db import (
Workflow,
WorkflowAttempt,
build_engine,
create_all,
session_scope,
)
from tools.controller.master.ci_status_poll import (
CIStatusPollReport,
run_ci_status_poll_tick,
)
@pytest.fixture
def engine():
eng = build_engine("sqlite:///:memory:")
create_all(eng)
yield eng
def _seed_awaiting_ci_with_attempt(
engine, *, head_sha_after: str | None = "abc1234567",
entity_number: int = 30,
) -> int:
"""Seed an AWAITING_CI workflow with a completed implementer
attempt whose head_sha_after is set (so ci_status_poll has a SHA
to query)."""
with session_scope(engine) as s:
w = Workflow(
kind="pr", owner="o", repo="r", entity_number=entity_number,
current_state="AWAITING_CI",
)
s.add(w); s.flush()
wf_id = w.workflow_id
a = WorkflowAttempt(
workflow_id=wf_id, attempt_number=1, role="implementer",
tier=0, status="complete",
input_payload={}, input_version="V1",
output_payload={"output_version": "V1", "outcome": "resolved",
"files_touched": ["x.py"], "commit_shas": [],
"confidence": "high", "blockers": [],
"used_tier": 0, "wallclock_seconds": 1.0},
output_version="V1", outcome="resolved",
head_sha_before="def1234567",
head_sha_after=head_sha_after,
)
s.add(a)
return wf_id
class TestHappyPaths:
def test_empty_db_no_op(self, engine):
report = run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: pytest.fail("no fetch"),
)
assert report.workflows_scanned == 0
assert report.workflows_advanced_green == 0
assert report.workflows_advanced_red == 0
def test_ci_success_transitions_to_reviewing(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
report = run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "success"},
)
assert report.workflows_advanced_green == 1
with session_scope(engine) as s:
w = s.query(Workflow).filter_by(workflow_id=wf_id).one()
assert w.current_state == "REVIEWING"
def test_ci_failure_transitions_back_to_implementing(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
report = run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "failure"},
)
assert report.workflows_advanced_red == 1
with session_scope(engine) as s:
w = s.query(Workflow).filter_by(workflow_id=wf_id).one()
assert w.current_state == "IMPLEMENTING"
def test_ci_pending_waits_no_transition(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
report = run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "pending"},
)
assert report.workflows_waiting == 1
with session_scope(engine) as s:
w = s.query(Workflow).filter_by(workflow_id=wf_id).one()
assert w.current_state == "AWAITING_CI"
def test_fetch_failure_no_op(self, engine):
_seed_awaiting_ci_with_attempt(engine)
report = run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: None,
)
assert report.workflows_fetch_failed == 1
assert report.workflows_advanced_green == 0
class TestErrorPaths:
def test_callback_raises_handled(self, engine):
_seed_awaiting_ci_with_attempt(engine)
def boom(o, r, sha):
raise ConnectionError("forgejo down")
report = run_ci_status_poll_tick(
engine, owner="o", repo="r", get_ci_status=boom,
)
assert report.workflows_fetch_failed == 1
def test_workflow_without_head_sha_skipped(self, engine):
"""An AWAITING_CI workflow with no completed implementer
attempt that pushed (head_sha_after is NULL) → skip."""
_seed_awaiting_ci_with_attempt(engine, head_sha_after=None)
report = run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: pytest.fail("no fetch"),
)
assert report.workflows_waiting == 1
class TestEventRows:
def test_green_emits_ci_green_event(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "success"},
)
with session_scope(engine) as s:
events = s.execute(
text(
"SELECT event_type, from_state, to_state, payload "
"FROM controller_events "
"WHERE event_type = 'ci-green'"
),
).all()
assert len(events) == 1
ev = events[0]
assert ev.from_state == "AWAITING_CI"
assert ev.to_state == "REVIEWING"
payload = json.loads(ev.payload)
assert payload["reason"] == "ci_green"
assert payload["ci_state"] == "success"
def test_red_emits_ci_red_event(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "failure"},
)
with session_scope(engine) as s:
events = s.execute(
text(
"SELECT event_type, payload FROM controller_events "
"WHERE event_type = 'ci-red'"
),
).all()
assert len(events) == 1
payload = json.loads(events[0].payload)
assert payload["reason"] == "ci_red_retry_same_tier"
class TestExtendedStateMapping:
def test_cancelled_treated_as_red(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "cancelled"},
)
with session_scope(engine) as s:
w = s.query(Workflow).filter_by(workflow_id=wf_id).one()
assert w.current_state == "IMPLEMENTING"
def test_neutral_treated_as_green(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "neutral"},
)
with session_scope(engine) as s:
w = s.query(Workflow).filter_by(workflow_id=wf_id).one()
assert w.current_state == "REVIEWING"
def test_in_progress_waits(self, engine):
wf_id = _seed_awaiting_ci_with_attempt(engine)
report = run_ci_status_poll_tick(
engine, owner="o", repo="r",
get_ci_status=lambda o, r, sha: {"state": "in_progress"},
)
assert report.workflows_waiting == 1
class TestOtherRepoExcluded:
def test_workflows_for_other_repo_not_targeted(self, engine):
_seed_awaiting_ci_with_attempt(engine)
# Poll a different (owner, repo).
report = run_ci_status_poll_tick(
engine, owner="other", repo="other",
get_ci_status=lambda o, r, sha: pytest.fail("no fetch"),
)
assert report.workflows_scanned == 0
class TestLoopIntegration:
def test_loop_runs_ci_status_poll_on_cadence(self, engine):
"""End-to-end through master_main_loop: AWAITING_CI →
ci_status_poll fetches success → REVIEWING. Pins the
autonomous trial path (no operator SQL needed)."""
import threading
from tools.controller.master.loop import (
MasterConfig,
MasterTickReport,
master_main_loop,
)
wf_id = _seed_awaiting_ci_with_attempt(engine)
stop = threading.Event()
seen: list[MasterTickReport] = []
def on_iter(r: MasterTickReport):
seen.append(r)
if (
r.ci_status_poll
and r.ci_status_poll.workflows_advanced_green
):
stop.set()
# Safety net so the test fails fast on regression.
safety = threading.Timer(5.0, stop.set)
safety.daemon = True
safety.start()
try:
cfg = MasterConfig(
tick_interval_s=0.05,
reaper_interval_s=10,
reconciliation_interval_s=10,
ci_poll_exhaustion_interval_s=10,
ci_status_poll_interval_s=0.05,
)
master_main_loop(
engine, config=cfg, stop_event=stop,
on_iteration=on_iter,
ci_status_poll_args=(
"o", "r", lambda o, r, sha: {"state": "success"},
),
)
finally:
safety.cancel()
advanced = [
r for r in seen
if r.ci_status_poll
and r.ci_status_poll.workflows_advanced_green
]
assert advanced, (
"ci_status_poll never saw a green transition; safety timer "
"likely fired first (test logic regressed)"
)
with session_scope(engine) as s:
w = s.query(Workflow).filter_by(workflow_id=wf_id).one()
assert w.current_state == "REVIEWING"
tools/controller/deploy/RUNBOOK.md
+22 -28
View File
@@ -291,37 +291,29 @@ the fallback instruction work fine. If an agent ignores the fallback
`worker-internal-error`. Watch the log for "did not emit canonical
output within Ns" messages.
### CI status polling not wired (AWAITING_CI exits only via timeout)
The state machine transitions to AWAITING_CI after the implementer
pushes. From there, the workflow needs `ci_green` / `ci_red_*`
events to advance to REVIEWING. Those events are NOT emitted by
any production code — only `ci_polling_exhausted` fires (default 2h
timeout → STUCK).
### CI status polling — autonomous (no manual SQL needed)
The master loop runs ``run_ci_status_poll_tick`` every
``CONTROLLER_CI_STATUS_POLL_INTERVAL_S`` (default 60s). For each
workflow in AWAITING_CI, it calls Forgejo's
``/commits/{sha}/status`` combined-status endpoint and applies the
matching state transition:
**Trial impact**: a successful implementation will sit in
AWAITING_CI for 2h then STUCK. To manually advance during the trial:
| Forgejo state | Event fired | Next state |
|---|---|---|
| success / neutral / skipped / warning | ``ci_green`` | REVIEWING |
| failure / error / cancelled / timed_out / stale | ``ci_red_retry_same_tier`` | IMPLEMENTING |
| pending / queued / in_progress / action_required | (no-op) | stays AWAITING_CI |
```sql
-- Mark CI as green to advance to REVIEWING:
UPDATE workflows
SET current_state = 'REVIEWING',
last_transition_at = NOW(),
entered_state_at = NOW()
WHERE workflow_id = <id> AND current_state = 'AWAITING_CI';
Operators no longer need the manual SQL workaround. If CI hangs
permanently (broken integration, runner outage), the
``ci_polling_exhausted`` timeout (default 2h) still fires as the
safety net → STUCK.
INSERT INTO controller_events
(workflow_id, ts, event_type, from_state, to_state, payload,
forgejo_write_pending, replay_attempts)
VALUES (<id>, NOW(), 'manual', 'AWAITING_CI', 'REVIEWING',
'{"reason":"operator-marked-ci-green-during-trial"}', 0, 0);
To tune for trial speed:
```
To shorten the timeout for the trial (e.g. 5 min):
CONTROLLER_CI_STATUS_POLL_INTERVAL_S=10 # poll every 10s
CONTROLLER_AWAITING_CI_TIMEOUT_S=300 # 5-min timeout instead of 2h
```
CONTROLLER_AWAITING_CI_TIMEOUT_S=300
```
A real CI poller is queued as Phase 1n.
### Trial checklist
1. Set FORGEJO_URL + FORGEJO_TOKEN + CLEVERAGENTS_DB_URL on master
@@ -339,8 +331,10 @@ A real CI poller is queued as Phase 1n.
- Scheduler enqueues an implementer attempt
- Worker spawns the implementer agent
- Implementer commits + pushes → workflow → AWAITING_CI
7. **Manual step**: once you've confirmed the push, run the SQL above
to mark CI green + advance to REVIEWING.
7. CI runs in Forgejo Actions; the master's CI-status poller fires
every 60s (or whatever you set ``CONTROLLER_CI_STATUS_POLL_INTERVAL_S``
to) — when CI goes green the workflow advances to REVIEWING
automatically.
8. The reviewer attempt runs; if `verdict='approve'`, the workflow
transitions to MERGING and the controller calls Forgejo's merge
endpoint.
tools/controller/master/__init__.py
+9
View File
@@ -76,6 +76,11 @@ from .ci_poll import (
DEFAULT_AWAITING_CI_TIMEOUT_S,
run_ci_poll_exhaustion_tick,
)
from .ci_status_poll import (
CIStatusPollReport,
GetCIStatusCallback,
run_ci_status_poll_tick,
)
from .promote import (
PromoteDiscoveredReport,
run_promote_discovered_tick,
@@ -193,6 +198,10 @@ __all__ = [
"CIPollExhaustionReport",
"DEFAULT_AWAITING_CI_TIMEOUT_S",
"run_ci_poll_exhaustion_tick",
# CI status poll (Phase 1k++++)
"CIStatusPollReport",
"GetCIStatusCallback",
"run_ci_status_poll_tick",
# Promote DISCOVERED → ANALYZING (Phase 1k+++)
"PromoteDiscoveredReport",
"run_promote_discovered_tick",
tools/controller/master/__main__.py
+3
View File
@@ -217,6 +217,9 @@ def main(argv: list[str] | None = None) -> int:
callbacks.list_prs, callbacks.list_issues,
{"require_opt_in_label": require_opt_in_label},
),
ci_status_poll_args=(
args.owner, args.repo, callbacks.get_ci_status,
),
)
return 0
tools/controller/master/ci_status_poll.py
+236
View File
@@ -0,0 +1,236 @@
"""AWAITING_CI status poll — autonomous CI exit (Phase 1k++++ trial).
Closes the last operator-intervention gap for the Phase 2 trial.
Previously, workflows that transitioned to AWAITING_CI sat there
forever (only ``ci_polling_exhausted`` fired, leading to STUCK after
the 2h timeout). The RUNBOOK had a manual SQL workaround.
This module ships an autonomous tick:
1. SELECT workflows in AWAITING_CI + their latest implementer
attempt's ``head_sha_after`` (the SHA the implementer pushed).
2. For each: call ``get_ci_status(owner, repo, head_sha)`` — wraps
Forgejo's ``/commits/{sha}/status`` combined-status endpoint.
3. Decide the next event based on the combined ``state`` field:
- ``success`` → ``ci_green`` → REVIEWING
- ``failure`` / ``error`` → ``ci_red_retry_same_tier`` →
IMPLEMENTING (the scheduler will then enqueue a fresh
implementer attempt at the same tier; pickup_guard +
ci_poll_exhaustion catch infinite loops)
- ``pending`` / ``queued`` / ``in_progress`` → no-op (wait for
the next tick)
- None / fetch failure → no-op (transient; reconciliation also
re-checks PR state independently)
What this module DOES NOT do (yet):
- Per-gate CI summarization. ``ci_summarize.py`` exists but plugging
it in would require fetching every job log per gate — heavy. For
the trial we only need the overall state to advance the workflow.
The next implementer attempt's prefetch picks up the failing
gates via the existing ci_summary path.
- Escalation to ``ci_red_escalate``. Always uses retry_same_tier;
ESCALATING is reached via the regular attempts-per-tier exhaustion
path (pickup_guard / outcome mapper).
- Flake detection. Future ``ci_flake_retry`` integration ships when
the flake classifier lands.
"""
from __future__ import annotations
import json
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timezone
from sqlalchemy import text
from sqlalchemy.engine import Engine
from ..db.session import session_scope
from ..state_machine import IllegalTransitionError, apply_event
logger = logging.getLogger(__name__)
# get_ci_status(owner, repo, head_sha) → Forgejo combined-status
# dict ({"state": "...", "statuses": [...]}) or None on transient
# fetch failure.
GetCIStatusCallback = Callable[[str, str, str], dict | None]
# Map Forgejo combined-status state → state-machine event.
# None = no-op (wait for next tick).
_STATE_TO_EVENT: dict[str | None, str | None] = {
"success": "ci_green",
"failure": "ci_red_retry_same_tier",
"error": "ci_red_retry_same_tier",
"pending": None,
"queued": None,
"in_progress": None,
"warning": "ci_green", # advisory; treat as passed
"neutral": "ci_green",
"skipped": "ci_green",
"cancelled": "ci_red_retry_same_tier",
"timed_out": "ci_red_retry_same_tier",
"action_required": None, # human intervention needed; wait
"stale": "ci_red_retry_same_tier",
None: None,
}
@dataclass
class CIStatusPollReport:
"""Per-sweep summary."""
workflows_scanned: int = 0
workflows_advanced_green: int = 0
workflows_advanced_red: int = 0
workflows_waiting: int = 0
workflows_fetch_failed: int = 0
transitions: list[tuple[int, str, str]] = field(default_factory=list)
# Each entry: (workflow_id, ci_state, event_fired)
def run_ci_status_poll_tick(
engine: Engine, *,
owner: str, repo: str,
get_ci_status: GetCIStatusCallback,
) -> CIStatusPollReport:
"""One sweep: poll Forgejo CI status for every AWAITING_CI
workflow in this (owner, repo) + apply state-machine transitions.
"""
report = CIStatusPollReport()
now = datetime.now(timezone.utc)
with session_scope(engine) as session:
# Find AWAITING_CI workflows + their latest implementer
# attempt's head_sha_after. The push SHA is what we need to
# query Forgejo's CI status against.
rows = session.execute(
text(
"SELECT w.workflow_id, w.entity_number, "
" (SELECT a.head_sha_after FROM workflow_attempts a "
" WHERE a.workflow_id = w.workflow_id "
" AND a.role = 'implementer' "
" AND a.status = 'complete' "
" AND a.head_sha_after IS NOT NULL "
" ORDER BY a.attempt_number DESC LIMIT 1) AS head_sha "
" FROM workflows w "
" WHERE w.current_state = 'AWAITING_CI' "
" AND w.owner = :owner AND w.repo = :repo "
" AND w.kind = 'pr'"
),
{"owner": owner, "repo": repo},
).all()
report.workflows_scanned = len(rows)
for row in rows:
wf_id = row.workflow_id
head_sha = row.head_sha
if not head_sha:
logger.warning(
"ci_status_poll: workflow %s in AWAITING_CI without "
"a head_sha (no completed implementer attempt found); "
"skipping", wf_id,
)
report.workflows_waiting += 1
continue
try:
ci = get_ci_status(owner, repo, head_sha)
except Exception as exc: # noqa: BLE001 — transient
logger.warning(
"ci_status_poll: fetch failed for workflow %s "
"(head=%s): %s", wf_id, head_sha[:12], exc,
)
report.workflows_fetch_failed += 1
continue
if ci is None:
report.workflows_fetch_failed += 1
continue
ci_state = ci.get("state")
event = _STATE_TO_EVENT.get(ci_state)
if event is None:
# Pending / queued / in_progress / unknown → wait.
report.workflows_waiting += 1
continue
try:
new_state = apply_event("AWAITING_CI", event)
except (IllegalTransitionError, ValueError) as exc:
logger.warning(
"ci_status_poll: apply_event(AWAITING_CI, %s) "
"raised %s for workflow %s; skipping",
event, exc, wf_id,
)
continue
# Apply via direct UPDATE filtered on current_state so a
# concurrent reconciliation can't double-transition (TOCTOU
# defense, same pattern as promote.py).
result = session.execute(
text(
"UPDATE workflows SET "
" current_state = :to_state, "
" last_transition_at = :now, "
" entered_state_at = :now "
"WHERE workflow_id = :wf_id "
" AND current_state = 'AWAITING_CI'"
),
{
"to_state": new_state, "now": now, "wf_id": wf_id,
},
)
if (result.rowcount or 0) == 0:
logger.info(
"ci_status_poll: workflow %s no longer in AWAITING_CI "
"(race lost); skipping event row",
wf_id,
)
continue
session.execute(
text(
"INSERT INTO controller_events "
"(workflow_id, ts, event_type, from_state, to_state, "
" payload, forgejo_write_pending, replay_attempts) "
"VALUES (:wf_id, :ts, :event_type, 'AWAITING_CI', "
" :to_state, :payload, 0, 0)"
),
{
"wf_id": wf_id, "ts": now,
"event_type": (
"ci-green" if event == "ci_green" else "ci-red"
),
"to_state": new_state,
"payload": json.dumps({
"reason": event,
"ci_state": ci_state,
"head_sha": head_sha,
"source": "ci_status_poll",
}),
},
)
if event == "ci_green":
report.workflows_advanced_green += 1
else:
report.workflows_advanced_red += 1
report.transitions.append((wf_id, ci_state, event))
if report.workflows_advanced_green or report.workflows_advanced_red:
logger.info(
"ci_status_poll: %d scanned (green=%d, red=%d, waiting=%d, "
"fetch_failed=%d)",
report.workflows_scanned, report.workflows_advanced_green,
report.workflows_advanced_red, report.workflows_waiting,
report.workflows_fetch_failed,
)
return report
__all__ = [
"CIStatusPollReport",
"GetCIStatusCallback",
"run_ci_status_poll_tick",
]
tools/controller/master/forgejo_http.py
+38 -1
View File
@@ -48,6 +48,12 @@ class _ClaimRuntime(Protocol):
def delete(self, path: str, cfg: Any) -> dict[str, Any]: ...
# get_ci_status(owner, repo, head_sha) → Forgejo combined-status dict
# (``{"state": "...", "statuses": [...]}``) or None on fetch failure.
from collections.abc import Callable as _Callable
GetCIStatusCallback = _Callable[[str, str, str], dict | None]
@dataclass
class ForgejoCallbacks:
"""Bundle of every callback the controller needs. Built by
@@ -65,6 +71,8 @@ class ForgejoCallbacks:
# Reconciliation callbacks (Phase 1g):
get_pr_state: rec.GetPRStateCallback
get_issue_state: rec.GetIssueStateCallback
# CI status callback (Phase 1k++++ trial — wired to ci_status_poll):
get_ci_status: "GetCIStatusCallback"
# Prefetch callbacks (Phase 1h):
get_pr_details: pf.GetPRDetailsCallback
get_pr_diff: pf.GetPRDiffCallback
@@ -105,6 +113,7 @@ def build_callbacks(
get_pr_diff=_make_get_pr_diff(cfg, runtime),
list_pr_reviews=_make_list_pr_reviews(cfg, runtime),
list_pr_comments=_make_list_pr_comments(cfg, runtime),
get_ci_status=_make_get_ci_status(cfg, runtime),
)
@@ -388,4 +397,32 @@ def _make_list_pr_comments(cfg, runtime):
return list_pr_comments
__all__ = ["ForgejoCallbacks", "build_callbacks"]
# ─── CI status (Phase 1k++++ trial — wired to ci_status_poll) ───────
def _make_get_ci_status(cfg, runtime):
def get_ci_status(owner: str, repo: str, head_sha: str) -> dict | None:
if not head_sha:
return None
path = f"/repos/{owner}/{repo}/commits/{head_sha}/status"
try:
resp = runtime.get(path, cfg)
except Exception as exc: # noqa: BLE001 — transient
logger.warning(
"get_ci_status: transport error for %s/%s @%s: %s",
owner, repo, head_sha[:12], exc,
)
return None
status = int(resp.get("status") or 0)
if status != 200:
logger.warning(
"get_ci_status: HTTP %s for %s/%s @%s",
status, owner, repo, head_sha[:12],
)
return None
body = resp.get("body")
return body if isinstance(body, dict) else None
return get_ci_status
__all__ = ["ForgejoCallbacks", "GetCIStatusCallback", "build_callbacks"]
tools/controller/master/loop.py
+45
View File
@@ -39,6 +39,11 @@ from ..pickup_guard import (
)
from ..reaper import ReaperReport, reap_stale_attempts
from .ci_poll import CIPollExhaustionReport, run_ci_poll_exhaustion_tick
from .ci_status_poll import (
CIStatusPollReport,
GetCIStatusCallback,
run_ci_status_poll_tick,
)
from .discovery import DiscoveryReport, run_discovery
from .merging import MergeCallback, MergingHandlerReport, run_merging_tick
from .promote import PromoteDiscoveredReport, run_promote_discovered_tick
@@ -77,6 +82,12 @@ class MasterConfig:
ci_poll_exhaustion_interval_s: float = float(
os.environ.get("CONTROLLER_CI_POLL_EXHAUSTION_INTERVAL_S", "300")
)
# CI status polling — how often the master checks Forgejo for CI
# results on AWAITING_CI workflows. Default 60s; faster than
# reconciliation since CI usually finishes in minutes.
ci_status_poll_interval_s: float = float(
os.environ.get("CONTROLLER_CI_STATUS_POLL_INTERVAL_S", "60")
)
# Periodic discovery — how often the master polls Forgejo for
# new PRs/issues. Without this, only PRs that existed at master
# startup get discovered (backfill is one-shot). Default 30s.
@@ -99,6 +110,7 @@ class MasterTickReport:
scheduler: SchedulerReport | None = None
merging: MergingHandlerReport | None = None
discovery: DiscoveryReport | None = None
ci_status_poll: CIStatusPollReport | None = None
def run_master_iteration(
@@ -157,6 +169,12 @@ def master_main_loop(
# this, only PRs that existed at master startup (via backfill)
# are ever managed — PRs created after master startup wait until
# the master restarts.
ci_status_poll_args: tuple | None = None,
# If set: (owner, repo, get_ci_status). The CI status poller
# fires every ci_status_poll_interval_s. Without this, workflows
# in AWAITING_CI exit only via the polling-exhaustion timeout
# (operator-intervention path); with it, ci_green / ci_red
# transitions fire autonomously.
) -> None:
"""Run the master loop until ``stop_event`` is set.
@@ -172,6 +190,7 @@ def master_main_loop(
last_reconcile_at_iteration = 0
last_ci_poll_exhaustion_at_iteration = 0
last_discovery_at_iteration = 0
last_ci_status_poll_at_iteration = 0
iteration = 0
logger.info(
@@ -332,6 +351,31 @@ def master_main_loop(
)
last_discovery_at_iteration = iteration
# CI status poll: scan AWAITING_CI workflows + apply
# ci_green / ci_red transitions based on Forgejo's
# combined-status. Without this the workflow only exits
# AWAITING_CI via the ci_poll_exhaustion timeout. Runs
# on ci_status_poll_interval_s cadence.
ci_status_report: CIStatusPollReport | None = None
should_ci_status_poll = (
ci_status_poll_args is not None
and (iteration - last_ci_status_poll_at_iteration)
* cfg.tick_interval_s
>= cfg.ci_status_poll_interval_s
)
if should_ci_status_poll and ci_status_poll_args is not None:
try:
csp_owner, csp_repo, csp_get_ci = ci_status_poll_args
ci_status_report = run_ci_status_poll_tick(
engine, owner=csp_owner, repo=csp_repo,
get_ci_status=csp_get_ci,
)
except Exception:
logger.exception(
"ci_status_poll tick raised; continuing"
)
last_ci_status_poll_at_iteration = iteration
if on_iteration is not None:
try:
on_iteration(MasterTickReport(
@@ -344,6 +388,7 @@ def master_main_loop(
scheduler=scheduler_report,
merging=merging_report,
discovery=discovery_report,
ci_status_poll=ci_status_report,
))
except Exception:
logger.exception("on_iteration callback raised")