Files
cleveragents-core/tools/controller/master/__main__.py
T
drew a21466add2 feat(controller): Phase 4 metadata-hygiene + round-2 adversarial fixes
Five deterministic, idempotent Phase 4 checks:
  1. completed_not_closed — close linked issues on MERGED
  2. closing_keyword_fixup — add Closes #N to PR bodies
  3. label_sync_from_issue — copy Priority/Type/MoSCoW labels
  4. state_label_inference — sync State/* label to current_state
  5. milestone_assignment — copy milestone from linked issue

All default-off via CONTROLLER_METADATA_HYGIENE_ENABLED +
per-check granular env flags. Dry-run mode shares the grooming
CONTROLLER_GROOMING_DRY_RUN flag.

Round-1 fixes (applied before this commit):
- False-positive idempotency lock (executed=True on skip)
- Unbounded MERGED scan → LEFT JOIN candidate query
- Duplicate _classify_forgejo_status → import from forgejo_writes
- Bare-ref regex too broad ([#42](url) misread) → add [ lookbehind
- Wrong audit stage → 'metadata_hygiene'

Round-2 adversarial fixes (3 architect, 4 principal, 7 test engineer):
- completed_not_closed: executed=True only when ALL refs close;
  partial success writes executed=False so remaining issues retry
- milestone_assignment: was calling get_pr_details (hits /pulls/,
  returns 404 for plain issues) → now uses get_issue_state
  (/issues/{n}) so milestone fetch works for all issue types
- label_sync failure path: write executed=False audit row for
  observability; pre-fix left no audit trail for persistent failures
- _BARE_REF_RE: add ( to lookbehind to exclude (#42) link destinations
- state_label_inference: re-read current_state inside inner session to
  avoid stale-snapshot spurious label writes across session boundaries
- _last_synced_state: add decision_id DESC tiebreaker for same-second
  wall-clock rows
- dry-run completed_not_closed: separate early-return path to avoid
  inflating completed_not_closed_executed counter

71 tests (54 round-1 + 17 round-2):
- TestCompletedNotClosedPartialSuccess (3) — partial/zero/full success
- TestLabelSyncAdjustLabelsFailure (2) — failure audit + retry
- TestStateLabelAdjustLabelsFailure (2) — no executed=1 on failure
- TestStateLabelInferenceTerminalWorkflows (3) — MERGED/ABANDONED sync
- TestLastSyncedStateDryRunThenReal (2) — dry-run → real-run
- TestClosingKeywordFixupBareRefAlreadyCovered (2) — candidates subtraction
- TestErrorPathHandlingRound2 (3) — label_sync + state_label errors

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-25 17:41:49 -04:00

559 lines
22 KiB
Python

"""Master controller entry point — runnable via systemd.
Usage:
python -m tools.controller.master \
--owner=cleveragents \
--repo=cleveragents-core \
--opencode-url=http://localhost:4096
Reads ``CLEVERAGENTS_DB_URL`` (default in-memory SQLite — operators
MUST set this for production). Wires the production callbacks via
``forgejo_http.build_callbacks`` + ``_claim_runtime``.
Per plan v9 the master is a singleton per (owner, repo). For v1 the
singleton constraint is enforced by deployment (one systemd unit per
repo). Lease-based multi-master HA is deferred per the plan.
"""
from __future__ import annotations
import argparse
import logging
import os
import signal
import threading
from ..db import build_engine, create_all
from .backfill import run_startup_backfill
from .discovery import run_discovery
from .forgejo_cfg import ControllerForgejoConfig, from_environment
from .forgejo_http import build_callbacks
from .estimator_abandon_side_effects import EstimatorAbandonCallbacks
from .gate2_abandon_config import (
get_gate2_abandon_config as _get_gate2_abandon_cfg,
log_effective_config as _log_gate2_abandon_cfg,
)
from .gate3_abandon_config import (
get_gate3_abandon_config as _get_gate3_abandon_cfg,
log_effective_config as _log_gate3_abandon_cfg,
)
from .grooming_side_effects import GroomingCallbacks
from .loop import MasterConfig, master_main_loop
from .metadata_hygiene import MetadataHygieneCallbacks
from .metadata_hygiene_config import (
get_metadata_hygiene_config as _get_metadata_hygiene_cfg,
log_effective_config as _log_metadata_hygiene_cfg,
)
from .reviewer_abandon_side_effects import ReviewerAbandonCallbacks
from .prefetch import PrefetchDataCallbacks, make_prefetch_callback
def _setup_logging(level: str) -> None:
logging.basicConfig(
level=getattr(logging, level.upper(), logging.INFO),
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
def _run_ci_local_enabled() -> bool:
"""True when ``RUN_CI_LOCAL`` is set to a truthy value — the master
then serves CI verdicts from local ``run-ci-full-local.sh`` runs
instead of Forgejo's combined-status API."""
return os.environ.get("RUN_CI_LOCAL", "").strip().lower() in {
"1",
"true",
"yes",
"on",
}
def build_cfg_stub(owner: str, repo: str) -> ControllerForgejoConfig:
"""Build the controller-owned ForgejoConfig from env + the
deployment-specified (owner, repo).
Phase 1k+: replaced sys.path injection + ``tools/_mcp_common.ForgejoCfg``
mutation with a tight ``ControllerForgejoConfig`` owned by
``controller/master/forgejo_cfg.py``. The deps now flow inward
(controller depends on nothing in ``tools/*`` other than the
HTTP helper imports inside ``forgejo_http.py``).
"""
return from_environment(owner, repo)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
parser.add_argument(
"--owner", required=True, help="Forgejo owner (org/user) — e.g. cleveragents"
)
parser.add_argument(
"--repo", required=True, help="Forgejo repo name — e.g. cleveragents-core"
)
parser.add_argument(
"--opencode-url",
default=os.environ.get(
"CONTROLLER_OPENCODE_URL",
"http://localhost:4096",
),
help="OpenCode server URL",
)
parser.add_argument(
"--log-level",
default=os.environ.get(
"CONTROLLER_LOG_LEVEL",
"INFO",
),
)
parser.add_argument(
"--tick-interval",
type=float,
default=None,
help="Override CONTROLLER_MASTER_TICK_INTERVAL_S",
)
parser.add_argument(
"--discovery-only-once",
action="store_true",
help="Run discovery once + exit (for smoke testing)",
)
parser.add_argument(
"--skip-backfill",
action="store_true",
help="Skip the startup backfill sweep (for tests + warm restarts)",
)
parser.add_argument(
"--no-opt-in-label",
action="store_true",
help="Disable the opt-in label gate (manage all "
"PRs/issues). DEFAULT is to manage ONLY "
"entities carrying CONTROLLER_OPT_IN_LABEL "
"(default 'controller-managed').",
)
# T5-7 (2026-05-19): the controller-internal MERGING handler was
# retired by T5-10 in favor of ``tools/merge_drive.py`` acting as
# the singleton merge process via the controller-DB bridge. Run
# exactly one ``merge_drive.py`` per (owner, repo) with
# ``CLEVERAGENTS_DB_URL`` set; this controller process never
# touches MERGING/APPROVED workflows. The pre-T5-10 ``--enable-merging``
# flag was removed to prevent operators from accidentally racing
# two writers against the same Forgejo merge POST.
args = parser.parse_args(argv)
require_opt_in_label = not args.no_opt_in_label
_setup_logging(args.log_level)
logger = logging.getLogger(__name__)
db_url = os.environ.get("CLEVERAGENTS_DB_URL")
if not db_url:
logger.error(
"CLEVERAGENTS_DB_URL not set. For production set to a "
"Postgres URL; for local dev sqlite:///./controller.db"
)
return 2
engine = build_engine(db_url)
create_all(engine)
# Wire Forgejo callbacks.
cfg = build_cfg_stub(args.owner, args.repo)
callbacks = build_callbacks(cfg)
# Predicate the ci_poll_exhaustion sweep consults to avoid STUCKing
# AWAITING_CI workflows while local CI is mid-run; stays None (a
# no-op) unless the RUN_CI_LOCAL block below wires it.
local_ci_in_flight = None
# RUN_CI_LOCAL: swap the Forgejo CI-status callbacks for ones backed
# by local ``tools/run-ci-full-local.sh`` runs (forgejo-runner exec).
# Use this when the cluster's CI infra is broken — forgejo-runner
# sources the code from a local checkout, so it still produces real
# gate verdicts. The ci_gate / ci_status_poll / prefetch consumers
# all read these two callbacks, so the swap covers every CI path.
if _run_ci_local_enabled():
import dataclasses as _dc
from pathlib import Path as _Path
from .local_ci import (
any_run_in_flight,
build_local_ci_callbacks,
default_clone_url,
preflight_local_ci,
)
repo_root = _Path(__file__).resolve().parents[3]
script_path = repo_root / "tools" / "run-ci-full-local.sh"
clone_url = default_clone_url(args.owner, args.repo)
if clone_url is None:
logger.error(
"RUN_CI_LOCAL is set but FORGEJO_URL/FORGEJO_API_BASE is "
"unset/invalid — cannot build a clone URL for local CI"
)
return 2
if not script_path.exists():
logger.error("RUN_CI_LOCAL is set but %s is missing", script_path)
return 2
# Fail loudly here if the host can't actually run local CI —
# otherwise a missing forgejo-runner / dead Docker daemon
# surfaces as a red CI verdict on every PR instead.
ci_local_error = preflight_local_ci(repo_root)
if ci_local_error:
logger.error(
"RUN_CI_LOCAL is set but local CI is not ready: %s",
ci_local_error,
)
return 2
local_get_ci, local_get_logs = build_local_ci_callbacks(
clone_url=clone_url,
script_path=str(script_path),
repo_root=str(repo_root),
forgejo_token=getattr(cfg, "token", "") or "",
)
callbacks = _dc.replace(
callbacks,
get_ci_status=local_get_ci,
get_failure_logs=local_get_logs,
)
logger.info(
"RUN_CI_LOCAL enabled — CI verdicts come from local "
"run-ci-full-local.sh runs (forgejo-runner exec), NOT the "
"Forgejo combined-status API"
)
local_ci_in_flight = any_run_in_flight
if args.discovery_only_once:
report = run_discovery(
engine,
owner=args.owner,
repo=args.repo,
list_prs=callbacks.list_prs,
list_issues=callbacks.list_issues,
require_opt_in_label=require_opt_in_label,
)
logger.info(
"discovery: PRs=%d issues=%d new=%d existing-skipped=%d",
report.prs_seen,
report.issues_seen,
report.new_workflows,
report.existing_skipped,
)
return 0
# Stop event wired to SIGTERM/SIGINT for clean systemd shutdown.
stop = threading.Event()
def _on_signal(signum, _frame):
logger.info("received signal %d; setting stop event", signum)
stop.set()
signal.signal(signal.SIGTERM, _on_signal)
signal.signal(signal.SIGINT, _on_signal)
cfg_loop = MasterConfig()
if args.tick_interval is not None:
# R-round4 P5: previously this rebuild dropped reconcile /
# ci_poll / discovery intervals — operators who passed
# --tick-interval silently reverted those to defaults. Use
# dataclasses.replace to preserve every other field.
import dataclasses as _dc
cfg_loop = _dc.replace(cfg_loop, tick_interval_s=args.tick_interval)
logger.info(
"master starting: owner=%s repo=%s opencode=%s db=%s tick=%.1fs",
args.owner,
args.repo,
args.opencode_url,
db_url,
cfg_loop.tick_interval_s,
)
# Phase 1 grooming plan (decision #21 / "Startup forensics"): log
# the effective grooming config at INFO so operators have a
# log-grep anchor for "what was set when this incident happened?".
# Always emits one line regardless of whether grooming is enabled.
from .grooming_config import (
get_grooming_config as _get_grooming_cfg,
log_effective_config as _log_grooming_cfg,
)
_log_grooming_cfg()
_grooming_cfg = _get_grooming_cfg()
# Phase 2 (Gate 2 estimator abandon, 2026-05-25): log + load the
# kill-switch config. Same startup-forensics rationale as
# grooming — operators get a log-grep anchor for "what was set
# when this incident happened?".
_log_gate2_abandon_cfg()
_gate2_abandon_cfg = _get_gate2_abandon_cfg()
# Phase 3 (Gate 3 reviewer abandon, 2026-05-25): log + load the
# kill-switch config. Same startup-forensics rationale as Phase 2 —
# operators get a log-grep anchor for "what was set when this
# incident happened?".
_log_gate3_abandon_cfg()
_gate3_abandon_cfg = _get_gate3_abandon_cfg()
# Phase 4 (metadata-hygiene, 2026-05-25): log + load the config.
# Same startup-forensics rationale — operators get a log-grep
# anchor for "what was set when this incident happened?".
_log_metadata_hygiene_cfg()
_metadata_hygiene_cfg = _get_metadata_hygiene_cfg()
# Phase 1k++ (N6): parser-coverage check runs BEFORE backfill +
# main loop so strict-mode failure exits 2 without wasting a
# Forgejo round-trip + without dependent code paths firing. The
# check is about deployment readiness, not run-state.
from ..ci_summary_parsers._registry import validate_parser_coverage
coverage = validate_parser_coverage()
real = sum(1 for v in coverage.values() if v)
total = len(coverage)
stubs = [name for name, is_real in coverage.items() if not is_real]
if stubs:
logger.warning(
"CI parser coverage: %d/%d real (%d stub: %s). Stub-parser "
"gates fall back to raw_log_excerpt; implementers see the "
"log but not structured findings.",
real,
total,
len(stubs),
sorted(stubs),
)
else:
logger.info("CI parser coverage: %d/%d real (no stubs)", real, total)
strict = os.environ.get(
"CONTROLLER_STRICT_PARSER_COVERAGE",
"0",
).lower() in {"1", "true", "yes"}
if strict and stubs:
logger.error(
"CONTROLLER_STRICT_PARSER_COVERAGE=1 and %d stub parsers "
"remain (%s); refusing to start. Either implement them or "
"unset the env var.",
len(stubs),
sorted(stubs),
)
return 2
# Run backfill BEFORE entering the main loop so existing PRs are
# known by the time the first tick fires.
if not args.skip_backfill:
try:
run_startup_backfill(
engine,
owner=args.owner,
repo=args.repo,
list_prs=callbacks.list_prs,
list_issues=callbacks.list_issues,
require_opt_in_label=require_opt_in_label,
)
except Exception:
logger.exception(
"startup backfill raised; continuing to main loop "
"(discovery tick will retry)"
)
if require_opt_in_label:
from .label_gate import opt_in_label_name
logger.info(
"controller opt-in label gate ENABLED: only managing "
"PRs/issues with label %r",
opt_in_label_name(),
)
else:
logger.info(
"controller opt-in label gate DISABLED (--no-opt-in-label): "
"managing all PRs/issues"
)
# Phase 1k+++ (real-run): wire the production prefetch callback
# so the master loop actually schedules attempts. Without this
# the loop would tick + reconcile but never enqueue anything.
prefetch_data = PrefetchDataCallbacks(
get_pr_details=callbacks.get_pr_details,
get_pr_diff=callbacks.get_pr_diff,
list_pr_reviews=callbacks.list_pr_reviews,
list_pr_comments=callbacks.list_pr_comments,
# P2: wire the CI-status fetcher so prefetch populates
# ci_summary / failing_gates instead of leaving them null.
get_ci_status=callbacks.get_ci_status,
# Unified CI-log fetcher so each failed gate's raw_log_excerpt
# is filled from the cache (not left empty).
get_ci_logs=callbacks.get_ci_logs,
# Actions-task fetcher — lets the implementer's ci_summary run
# the same zombie-CI active-run check the poll does, so the two
# agree (no ci-not-ready <-> ci_red ping-pong on a dead run).
get_action_tasks=callbacks.get_action_tasks,
# Phase 1 grooming gate: wire the discovery list_prs callback as
# the open-PR universe fetcher so the grooming worker compares
# the anchor PR against every currently-open PR without any
# Forgejo I/O of its own.
list_open_prs=callbacks.list_prs,
)
prefetch_cb = make_prefetch_callback(engine, prefetch_data)
# T5-7 + T5-10: controller-internal MERGING handler retired in
# favor of tools/merge_drive.py acting as the singleton merge
# process via the controller-DB bridge. ``merging_args=None``
# disables the master's MERGING handler entirely.
merging_args = None
logger.info(
"MERGING handler DISABLED (T5-7/T5-10): merge_drive.py owns "
"the merge step via the controller-DB bridge; this master "
"writes workflows to APPROVED and stops."
)
master_main_loop(
engine,
config=cfg_loop,
stop_event=stop,
reconciliation_args=(
args.owner,
args.repo,
callbacks.get_pr_state,
callbacks.get_issue_state,
{"require_opt_in_label": require_opt_in_label},
),
prefetch=prefetch_cb,
merging_args=merging_args,
discovery_args=(
args.owner,
args.repo,
callbacks.list_prs,
callbacks.list_issues,
{"require_opt_in_label": require_opt_in_label},
),
# ci_status_poll: ``get_failure_logs`` lets the poll classify a
# failure infra-vs-real — a reran-CI that is ALSO infra-broken
# routes back to DISCOVERED for the gate's rerun budget instead
# of sending an implementer to flail on infra. ``get_pr_details``
# feeds the pre-review mergeable gate: a green-CI PR that no
# longer merges cleanly is routed to CONFLICT_RESOLVING instead
# of REVIEWING, so the LLM reviewer is not spent on code that
# must be rebased.
# ``get_action_tasks`` (6th) is the zombie-CI active-run check:
# it lets the poll ask Forgejo whether a still-pending run is
# actually executing, so a dead run is acted on, not waited on.
ci_status_poll_args=(
args.owner,
args.repo,
callbacks.get_ci_status,
callbacks.get_failure_logs,
callbacks.get_pr_details,
callbacks.get_action_tasks,
),
# CI-freshness gate: divert DISCOVERED PRs whose CI is stale /
# infra-broken (re-trigger CI via empty-commit push → AWAITING_CI,
# or STUCK once the rerun budget is exhausted) before they burn
# an estimator + implementer attempt. ``get_failure_logs`` gives
# the gate's classifier the real CI job-log content it scans for
# infra signatures.
ci_gate_args=(
args.owner,
args.repo,
callbacks.get_ci_status,
callbacks.get_pr_details,
callbacks.trigger_ci_rerun,
callbacks.get_failure_logs,
),
# Phase 1 corrected dispatch (2026-05-25): grooming side-effect
# tick. Only wired when CONTROLLER_GROOMING_ENABLED=true; the
# state-machine events (groom_verdict_*) are also only fired by
# promote.py when the same config is enabled, so the tick has
# nothing to do under the disabled default. ``dry_run`` from
# the same config gates whether the Forgejo writes actually
# fire (vs audit-only).
grooming_callbacks=(
GroomingCallbacks(
list_comments=callbacks.list_comments,
post_comment=callbacks.post_comment,
patch_pr_state=callbacks.patch_pr_state,
get_labels=callbacks.get_labels,
add_label=callbacks.add_label,
remove_label=callbacks.remove_label,
dry_run=_grooming_cfg.dry_run,
)
if _grooming_cfg.enabled
else None
),
# Phase 2 corrected dispatch (2026-05-25): estimator-abandon
# side-effect tick. Gated on CONTROLLER_GATE2_ABANDON_ENABLED
# (default False) so a fresh Phase 2 deploy is audit-only —
# the agent prompt instructs the estimator to use abandon
# when applicable, but no Forgejo close fires until the
# operator explicitly flips the kill switch. Audit rows
# accumulate in ``grooming_decisions`` for operator review
# regardless. ``dry_run`` (from CONTROLLER_GROOMING_DRY_RUN)
# is the second safe-rollout layer: when ENABLED=true AND
# DRY_RUN=true, the tick runs but close_act short-circuits
# before any Forgejo write.
estimator_abandon_callbacks=(
EstimatorAbandonCallbacks(
list_comments=callbacks.list_comments,
post_comment=callbacks.post_comment,
patch_pr_state=callbacks.patch_pr_state,
dry_run=_grooming_cfg.dry_run,
)
if _gate2_abandon_cfg.enabled
else None
),
# Phase 3 (2026-05-25): reviewer-abandon side-effect tick.
# Gated on CONTROLLER_GATE3_ABANDON_ENABLED (default False) so
# a fresh Phase 3 deploy is audit-only — the agent prompt
# instructs the reviewer to use abandon when applicable, but
# no Forgejo close fires until the operator explicitly flips
# the kill switch. Audit rows accumulate in
# ``grooming_decisions`` for operator review regardless.
# Shares ``dry_run`` with grooming + Gate 2 so all three abandon
# paths roll out under one safe-rollout toggle.
reviewer_abandon_callbacks=(
ReviewerAbandonCallbacks(
list_comments=callbacks.list_comments,
post_comment=callbacks.post_comment,
patch_pr_state=callbacks.patch_pr_state,
dry_run=_grooming_cfg.dry_run,
)
if _gate3_abandon_cfg.enabled
else None
),
# Phase 4 (2026-05-25): metadata-hygiene tick. Gated on the
# master CONTROLLER_METADATA_HYGIENE_ENABLED flag (default
# False) — each of the five sub-checks has its own per-check
# flag. ``dry_run`` shares the grooming flag for symmetry with
# Gates 1/2/3 (one safe-rollout toggle covers all metadata
# write paths).
metadata_hygiene_callbacks=(
MetadataHygieneCallbacks(
get_pr_details=callbacks.get_pr_details,
# get_issue_state hits /issues/{n} — needed by check #5
# to read an issue's milestone. Using get_pr_details
# (/pulls/{n}) would 404 on plain issues. (Round-2 fix.)
get_issue_state=callbacks.get_issue_state,
get_labels=callbacks.get_labels,
add_label=callbacks.add_label,
remove_label=callbacks.remove_label,
patch_pr_state=callbacks.patch_pr_state,
patch_pr_body=callbacks.patch_pr_body,
patch_pr_milestone=callbacks.patch_pr_milestone,
completed_not_closed_enabled=(
_metadata_hygiene_cfg.completed_not_closed
),
closing_keyword_fixup_enabled=(
_metadata_hygiene_cfg.closing_keyword_fixup
),
label_sync_enabled=_metadata_hygiene_cfg.label_sync_from_issue,
state_label_inference_enabled=(
_metadata_hygiene_cfg.state_label_inference
),
milestone_assignment_enabled=(
_metadata_hygiene_cfg.milestone_assignment
),
dry_run=_grooming_cfg.dry_run,
)
if _metadata_hygiene_cfg.enabled
else None
),
# RUN_CI_LOCAL: skip ci_poll_exhaustion while local CI is busy
# (None — a no-op — under remote CI).
local_ci_in_flight=local_ci_in_flight,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())