drew
a6986008ee
T5-1 reviewer feedback rendered full-body to the implementer
T5-4/9 implementer dispute path — dispute-at-any-tier with per-tier cap,
OPERATOR_ATTENTION state on stalemate, pr-review-worker-dispute agent
T5-5 reviewer BLOCKING ISSUE EVIDENCE RULE + 5-step validation
T5-7 merge step split into a singleton process — impl/review masters write
APPROVED and stop; merge_drive owns APPROVED -> MERGING -> MERGED
T5-10 merge process is fully deterministic; base conflicts bounce to the
controller's CONFLICT_RESOLVING (LLM); conflict_drive sidecar retired
T5-11 implementer fast success path — verified-clean outcome so a no-op
after conflict resolution doesn't force busywork
T5-12 conflict-resolver permissions fixed across all paths (/tmp/** glob)
T5-13 conflict-resolver PR-intent prehydration (title/body/comments)
Adds tools/_controller_db_bridge.py so merge_drive reads the controller DB
directly (Option B), plus APPROVED + OPERATOR_ATTENTION states, the
dispute/verified-clean events, and the V1 contract fields backing them.
Reviewer model: baseline -> sonnet, dispute -> opus.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
482 lines
19 KiB
Python
482 lines
19 KiB
Python
"""MERGING handler — calls Forgejo's merge endpoint + maps the
|
|
6-response-shape table to state machine transitions.
|
|
|
|
.. deprecated:: 2026-05-19 (T5-7 + T5-10)
|
|
The controller-internal MERGING handler is RETIRED in production.
|
|
``tools/merge_drive.py`` is now the singleton merge process; it
|
|
talks to the controller DB via ``tools/_controller_db_bridge.py``
|
|
(APPROVED → MERGING claim + outcome events). The production master
|
|
no longer registers this handler (``merging_args=None`` in
|
|
``master/__main__.py``).
|
|
|
|
This module is preserved because:
|
|
- ``MergeResponse`` and ``MergeCallback`` types are used by
|
|
``master/forgejo_http.py`` and by the bridge's typing layer.
|
|
- ``run_merging_tick`` and ``MergingHandlerReport`` are still
|
|
exercised by unit tests as a reference implementation of the
|
|
response-shape → state-transition mapping. ``merge_drive.py``'s
|
|
bridge integration MUST produce the same transitions.
|
|
- The 6-response-shape mapping itself (200/409/422/403/404/5xx)
|
|
is documentation that future merge-process implementations
|
|
should consult.
|
|
|
|
The handler will be deleted once an end-to-end test of
|
|
``merge_drive.py`` covers each response shape, at which point
|
|
the reference role this module plays is met by that test.
|
|
|
|
Per plan v9 non-blocking retry: MERGING stays in MERGING across
|
|
ticks. The handler:
|
|
1. Checks if workflows.merging_retry_next_attempt_at is in the future
|
|
(if so, skip this tick).
|
|
2. Calls the merge callback (production: Forgejo POST /pulls/{n}/merge).
|
|
3. Maps the response (T5-10 update for 409):
|
|
200 success → MERGED
|
|
409 conflict → CONFLICT_RESOLVING with
|
|
reason='post-approval-base-conflict'
|
|
403 forbidden → STUCK (branch protection)
|
|
422 missing-checks → AWAITING_CI (rare race)
|
|
5xx → stay MERGING, retry_count++, backoff
|
|
STUCK at retry_count >= 5
|
|
404 → check PR state → MERGED (extern-merged)
|
|
OR ABANDONED (extern-closed)
|
|
|
|
Forgejo HTTP injected via two callbacks so tests don't need a live
|
|
instance. Production wires to existing _claim_runtime helpers.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.engine import Engine
|
|
|
|
from ..db.session import session_scope
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Per plan v9: STUCK after 5 retries on transient errors.
|
|
MAX_MERGE_RETRIES = 5
|
|
# Backoff: 2^retry_count seconds, capped at 60s.
|
|
MAX_BACKOFF_S = 60.0
|
|
|
|
|
|
@dataclass
|
|
class MergeResponse:
|
|
"""Normalized Forgejo merge response shape.
|
|
|
|
The callback returns this rather than raw HTTP details so the
|
|
handler can route deterministically. Maps to plan v9's 6-response
|
|
table.
|
|
"""
|
|
|
|
status_code: int
|
|
pr_state: str | None = None # 'merged' | 'open' | 'closed' | None
|
|
error_message: str | None = None
|
|
|
|
|
|
# Forgejo merge call callback. Returns a MergeResponse summarizing
|
|
# the HTTP attempt + (for 404 paths) the PR's actual state.
|
|
#
|
|
# Signature: ``(owner, repo, pr_number, approved_at_sha)``. The
|
|
# ``approved_at_sha`` arg is the exact SHA the reviewer signed off
|
|
# on (from ``ReviewerOutputV1.approved_at_sha``); the callback
|
|
# SHOULD pass it to Forgejo's merge endpoint as ``head_commit_id``
|
|
# so a race-condition push between approval and merge produces a
|
|
# 409 instead of silently merging unapproved code. May be None if
|
|
# the controller can't locate a reviewer attempt (defensive
|
|
# fallback — Forgejo merges whatever HEAD currently is).
|
|
MergeCallback = Callable[[str, str, int, str | None], MergeResponse]
|
|
|
|
|
|
@dataclass
|
|
class MergingHandlerReport:
|
|
"""Per-tick summary of the MERGING handler."""
|
|
|
|
workflows_processed: int = 0
|
|
workflows_merged: int = 0
|
|
# T5-10: 409 (base conflict) now bounces to CONFLICT_RESOLVING
|
|
# instead of IMPLEMENTING. The old counter name is kept for back-
|
|
# compat but is always zero post-T5-10; readers should use
|
|
# ``workflows_to_conflict_resolving`` instead.
|
|
workflows_back_to_implementing: int = 0
|
|
workflows_to_conflict_resolving: int = 0
|
|
workflows_stuck: int = 0
|
|
workflows_awaiting_ci: int = 0
|
|
workflows_abandoned: int = 0
|
|
workflows_retrying: int = 0 # retry scheduled for future tick
|
|
transitions: list[dict] = field(default_factory=list)
|
|
|
|
|
|
def run_merging_tick(
|
|
engine: Engine,
|
|
*,
|
|
merge: MergeCallback,
|
|
owner: str,
|
|
repo: str,
|
|
max_retries: int = MAX_MERGE_RETRIES,
|
|
) -> MergingHandlerReport:
|
|
"""One MERGING-handling sweep.
|
|
|
|
For each workflow with current_state='MERGING' where
|
|
merging_retry_next_attempt_at is null or in the past:
|
|
1. Call the merge callback.
|
|
2. Apply the response → state transition (via direct
|
|
UPDATE because MERGING transitions don't fit the
|
|
outcome-mapper protocol).
|
|
3. Insert a controller_events row.
|
|
"""
|
|
report = MergingHandlerReport()
|
|
now = datetime.now(timezone.utc)
|
|
|
|
with session_scope(engine) as session:
|
|
rows = session.execute(
|
|
text(
|
|
"SELECT w.workflow_id, w.entity_number, "
|
|
" w.current_tier, w.tier_last_succeeded, "
|
|
" w.merging_retry_count, "
|
|
" w.merging_retry_next_attempt_at, "
|
|
# Pull approved_at_sha from the latest reviewer
|
|
# attempt's output_payload. ReviewerOutputV1.verdict='approve'
|
|
# carries the exact SHA the reviewer signed off on; the
|
|
# merge callback uses it as Forgejo's head_commit_id
|
|
# parameter so a race-condition push between approval
|
|
# and merge results in a 409 instead of silently merging
|
|
# unapproved code.
|
|
" (SELECT a.output_payload FROM workflow_attempts a "
|
|
" WHERE a.workflow_id = w.workflow_id "
|
|
" AND a.role = 'reviewer' "
|
|
" AND a.status = 'complete' "
|
|
" ORDER BY a.attempt_number DESC LIMIT 1) AS reviewer_payload "
|
|
" FROM workflows w "
|
|
" WHERE w.current_state = 'MERGING' "
|
|
" AND w.owner = :owner AND w.repo = :repo "
|
|
" AND w.kind = 'pr' "
|
|
" AND (w.merging_retry_next_attempt_at IS NULL "
|
|
" OR w.merging_retry_next_attempt_at <= :now)"
|
|
),
|
|
{"owner": owner, "repo": repo, "now": now},
|
|
).all()
|
|
|
|
for r in rows:
|
|
report.workflows_processed += 1
|
|
transition = _process_merging_row(
|
|
session, r, merge=merge, owner=owner, repo=repo,
|
|
max_retries=max_retries, now=now,
|
|
)
|
|
report.transitions.append(transition)
|
|
kind = transition.get("kind", "unknown")
|
|
if kind == "merged":
|
|
report.workflows_merged += 1
|
|
elif kind == "implementing":
|
|
# Pre-T5-10 path; should no longer fire but kept for
|
|
# back-compat with any external callers.
|
|
report.workflows_back_to_implementing += 1
|
|
elif kind == "conflict_resolving":
|
|
report.workflows_to_conflict_resolving += 1
|
|
elif kind == "stuck":
|
|
report.workflows_stuck += 1
|
|
elif kind == "awaiting_ci":
|
|
report.workflows_awaiting_ci += 1
|
|
elif kind == "abandoned":
|
|
report.workflows_abandoned += 1
|
|
elif kind == "retry":
|
|
report.workflows_retrying += 1
|
|
|
|
return report
|
|
|
|
|
|
# ─── per-row processing ───────────────────────────────────────────────
|
|
|
|
|
|
def _process_merging_row(
|
|
session, row, *,
|
|
merge: MergeCallback, owner: str, repo: str,
|
|
max_retries: int, now: datetime,
|
|
) -> dict:
|
|
"""Call the merge callback + apply the response. Returns a
|
|
transition record (used for telemetry + reporting)."""
|
|
# Extract approved_at_sha from the latest reviewer attempt's
|
|
# output_payload. None when no reviewer attempt is found OR when
|
|
# the payload doesn't carry the field (e.g., verdict was 'abstain'
|
|
# rather than 'approve' — that shouldn't reach MERGING, but be
|
|
# defensive). When None, the merge callback falls back to merging
|
|
# whatever HEAD currently is.
|
|
approved_at_sha: str | None = None
|
|
raw_payload = row.reviewer_payload if hasattr(row, "reviewer_payload") else None
|
|
if raw_payload:
|
|
try:
|
|
payload = (
|
|
json.loads(raw_payload)
|
|
if isinstance(raw_payload, str) else raw_payload
|
|
)
|
|
if isinstance(payload, dict):
|
|
candidate = payload.get("approved_at_sha")
|
|
if isinstance(candidate, str) and candidate:
|
|
approved_at_sha = candidate
|
|
except (ValueError, TypeError):
|
|
logger.warning(
|
|
"merging: could not decode reviewer payload for PR #%s",
|
|
row.entity_number,
|
|
)
|
|
if approved_at_sha is None:
|
|
logger.warning(
|
|
"merging: PR #%s has no approved_at_sha from reviewer; "
|
|
"Forgejo will merge whatever HEAD currently is (race risk)",
|
|
row.entity_number,
|
|
)
|
|
|
|
try:
|
|
resp = merge(owner, repo, row.entity_number, approved_at_sha)
|
|
except Exception as exc: # noqa: BLE001 — treat as transient
|
|
# Wrap in a synthetic 503 so the retry logic kicks in.
|
|
logger.warning(
|
|
"merge callback raised for PR #%s: %s; treating as 5xx",
|
|
row.entity_number, exc,
|
|
)
|
|
resp = MergeResponse(
|
|
status_code=503, error_message=f"callback raised: {exc}"
|
|
)
|
|
|
|
status = resp.status_code
|
|
|
|
# ── 200: merged ──
|
|
if status == 200:
|
|
_transition(
|
|
session, row.workflow_id, "MERGING", "MERGED", now,
|
|
event="merge_ok",
|
|
payload={"reason": "merged", "http_status": status},
|
|
)
|
|
return {"kind": "merged", "workflow_id": row.workflow_id}
|
|
|
|
# ── 409: conflict (base advanced post-approval) ──
|
|
if status == 409:
|
|
# T5-10 (2026-05-19): route directly to CONFLICT_RESOLVING so
|
|
# the controller's conflict_resolver role (LLM) handles the
|
|
# rebase + conflict resolution. Pre-T5-10 this went to
|
|
# IMPLEMENTING, which burned a wasted implementer attempt just
|
|
# to re-discover the conflict and emit ``implementer_rebase_failed``.
|
|
# Merge processes stay 100% deterministic; LLM work bounces
|
|
# back to the controller exactly once.
|
|
_transition(
|
|
session, row.workflow_id, "MERGING", "CONFLICT_RESOLVING", now,
|
|
event="merge_base_conflict",
|
|
payload={
|
|
"reason": "post-approval-base-conflict",
|
|
"http_status": status,
|
|
"source": "merging_handler",
|
|
},
|
|
)
|
|
# Reset merging retry counter for next time.
|
|
_reset_merging_retries(session, row.workflow_id)
|
|
return {"kind": "conflict_resolving", "workflow_id": row.workflow_id}
|
|
|
|
# ── 403: branch protection ──
|
|
if status == 403:
|
|
_transition(
|
|
session, row.workflow_id, "MERGING", "STUCK", now,
|
|
event="merge_branch_protection_blocked",
|
|
payload={
|
|
"reason": "branch-protection",
|
|
"http_status": status,
|
|
"error": resp.error_message,
|
|
},
|
|
)
|
|
# E-8 fix: reset retry counter so an operator_unstick that
|
|
# routes the workflow back through MERGING doesn't inherit
|
|
# stale retry state.
|
|
_reset_merging_retries(session, row.workflow_id)
|
|
return {"kind": "stuck", "workflow_id": row.workflow_id}
|
|
|
|
# ── 422: CI required-status missing (race) ──
|
|
if status == 422:
|
|
_transition(
|
|
session, row.workflow_id, "MERGING", "AWAITING_CI", now,
|
|
event="merge_ci_required_missing",
|
|
payload={
|
|
"reason": "ci-required-status-missing",
|
|
"http_status": status,
|
|
},
|
|
)
|
|
_reset_merging_retries(session, row.workflow_id)
|
|
return {"kind": "awaiting_ci", "workflow_id": row.workflow_id}
|
|
|
|
# ── 404: PR closed externally (merged or abandoned) ──
|
|
if status == 404:
|
|
# Use the response's pr_state to determine which side.
|
|
# If the callback didn't populate it, default to ABANDONED
|
|
# (conservative — operator can re-open via controller-cli).
|
|
if resp.pr_state == "merged":
|
|
_transition(
|
|
session, row.workflow_id, "MERGING", "MERGED", now,
|
|
event="merge_external_action",
|
|
payload={"reason": "externally-merged", "http_status": status},
|
|
)
|
|
_reset_merging_retries(session, row.workflow_id)
|
|
return {"kind": "merged", "workflow_id": row.workflow_id}
|
|
_transition(
|
|
session, row.workflow_id, "MERGING", "ABANDONED", now,
|
|
event="merge_external_action",
|
|
payload={
|
|
"reason": "externally-closed-or-removed",
|
|
"http_status": status, "pr_state": resp.pr_state,
|
|
},
|
|
)
|
|
# E-8 fix: reset retry counter on terminal transition.
|
|
_reset_merging_retries(session, row.workflow_id)
|
|
return {"kind": "abandoned", "workflow_id": row.workflow_id}
|
|
|
|
# ── 5xx / anything else: transient. Retry until exhausted ──
|
|
next_retry_count = (row.merging_retry_count or 0) + 1
|
|
if next_retry_count >= max_retries:
|
|
_transition(
|
|
session, row.workflow_id, "MERGING", "STUCK", now,
|
|
event="merge_retry_exhausted",
|
|
payload={
|
|
"reason": "merge-retry-exhausted",
|
|
"http_status": status,
|
|
"retries": next_retry_count,
|
|
"error": resp.error_message,
|
|
},
|
|
)
|
|
# E-8 fix: reset retry counter on terminal transition so
|
|
# operator_unstick doesn't inherit stale state.
|
|
_reset_merging_retries(session, row.workflow_id)
|
|
return {"kind": "stuck", "workflow_id": row.workflow_id}
|
|
|
|
backoff = min(MAX_BACKOFF_S, float(2 ** next_retry_count))
|
|
next_at = now + timedelta(seconds=backoff)
|
|
session.execute(
|
|
text(
|
|
"UPDATE workflows SET "
|
|
" merging_retry_count = :count, "
|
|
" merging_retry_next_attempt_at = :next_at "
|
|
"WHERE workflow_id = :wf_id"
|
|
),
|
|
{
|
|
"count": next_retry_count, "next_at": next_at,
|
|
"wf_id": row.workflow_id,
|
|
},
|
|
)
|
|
# Record the retry attempt as a controller_event (not a transition).
|
|
session.execute(
|
|
text(
|
|
"INSERT INTO controller_events "
|
|
"(workflow_id, ts, event_type, payload, "
|
|
" forgejo_write_pending, replay_attempts) "
|
|
"VALUES (:wf_id, :ts, 'merge-retry-scheduled', :payload, 0, 0)"
|
|
),
|
|
{
|
|
"wf_id": row.workflow_id, "ts": now,
|
|
"payload": json.dumps({
|
|
"http_status": status,
|
|
"retry_count": next_retry_count,
|
|
"next_attempt_at": next_at.isoformat(),
|
|
"backoff_s": backoff,
|
|
"error": resp.error_message,
|
|
}),
|
|
},
|
|
)
|
|
return {
|
|
"kind": "retry", "workflow_id": row.workflow_id,
|
|
"retry_count": next_retry_count,
|
|
"next_attempt_at": next_at.isoformat(),
|
|
}
|
|
|
|
|
|
# ─── helpers ──────────────────────────────────────────────────────────
|
|
|
|
|
|
def _transition(
|
|
session, workflow_id: int, from_state: str, to_state: str,
|
|
now: datetime, *,
|
|
event: str, tier: int | None = None, payload: dict[str, Any] | None = None,
|
|
) -> None:
|
|
"""Write a workflow state transition + controller_events row.
|
|
|
|
R-8 fix (2026-05-19): the UPDATE now filters
|
|
``WHERE current_state = :from_state`` matching the TOCTOU defense
|
|
already used in ci_status_poll + reconciliation. If another tick
|
|
advanced the workflow between the merging-handler SELECT and this
|
|
UPDATE, we skip the no-op event row to keep the audit trail
|
|
accurate.
|
|
"""
|
|
if tier is not None:
|
|
result = session.execute(
|
|
text(
|
|
"UPDATE workflows SET "
|
|
" current_state = :to, current_tier = :tier, "
|
|
" last_transition_at = :now, entered_state_at = :now "
|
|
"WHERE workflow_id = :wf_id "
|
|
" AND current_state = :from"
|
|
),
|
|
{
|
|
"to": to_state, "tier": tier, "now": now,
|
|
"wf_id": workflow_id, "from": from_state,
|
|
},
|
|
)
|
|
else:
|
|
result = session.execute(
|
|
text(
|
|
"UPDATE workflows SET "
|
|
" current_state = :to, "
|
|
" last_transition_at = :now, entered_state_at = :now "
|
|
"WHERE workflow_id = :wf_id "
|
|
" AND current_state = :from"
|
|
),
|
|
{
|
|
"to": to_state, "now": now,
|
|
"wf_id": workflow_id, "from": from_state,
|
|
},
|
|
)
|
|
if (result.rowcount or 0) == 0:
|
|
logger.info(
|
|
"merging: workflow %s no longer in %s (race lost); "
|
|
"skipping event row",
|
|
workflow_id, from_state,
|
|
)
|
|
return
|
|
session.execute(
|
|
text(
|
|
"INSERT INTO controller_events "
|
|
"(workflow_id, ts, event_type, from_state, to_state, payload, "
|
|
" forgejo_write_pending, replay_attempts) "
|
|
"VALUES (:wf_id, :ts, 'transition', :from_state, :to_state, "
|
|
" :payload, 0, 0)"
|
|
),
|
|
{
|
|
"wf_id": workflow_id, "ts": now,
|
|
"from_state": from_state, "to_state": to_state,
|
|
"payload": json.dumps({**(payload or {}), "event": event}),
|
|
},
|
|
)
|
|
|
|
|
|
def _reset_merging_retries(session, workflow_id: int) -> None:
|
|
"""Clear the merging retry counter + backoff after a non-retry
|
|
response (success / conflict / 422)."""
|
|
session.execute(
|
|
text(
|
|
"UPDATE workflows SET "
|
|
" merging_retry_count = 0, "
|
|
" merging_retry_next_attempt_at = NULL "
|
|
"WHERE workflow_id = :wf_id"
|
|
),
|
|
{"wf_id": workflow_id},
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"MAX_BACKOFF_S",
|
|
"MAX_MERGE_RETRIES",
|
|
"MergeCallback",
|
|
"MergeResponse",
|
|
"MergingHandlerReport",
|
|
"run_merging_tick",
|
|
]
|