cleveragents-core/tools/controller/master/reconciliation.py

"""Periodic reconciliation tick — DB ↔ Forgejo state sync.

Per plan v9: every ``CONTROLLER_RECONCILIATION_INTERVAL_S`` (default
300s = 5 min), the master compares its DB view of each non-terminal
workflow against Forgejo's actual state and resolves divergence:

- Forgejo says merged but DB says non-terminal → workflow → MERGED
  (someone merged externally; e.g., an operator clicked the merge
  button on Forgejo's UI bypassing the controller)
- Forgejo says closed (not merged) but DB says non-terminal →
  workflow → ABANDONED
- Forgejo says the PR was reopened after the controller marked it
  ABANDONED → workflow → DISCOVERED (re-enter the state machine)
- Forgejo doesn't know about the PR (404) → workflow → STUCK with
  reason='pr-not-found' (operator investigates)

This module is the inverse of the discovery tick: discovery adds NEW
entities; reconciliation re-syncs KNOWN ones. Both are idempotent.

Forgejo HTTP is callback-injected for testability — production wires
through the existing get/post helpers.
"""

from __future__ import annotations

import json
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timezone

from sqlalchemy import text
from sqlalchemy.engine import Engine

from ..db.session import session_scope
from .label_gate import (
    has_opt_in_label as _has_opt_in_label,
    opt_in_label_name as _opt_in_label_name,
)

logger = logging.getLogger(__name__)


# Callback: fetch one PR's state from Forgejo.
# Returns: dict like {"state": "open"|"closed"|"merged", "merged": bool, ...}
#          OR None if the PR doesn't exist (404).
GetPRStateCallback = Callable[[str, str, int], dict | None]
# Same for issues — though issues just have state="open"|"closed".
GetIssueStateCallback = Callable[[str, str, int], dict | None]


@dataclass
class ReconciliationAction:
    """One per (workflow, decision) pair the sweep produced."""

    workflow_id: int
    kind: str
    entity_number: int
    from_state: str
    to_state: str | None  # None = no action (state already consistent)
    reason: str


@dataclass
class ReconciliationReport:
    """Per-sweep summary."""

    workflows_checked: int = 0
    workflows_transitioned: int = 0
    workflows_already_consistent: int = 0
    workflows_fetch_failed: int = 0
    actions: list[ReconciliationAction] = field(default_factory=list)


def run_reconciliation_tick(
    engine: Engine,
    *,
    owner: str,
    repo: str,
    get_pr_state: GetPRStateCallback,
    get_issue_state: GetIssueStateCallback | None = None,
    opt_in_label: str | None = None,
    require_opt_in_label: bool = False,
) -> ReconciliationReport:
    """One sweep — check every non-terminal workflow for this
    (owner, repo) against Forgejo.

    Terminal-state workflows are NEVER touched (MERGED / ABANDONED /
    STUCK / CREATED_PR are immutable except via operator unstick).

    Phase 1k: when the Forgejo response carries a ``labels`` field and
    the opt-in label is no longer present, the workflow transitions to
    ABANDONED with reason ``opt-in-label-removed``. Operators can
    pause controller management mid-flight by removing the label.
    Set ``require_opt_in_label=False`` to disable the gate.
    """
    report = ReconciliationReport()
    now = datetime.now(timezone.utc)
    label = (
        (opt_in_label if opt_in_label is not None else _opt_in_label_name())
        if require_opt_in_label
        else None
    )

    with session_scope(engine) as session:
        # PAUSED IS scanned (so we can detect label-restored and
        # resume). Terminal states (MERGED/ABANDONED/STUCK/CREATED_PR)
        # are not.
        rows = session.execute(
            text(
                "SELECT workflow_id, kind, entity_number, current_state, "
                "       pre_pause_state, deferred_reason "
                "  FROM workflows "
                " WHERE owner = :owner AND repo = :repo "
                "   AND current_state NOT IN "
                "       ('MERGED', 'ABANDONED', 'STUCK', 'CREATED_PR')"
            ),
            {"owner": owner, "repo": repo},
        ).all()

        for r in rows:
            report.workflows_checked += 1
            action = _reconcile_one(
                session,
                owner=owner,
                repo=repo,
                row=r,
                now=now,
                get_pr_state=get_pr_state,
                get_issue_state=get_issue_state,
                opt_in_label=label,
            )
            report.actions.append(action)
            if action.to_state is None:
                if action.reason == "fetch-failed":
                    report.workflows_fetch_failed += 1
                else:
                    report.workflows_already_consistent += 1
            else:
                report.workflows_transitioned += 1

    if report.workflows_transitioned:
        logger.info(
            "reconciliation: %d checked, %d transitioned, "
            "%d consistent, %d fetch-failed",
            report.workflows_checked,
            report.workflows_transitioned,
            report.workflows_already_consistent,
            report.workflows_fetch_failed,
        )
    return report


def _reconcile_one(
    session,
    *,
    owner: str,
    repo: str,
    row,
    now: datetime,
    get_pr_state: GetPRStateCallback,
    get_issue_state: GetIssueStateCallback | None,
    opt_in_label: str | None,
) -> ReconciliationAction:
    """Fetch Forgejo state for one workflow + apply any transition."""
    kind = row.kind
    entity = row.entity_number
    current = row.current_state

    try:
        if kind == "pr":
            forgejo = get_pr_state(owner, repo, entity)
        elif kind == "issue":
            if get_issue_state is None:
                # Issue reconciliation optional; without a callback
                # treat as consistent.
                return ReconciliationAction(
                    workflow_id=row.workflow_id,
                    kind=kind,
                    entity_number=entity,
                    from_state=current,
                    to_state=None,
                    reason="no-issue-callback",
                )
            forgejo = get_issue_state(owner, repo, entity)
        else:
            return ReconciliationAction(
                workflow_id=row.workflow_id,
                kind=kind,
                entity_number=entity,
                from_state=current,
                to_state=None,
                reason=f"unknown-kind:{kind}",
            )
    except Exception as exc:  # noqa: BLE001 — Forgejo flake; skip this row
        logger.warning(
            "reconciliation: fetch failed for %s/%s #%d: %s",
            owner,
            repo,
            entity,
            exc,
        )
        return ReconciliationAction(
            workflow_id=row.workflow_id,
            kind=kind,
            entity_number=entity,
            from_state=current,
            to_state=None,
            reason="fetch-failed",
        )

    # Order of checks (Phase 1k++ refinement, N3):
    # 1. Terminal-state mappings from Forgejo (merged / closed) win
    #    FIRST — operator removing the opt-in label from an already-
    #    merged PR should still see it transition to MERGED, not get
    #    stuck in PAUSED forever.
    # 2. Then the opt-in label gate (pause/resume).
    # 3. Falls through to "consistent" if neither applies.

    target_state, reason = _decide_transition(forgejo, kind=kind)
    # Phase 1k++ refinement (R2): all Forgejo-terminal transitions
    # (MERGED, ABANDONED, STUCK) short-circuit the opt-in label gate.
    # Operator removing the label on a 404'd PR must NOT pause it
    # into PAUSED forever — STUCK is the right operator-action state
    # for unrecoverable Forgejo-side conditions.
    if target_state in {"MERGED", "ABANDONED", "STUCK"}:
        # External terminal state — apply it regardless of label.
        if target_state != current:
            _apply_transition(
                session,
                workflow_id=row.workflow_id,
                from_state=current,
                to_state=target_state,
                now=now,
                reason=reason,
            )
            return ReconciliationAction(
                workflow_id=row.workflow_id,
                kind=kind,
                entity_number=entity,
                from_state=current,
                to_state=target_state,
                reason=reason,
            )
        # Already in the target terminal state — fall through to
        # "consistent" below.

    # Opt-in label gate (Phase 1k+): pause / resume.
    #
    # The labels-missing case (Forgejo response without a "labels"
    # field) is treated as "label state unknown" and falls through —
    # a partial response can't accidentally pause a workflow.
    if opt_in_label is not None and isinstance(forgejo, dict) and "labels" in forgejo:
        label_present = _has_opt_in_label(forgejo, opt_in_label)
        if not label_present and current != "PAUSED":
            # PAUSE: capture pre_pause_state so we can resume cleanly.
            _apply_transition_with_pre_pause(
                session,
                workflow_id=row.workflow_id,
                from_state=current,
                to_state="PAUSED",
                now=now,
                reason="opt-in-label-removed",
                pre_pause_state=current,
            )
            return ReconciliationAction(
                workflow_id=row.workflow_id,
                kind=kind,
                entity_number=entity,
                from_state=current,
                to_state="PAUSED",
                reason="opt-in-label-removed",
            )
        # The ``row.deferred_reason`` read here is attribute-access not
        # ``getattr(..., None)`` ON PURPOSE — Phase 0 originally used
        # defensive getattr, which silently no-op'd the entire guard
        # for an entire phase when the column was missing from the
        # row-load SELECT at line 113. AttributeError would have
        # surfaced that bug at the first reconciliation tick instead
        # of hiding it until live grooming validation. If you add a
        # new column to this guard, add it to the SELECT too.
        if (
            label_present
            and current == "PAUSED"
            and not row.deferred_reason
        ):
            # RESUME: read pre_pause_state; fall back to DISCOVERED
            # if the column was never set (legacy data).
            #
            # Phase 1 grooming plan (decision #42 / #22 amendment):
            # the ``and not deferred_reason`` guard prevents reconciliation
            # from un-pausing a workflow that the grooming gate has
            # deferred. Three race windows are covered by this guard +
            # the pause-clause's existing ``current != 'PAUSED'`` check:
            # (1) pre-PATCH (defer committed PAUSED but Forgejo still
            # shows the label) — guard makes resume skip; (2) post-PATCH
            # (label removed) — pause clause sees ``current == 'PAUSED'``
            # and the existing fall-through "Stays paused" handles it;
            # (3) re-pickup (operator cleared deferred_reason AND re-
            # added the label) — guard's NULL check satisfied; resume
            # fires. Without this guard, window (1) would un-pause
            # before defer's HTTP work completed.
            resume_to = getattr(row, "pre_pause_state", None) or "DISCOVERED"
            _apply_transition_with_pre_pause(
                session,
                workflow_id=row.workflow_id,
                from_state=current,
                to_state=resume_to,
                now=now,
                reason="opt-in-label-restored",
                pre_pause_state=None,  # clear it now that we've resumed.
            )
            return ReconciliationAction(
                workflow_id=row.workflow_id,
                kind=kind,
                entity_number=entity,
                from_state=current,
                to_state=resume_to,
                reason="opt-in-label-restored",
            )
        if current == "PAUSED" and not label_present:
            # Stays paused; nothing to do.
            return ReconciliationAction(
                workflow_id=row.workflow_id,
                kind=kind,
                entity_number=entity,
                from_state=current,
                to_state=None,
                reason="paused",
            )

    if target_state is None or target_state == current:
        return ReconciliationAction(
            workflow_id=row.workflow_id,
            kind=kind,
            entity_number=entity,
            from_state=current,
            to_state=None,
            reason=reason or "consistent",
        )

    # Apply the transition.
    _apply_transition(
        session,
        workflow_id=row.workflow_id,
        from_state=current,
        to_state=target_state,
        now=now,
        reason=reason,
    )
    return ReconciliationAction(
        workflow_id=row.workflow_id,
        kind=kind,
        entity_number=entity,
        from_state=current,
        to_state=target_state,
        reason=reason,
    )


def _decide_transition(
    forgejo: dict | None,
    *,
    kind: str,
) -> tuple[str | None, str]:
    """Map (Forgejo state, kind) → (new controller state | None, reason).

    Returns None for the target if the controller's state is already
    consistent.
    """
    if forgejo is None:
        return ("STUCK", "pr-not-found-on-forgejo")
    if not isinstance(forgejo, dict):
        return (None, "invalid-forgejo-shape")

    if kind == "pr":
        if forgejo.get("merged") is True:
            return ("MERGED", "externally-merged")
        state = forgejo.get("state")
        if state == "closed":
            return ("ABANDONED", "externally-closed-not-merged")
        # state in {"open", None} → consistent
        return (None, "consistent")

    if kind == "issue":
        state = forgejo.get("state")
        if state == "closed":
            return ("ABANDONED", "issue-closed-externally")
        return (None, "consistent")

    return (None, f"unsupported-kind:{kind}")


# Phase 1k++ refinement (R9): per-reason event_type so operators can
# filter ``controller_events`` by a single column instead of
# JSON-payload LIKE queries (which are dialect-specific). Each reason
# string used by reconciliation maps to a distinct event_type.
_REASON_TO_EVENT_TYPE: dict[str, str] = {
    "opt-in-label-removed": "label-pause",
    "opt-in-label-restored": "label-resume",
    "externally-merged": "external-merge",
    "externally-closed-not-merged": "external-close",
    "issue-closed-externally": "external-issue-close",
    "pr-not-found-on-forgejo": "external-pr-deleted",
}


def _event_type_for(reason: str) -> str:
    """Derive a distinct event_type from the reason string. Falls back
    to ``'reconciliation'`` for unknown reasons so a future contributor
    adding a new reason without updating the map still emits a
    well-formed row (just one operators can't filter by sub-kind)."""
    return _REASON_TO_EVENT_TYPE.get(reason, "reconciliation")


def _apply_transition_with_pre_pause(
    session,
    *,
    workflow_id: int,
    from_state: str,
    to_state: str,
    now: datetime,
    reason: str,
    pre_pause_state: str | None,
) -> None:
    """Like ``_apply_transition`` but also writes ``pre_pause_state``
    on the workflow row. Used by PAUSE / RESUME paths so the resume
    target survives across master restarts."""
    session.execute(
        text(
            "UPDATE workflows SET "
            "  current_state = :to_state, "
            "  last_transition_at = :now, "
            "  entered_state_at = :now, "
            "  pre_pause_state = :pre_pause_state "
            "WHERE workflow_id = :wf_id"
        ),
        {
            "to_state": to_state,
            "now": now,
            "wf_id": workflow_id,
            "pre_pause_state": pre_pause_state,
        },
    )
    session.execute(
        text(
            "INSERT INTO controller_events "
            "(workflow_id, ts, event_type, from_state, to_state, "
            " payload, forgejo_write_pending, replay_attempts) "
            "VALUES (:wf_id, :ts, :event_type, :from_state, :to_state, "
            "        :payload, 0, 0)"
        ),
        {
            "wf_id": workflow_id,
            "ts": now,
            "event_type": _event_type_for(reason),
            "from_state": from_state,
            "to_state": to_state,
            "payload": json.dumps({"reason": reason, "source": "reconciliation"}),
        },
    )


def _apply_transition(
    session,
    *,
    workflow_id: int,
    from_state: str,
    to_state: str,
    now: datetime,
    reason: str,
) -> None:
    """Apply a reconciliation-driven transition.

    R-1 + R-2 fix (2026-05-19): two safety guards.

    1. Skip if the workflow has an ``in_progress`` attempt — a worker
       is mid-session against a state that's about to change underneath
       it. Letting the worker finish (or get reaped) is safer than
       silently transitioning while it's still working. The worker's
       output will either land valid (no-op for already-advanced
       workflow) or be rejected by the lock-loss check.

    2. ``WHERE current_state = :from_state`` guard on the UPDATE,
       matching the TOCTOU defense ci_status_poll already uses. If
       another tick advanced the workflow between the row SELECT and
       this UPDATE, we skip the redundant transition + the event row.
    """
    # R-1: skip if there's an active worker on this workflow.
    in_progress = session.execute(
        text(
            "SELECT 1 FROM workflow_attempts "
            "WHERE workflow_id = :wf_id AND status = 'in_progress' "
            "LIMIT 1"
        ),
        {"wf_id": workflow_id},
    ).first()
    if in_progress is not None:
        logger.info(
            "reconciliation: skipping %s → %s for workflow %s: "
            "active worker attempt exists; let it finish or get reaped",
            from_state,
            to_state,
            workflow_id,
        )
        return
    # R-2: only transition if current_state still matches from_state
    # (concurrent ci_status_poll / tick may have already advanced).
    result = session.execute(
        text(
            "UPDATE workflows SET "
            "  current_state = :to_state, "
            "  last_transition_at = :now, "
            "  entered_state_at = :now "
            "WHERE workflow_id = :wf_id "
            "  AND current_state = :from_state"
        ),
        {
            "to_state": to_state,
            "now": now,
            "wf_id": workflow_id,
            "from_state": from_state,
        },
    )
    if (result.rowcount or 0) == 0:
        logger.info(
            "reconciliation: workflow %s no longer in %s "
            "(race lost to another tick); skipping event row",
            workflow_id,
            from_state,
        )
        return
    session.execute(
        text(
            "INSERT INTO controller_events "
            "(workflow_id, ts, event_type, from_state, to_state, "
            " payload, forgejo_write_pending, replay_attempts) "
            "VALUES (:wf_id, :ts, :event_type, :from_state, :to_state, "
            "        :payload, 0, 0)"
        ),
        {
            "wf_id": workflow_id,
            "ts": now,
            "event_type": _event_type_for(reason),
            "from_state": from_state,
            "to_state": to_state,
            "payload": json.dumps({"reason": reason, "source": "reconciliation"}),
        },
    )


__all__ = [
    "GetIssueStateCallback",
    "GetPRStateCallback",
    "ReconciliationAction",
    "ReconciliationReport",
    "run_reconciliation_tick",
]