cleveragents-core/tools/controller/master/backfill.py

"""Master startup backfill — discover all open PRs/issues at deploy time.

Per plan v9: when the master starts (first deploy OR after a long
outage), it needs to learn about existing open PRs/issues that
weren't created via discovery-tick-during-uptime. Without this,
existing PRs would have to wait for the next discovery tick AND
the operator would have no visibility into "what's the controller
about to take ownership of."

This module is a thin wrapper around ``run_discovery``: it runs the
same sweep, then records a one-time ``controller_events`` marker
("backfill complete") so subsequent restarts can skip the operator-
visible logging.

For v1 simplicity, backfill = "run discovery once, log loudly." The
idempotent skip in run_discovery prevents duplicate workflow rows.
"""
from __future__ import annotations

import json
import logging
from dataclasses import dataclass
from datetime import datetime, timezone

from sqlalchemy import text
from sqlalchemy.engine import Engine

from ..db.session import session_scope
from .discovery import (
    DiscoveryReport,
    ListIssuesCallback,
    ListPRsCallback,
    run_discovery,
)

logger = logging.getLogger(__name__)


# Magic event_type the marker uses. Looking for "is there a row with
# this event_type for this (owner, repo)?" tells us whether backfill
# has been completed.
BACKFILL_MARKER_EVENT_TYPE = "controller-backfill-complete"


@dataclass
class BackfillReport:
    """Per-backfill summary. Exposes the inner DiscoveryReport plus
    whether this was the first-ever backfill for this (owner, repo)."""

    first_time: bool
    discovery: DiscoveryReport


def has_backfill_run(engine: Engine, *, owner: str, repo: str) -> bool:
    """Check the controller_events table for the backfill marker.

    The marker is a single ``event_type='controller-backfill-complete'``
    row whose payload contains the (owner, repo). Idempotent: every
    backfill emits one such marker; the existence check determines
    whether the backfill is the first time vs. a re-run.
    """
    with session_scope(engine) as session:
        rows = session.execute(
            text(
                "SELECT e.payload "
                "FROM controller_events e "
                "WHERE e.event_type = :ev_type"
            ),
            {"ev_type": BACKFILL_MARKER_EVENT_TYPE},
        ).all()
        for r in rows:
            if not r.payload:
                continue
            try:
                payload = json.loads(r.payload)
            except (TypeError, ValueError, json.JSONDecodeError):
                continue
            if (
                isinstance(payload, dict)
                and payload.get("owner") == owner
                and payload.get("repo") == repo
            ):
                return True
    return False


def run_startup_backfill(
    engine: Engine,
    *,
    owner: str,
    repo: str,
    list_prs: ListPRsCallback,
    list_issues: ListIssuesCallback | None = None,
    opt_in_label: str | None = None,
    require_opt_in_label: bool = False,
) -> BackfillReport:
    """One backfill sweep at master startup.

    Calls ``run_discovery`` to populate DISCOVERED workflows for any
    open PRs/issues the controller hasn't seen yet. Records a
    backfill-complete marker in ``controller_events`` so subsequent
    restarts know "this isn't the first time."

    Always returns; the marker row is created regardless of how many
    new workflows landed (so a second startup correctly reports
    ``first_time=False`` even if discovery found zero new entities).

    Phase 1k: backfill honors the same opt-in label gate as discovery.
    Defaults to ``CONTROLLER_OPT_IN_LABEL`` env (or ``controller-managed``).
    """
    first = not has_backfill_run(engine, owner=owner, repo=repo)
    if first:
        logger.info(
            "controller backfill: first-time for %s/%s — sweeping "
            "all open PRs + issues",
            owner, repo,
        )
    else:
        logger.info(
            "controller backfill: re-run for %s/%s — skipping if "
            "no new entities",
            owner, repo,
        )

    discovery_report = run_discovery(
        engine, owner=owner, repo=repo,
        list_prs=list_prs, list_issues=list_issues,
        opt_in_label=opt_in_label,
        require_opt_in_label=require_opt_in_label,
    )

    _emit_backfill_marker(
        engine, owner=owner, repo=repo,
        first_time=first, discovery_report=discovery_report,
    )

    if first or discovery_report.new_workflows:
        logger.info(
            "controller backfill done: PRs=%d issues=%d new_workflows=%d "
            "skipped=%d first_time=%s",
            discovery_report.prs_seen, discovery_report.issues_seen,
            discovery_report.new_workflows,
            discovery_report.existing_skipped, first,
        )
    return BackfillReport(first_time=first, discovery=discovery_report)


def _emit_backfill_marker(
    engine: Engine, *, owner: str, repo: str,
    first_time: bool, discovery_report: DiscoveryReport,
) -> None:
    """Insert the backfill-complete marker row.

    The marker is associated with the FIRST new workflow created
    (if any) so it has a valid ``workflow_id`` FK. If no new workflows
    landed but a workflow exists for this (owner, repo), use that.
    If no workflows exist at all (truly empty Forgejo), the marker is
    skipped — the next backfill will revisit.
    """
    now = datetime.now(timezone.utc)
    with session_scope(engine) as session:
        # Pick a workflow_id to associate the marker with.
        if discovery_report.new_entities:
            wf_id = discovery_report.new_entities[0].workflow_id
        else:
            existing = session.execute(
                text(
                    "SELECT workflow_id FROM workflows "
                    "WHERE owner = :owner AND repo = :repo LIMIT 1"
                ),
                {"owner": owner, "repo": repo},
            ).first()
            if existing is None:
                logger.info(
                    "backfill marker NOT recorded for %s/%s: no workflows "
                    "exist to attach the marker FK to (empty Forgejo). "
                    "Next backfill will retry.",
                    owner, repo,
                )
                return
            wf_id = existing.workflow_id

        session.execute(
            text(
                "INSERT INTO controller_events "
                "(workflow_id, ts, event_type, payload, "
                " forgejo_write_pending, replay_attempts) "
                "VALUES (:wf_id, :ts, :ev_type, :payload, 0, 0)"
            ),
            {
                "wf_id": wf_id, "ts": now,
                "ev_type": BACKFILL_MARKER_EVENT_TYPE,
                "payload": json.dumps({
                    "owner": owner, "repo": repo,
                    "first_time": first_time,
                    "prs_seen": discovery_report.prs_seen,
                    "issues_seen": discovery_report.issues_seen,
                    "new_workflows": discovery_report.new_workflows,
                }),
            },
        )


__all__ = [
    "BACKFILL_MARKER_EVENT_TYPE",
    "BackfillReport",
    "has_backfill_run",
    "run_startup_backfill",
]