cleveragents-core/tools/controller/db/session.py

"""DB engine + session helpers.

The controller talks to SQLite (tests + local dev) or Postgres
(multi-machine production deploy). Engine selection is via
``CLEVERAGENTS_DB_URL`` env var; no URL → in-memory SQLite (test
default).

Connection lifetime: tests + master use short-lived sessions per
operation via ``session_scope`` (transactional + auto-rollback on
exception). Worker controllers (Phase 1c) hold longer-lived sessions
for the dequeue + heartbeat loop; that pattern lives in worker.py.
"""

from __future__ import annotations

import os
from collections.abc import Iterator
from contextlib import contextmanager

from sqlalchemy import Engine, create_engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool

from .models import Base


DEFAULT_DB_URL = "sqlite:///:memory:"


def _resolve_db_url(db_url: str | None = None) -> str:
    """Resolve the DB URL with env-var fallback."""
    if db_url:
        return db_url
    return os.environ.get("CLEVERAGENTS_DB_URL", DEFAULT_DB_URL)


def build_engine(db_url: str | None = None) -> Engine:
    """Create a SQLAlchemy engine. Sets per-dialect tuning:

    - SQLite: enable foreign keys, WAL mode, busy_timeout. ``echo``
      defaults to False; flip via ``CLEVERAGENTS_DB_ECHO=1``.
    - Postgres: pool_pre_ping=True so dead connections are noticed
      before a tick fires against a stale conn.
    """
    url = _resolve_db_url(db_url)
    echo = os.environ.get("CLEVERAGENTS_DB_ECHO", "0").lower() in {"1", "true", "yes"}
    if url.startswith("sqlite"):
        # SQLite-specific tuning: same-thread restriction off (the
        # controller's worker pool uses threads) + reasonable busy
        # timeout. ``check_same_thread=False`` is safe because each
        # thread acquires its own Session from sessionmaker; SQLAlchemy
        # serializes per-connection work.
        #
        # For in-memory SQLite (``:memory:``), StaticPool routes every
        # connection to the same underlying DB so the heartbeat thread
        # + the runner's write + the dequeue all see the same data.
        # Without it, ``:memory:`` gives each connection its own
        # DB — the test harness sees ghost rows.
        is_in_memory = ":memory:" in url
        connect_args: dict[str, object] = {
            "check_same_thread": False,
            "timeout": 30,
        }
        engine_kwargs: dict[str, object] = {"echo": echo, "connect_args": connect_args}
        if is_in_memory:
            engine_kwargs["poolclass"] = StaticPool
        engine = create_engine(url, **engine_kwargs)
        # WAL + foreign keys + reasonable defaults on every new
        # connection. The ``connect`` event fires per-connection.
        from sqlalchemy import event

        @event.listens_for(engine, "connect")
        def _sqlite_pragmas(conn, _record):
            cur = conn.cursor()
            cur.execute("PRAGMA foreign_keys = ON")
            cur.execute("PRAGMA journal_mode = WAL")
            cur.execute("PRAGMA synchronous = NORMAL")
            cur.execute("PRAGMA busy_timeout = 5000")
            cur.close()

        # Note on transaction isolation: SQLite defaults to
        # BEGIN DEFERRED, which means concurrent dequeue queries
        # (``UPDATE … WHERE attempt_id = (SELECT … LIMIT 1)``) can
        # race — two workers may both pick the inner SELECT before
        # either UPDATE commits, and the loser hits SQLITE_BUSY (and
        # waits up to ``busy_timeout``). This is **acceptable for
        # single-host dev** (where SQLite is the only option) but is
        # the reason MULTI-MACHINE deployments MUST use Postgres
        # (which has ``FOR UPDATE SKIP LOCKED`` — see db/dequeue.py
        # for the dual-dialect path). The RUNBOOK calls this out
        # under "Multiple machines, but one machine takes all the
        # work."
        return engine
    # Postgres + others
    return create_engine(url, echo=echo, pool_pre_ping=True)


def create_all(engine: Engine | None = None) -> None:
    """Create all controller tables on ``engine`` and apply additive
    column migrations. Idempotent — safe to call on every controller
    startup; existing tables are not re-created and existing columns
    are not re-added.

    v1 ships without Alembic; ``create_all`` handles fresh DBs and
    ``apply_additive_migrations`` covers column additions on live
    DBs. When the schema becomes complex enough that this pattern
    breaks (rename / backfill / non-NULL with default), graduate to
    Alembic and remove the migrations module.
    """
    if engine is None:
        engine = build_engine()
    Base.metadata.create_all(engine)
    from .migrations import apply_additive_migrations

    apply_additive_migrations(engine)


@contextmanager
def session_scope(engine: Engine | None = None) -> Iterator[Session]:
    """Transactional session context.

    Usage:
        with session_scope(engine) as session:
            session.add(workflow)
            # auto-commits on clean exit; rolls back on exception.

    Each ``with`` block gets a fresh session. SQLAlchemy's session
    factory handles connection pooling.
    """
    if engine is None:
        engine = build_engine()
    factory = sessionmaker(engine, expire_on_commit=False)
    session = factory()
    try:
        yield session
        session.commit()
    except Exception:
        session.rollback()
        raise
    finally:
        session.close()