feat(server): implement PostgreSQL storage backend for server mode #1118

Merged
HAL9000 merged 6 commits from feature/m7-postgresql-backend into master 2026-05-29 09:00:23 +00:00
15 changed files with 1097 additions and 28 deletions
+30
View File
@@ -0,0 +1,30 @@
# docker-compose.yml — Local PostgreSQL for CleverAgents server-mode development
#
# Usage:
# docker compose up -d # start PostgreSQL
# docker compose down -v # tear down and remove data volume
#
# Connect from CleverAgents:
# export CLEVERAGENTS_SERVER_MODE=true
# export CLEVERAGENTS_DATABASE_URL=postgresql://cleveragents:cleveragents@localhost:5432/cleveragents
services:
postgres:
image: postgres:16-alpine
environment:
POSTGRES_USER: cleveragents
# WARNING: Development credentials only — do not use in production
POSTGRES_PASSWORD: cleveragents
POSTGRES_DB: cleveragents
ports:
- "5432:5432"
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U cleveragents"]
interval: 5s
timeout: 5s
retries: 5
volumes:
pgdata:
+111
View File
@@ -0,0 +1,111 @@
@phase1 @database @postgresql @server_mode
Feature: PostgreSQL Storage Backend for Server Mode
As a platform operator
I want CleverAgents to support PostgreSQL as its storage backend
So that server mode can handle multi-user collaborative access
Background:
Given I have imported the database infrastructure modules
# ── Settings & URL Selection ──────────────────────────────────────
@settings @database_url
Scenario: Default database URL is SQLite in local mode
Given I create settings with server_mode disabled
Then the resolved database URL should start with "sqlite"
@settings @database_url
Scenario: Server mode with default URL raises ValueError
Given I create settings with server_mode enabled
And the database_url is the default SQLite value
Then resolving the database URL should raise a ValueError
@settings @database_url
Scenario: Server mode with explicit PostgreSQL URL uses that URL
Given I create settings with server_mode enabled
And the database_url is "postgresql://myhost/mydb"
Then the resolved database URL should be "postgresql://myhost/mydb"
@settings @database_url
Scenario: is_postgresql returns True for PostgreSQL URLs
Given I create settings with server_mode enabled
And the database_url is "postgresql://localhost/cleveragents"
Then is_postgresql should return True
@settings @database_url
Scenario: is_postgresql returns False for SQLite URLs
Given I create settings with server_mode disabled
Then is_postgresql should return False
@settings @database_url
Scenario: is_postgresql returns False when server mode raises ValueError for SQLite URL
Given I create settings with server_mode enabled
And the database_url is the default SQLite value
Then is_postgresql should return False
# ── Pool Configuration ────────────────────────────────────────────
@settings @pool
Scenario: Default pool settings are appropriate for multi-user
Given I create settings with server_mode enabled
Then the db_pool_size should be 5
And the db_max_overflow should be 10
And the db_pool_recycle should be 1800
@settings @pool
Scenario: Pool settings can be overridden via environment
Given I create settings with custom pool configuration
| setting | value |
| db_pool_size | 20 |
| db_max_overflow | 30 |
| db_pool_recycle | 900 |
Then the db_pool_size should be 20
And the db_max_overflow should be 30
And the db_pool_recycle should be 900
# ── UnitOfWork Engine Creation ────────────────────────────────────
@engine @sqlite
Scenario: UnitOfWork creates SQLite engine for local mode
Given I create a UnitOfWork with URL "sqlite:///:memory:"
When I access the engine
Then the engine dialect should be "sqlite"
@engine @pool
Scenario: UnitOfWork passes pool parameters to non-SQLite engines
Given I create a UnitOfWork with URL "sqlite:///:memory:" and pool_size 8
Then the UnitOfWork pool_size should be 8
And the UnitOfWork max_overflow should be 10
And the UnitOfWork pool_recycle should be 1800
# ── ORM Model Dialect Compatibility ───────────────────────────────
@models @dialect
Scenario: ORM models use only dialect-agnostic column types
Given I inspect all ORM models from the Base metadata
Then no column should use a SQLite-specific type
And all columns should use standard SQLAlchemy types
@models @dialect
Scenario: ORM models define no raw SQL or sqlite3 imports
Given I read the models module source code
Then the source should not contain "sqlite3"
And the source should not contain "PRAGMA"
And the source should not contain "raw_sql"
@models @dialect
Scenario: All table names are valid PostgreSQL identifiers
Given I inspect all ORM models from the Base metadata
Then all table names should be valid SQL identifiers
# ── Migration Runner ──────────────────────────────────────────────
@migration
Scenario: Migration runner handles SQLite URLs
Given I create a MigrationRunner with URL "sqlite:///:memory:"
Then the migration runner database_url should start with "sqlite"
@migration
Scenario: Migration runner handles PostgreSQL URLs
Given I create a MigrationRunner with URL "postgresql://localhost/test"
Then the migration runner database_url should start with "postgresql"
+369
View File
@@ -0,0 +1,369 @@
"""Step definitions for PostgreSQL storage backend BDD scenarios.
Steps specific to the ``postgresql_backend.feature`` file. The shared
``I have imported the database infrastructure modules`` step lives in
``database_infrastructure_steps.py`` and is reused here via Behave's
step-sharing mechanism.
"""
from __future__ import annotations
import inspect
import re
from behave import given, then, when
from behave.runner import Context
# ---------------------------------------------------------------------------
# Settings helpers
# ---------------------------------------------------------------------------
@given("I create settings with server_mode disabled")
def step_settings_local_mode(context: Context) -> None:
"""Create settings with local (SQLite) mode.
Clears ``CLEVERAGENTS_SERVER_MODE`` and restores the default
``database_url`` so Settings resolves to SQLite.
"""
import os
from cleveragents.config.settings import Settings
saved_db_url = os.environ.pop("CLEVERAGENTS_DATABASE_URL", None)
os.environ.pop("CLEVERAGENTS_SERVER_MODE", None)
try:
context.settings = Settings()
finally:
if saved_db_url is not None:
os.environ["CLEVERAGENTS_DATABASE_URL"] = saved_db_url
@given("I create settings with server_mode enabled")
def step_settings_server_mode(context: Context) -> None:
"""Create settings with server mode enabled.
The ``before_scenario`` hook injects a per-scenario SQLite URL into
``CLEVERAGENTS_DATABASE_URL``. We must clear it so that Settings
falls back to its declared default (``sqlite:///cleveragents.db``),
allowing ``resolve_database_url`` to detect the default and switch to
PostgreSQL.
"""
import os
from cleveragents.config.settings import Settings
saved_db_url = os.environ.pop("CLEVERAGENTS_DATABASE_URL", None)
os.environ["CLEVERAGENTS_SERVER_MODE"] = "true"
try:
context.settings = Settings()
finally:
os.environ.pop("CLEVERAGENTS_SERVER_MODE", None)
if saved_db_url is not None:
os.environ["CLEVERAGENTS_DATABASE_URL"] = saved_db_url
@given("the database_url is the default SQLite value")
def step_database_url_is_default(context: Context) -> None:
"""Ensure database_url is the SQLite default.
The default is already ``sqlite:///cleveragents.db`` so this is a
no-op guard ensuring the previous step didn't override it.
"""
assert context.settings.database_url.startswith("sqlite")
@given('the database_url is "{url}"')
def step_database_url_is(context: Context, url: str) -> None:
"""Recreate settings with an explicit database_url."""
import os
from cleveragents.config.settings import Settings
os.environ["CLEVERAGENTS_DATABASE_URL"] = url
server = "true" if context.settings.server_mode else "false"
os.environ["CLEVERAGENTS_SERVER_MODE"] = server
try:
context.settings = Settings()
finally:
os.environ.pop("CLEVERAGENTS_DATABASE_URL", None)
os.environ.pop("CLEVERAGENTS_SERVER_MODE", None)
@given("I create settings with custom pool configuration")
def step_settings_custom_pool(context: Context) -> None:
"""Create settings with custom pool values from the step table.
Pydantic-settings reads from environment variables, so we set the
``CLEVERAGENTS_*`` env vars before constructing the Settings instance
and clean them up afterward. The ``before_scenario`` per-scenario
database URL is temporarily cleared so the default applies.
"""
import os
from cleveragents.config.settings import Settings
env_map: dict[str, str] = {
"db_pool_size": "CLEVERAGENTS_DB_POOL_SIZE",
"db_max_overflow": "CLEVERAGENTS_DB_MAX_OVERFLOW",
"db_pool_recycle": "CLEVERAGENTS_DB_POOL_RECYCLE",
}
set_vars: list[str] = []
saved_db_url = os.environ.pop("CLEVERAGENTS_DATABASE_URL", None)
os.environ["CLEVERAGENTS_SERVER_MODE"] = "true"
set_vars.append("CLEVERAGENTS_SERVER_MODE")
if context.table is not None:
for row in context.table:
key = row["setting"]
value = row["value"]
env_name = env_map.get(key, f"CLEVERAGENTS_{key.upper()}")
os.environ[env_name] = str(value)
set_vars.append(env_name)
try:
context.settings = Settings()
finally:
for var in set_vars:
os.environ.pop(var, None)
if saved_db_url is not None:
os.environ["CLEVERAGENTS_DATABASE_URL"] = saved_db_url
# ---------------------------------------------------------------------------
# Settings assertions
# ---------------------------------------------------------------------------
@then("resolving the database URL should raise a ValueError")
def step_resolve_url_raises_value_error(context: Context) -> None:
"""Assert that resolve_database_url raises ValueError in server mode."""
try:
context.settings.resolve_database_url()
except ValueError:
pass # expected
else:
raise AssertionError("Expected ValueError from resolve_database_url()")
@then('the resolved database URL should start with "{prefix}"')
def step_resolved_url_starts_with(context: Context, prefix: str) -> None:
"""Assert the resolved database URL prefix."""
url = context.settings.resolve_database_url()
assert url.startswith(prefix), f"Expected URL to start with '{prefix}', got '{url}'"
@then('the resolved database URL should be "{expected}"')
def step_resolved_url_equals(context: Context, expected: str) -> None:
"""Assert the resolved database URL matches exactly."""
url = context.settings.resolve_database_url()
assert url == expected, f"Expected '{expected}', got '{url}'"
@then("is_postgresql should return True")
def step_is_postgresql_true(context: Context) -> None:
"""Assert is_postgresql returns True."""
assert context.settings.is_postgresql(), "Expected is_postgresql() to be True"
@then("is_postgresql should return False")
def step_is_postgresql_false(context: Context) -> None:
"""Assert is_postgresql returns False."""
assert not context.settings.is_postgresql(), "Expected is_postgresql() to be False"
@then("the db_pool_size should be {value:d}")
def step_pool_size_equals(context: Context, value: int) -> None:
"""Assert db_pool_size setting."""
assert context.settings.db_pool_size == value, (
f"Expected {value}, got {context.settings.db_pool_size}"
)
@then("the db_max_overflow should be {value:d}")
def step_max_overflow_equals(context: Context, value: int) -> None:
"""Assert db_max_overflow setting."""
assert context.settings.db_max_overflow == value, (
f"Expected {value}, got {context.settings.db_max_overflow}"
)
@then("the db_pool_recycle should be {value:d}")
def step_pool_recycle_equals(context: Context, value: int) -> None:
"""Assert db_pool_recycle setting."""
assert context.settings.db_pool_recycle == value, (
f"Expected {value}, got {context.settings.db_pool_recycle}"
)
# ---------------------------------------------------------------------------
# UnitOfWork engine
# ---------------------------------------------------------------------------
_VALID_SQL_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$")
@given('I create a UnitOfWork with URL "{url}"')
def step_create_uow(context: Context, url: str) -> None:
"""Create a UnitOfWork instance with the given URL."""
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
context.uow = UnitOfWork(
url,
prompt_for_migration=lambda _msg: True,
)
@given('I create a UnitOfWork with URL "{url}" and pool_size {size:d}')
def step_create_uow_with_pool(context: Context, url: str, size: int) -> None:
"""Create a UnitOfWork with explicit pool_size."""
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
context.uow = UnitOfWork(
url,
prompt_for_migration=lambda _msg: True,
pool_size=size,
)
@when("I access the engine")
def step_access_engine(context: Context) -> None:
"""Trigger engine creation."""
context.engine = context.uow.engine
@then('the engine dialect should be "{dialect}"')
def step_engine_dialect(context: Context, dialect: str) -> None:
"""Assert the engine dialect name."""
actual = context.engine.dialect.name
assert actual == dialect, f"Expected dialect '{dialect}', got '{actual}'"
@then("the UnitOfWork pool_size should be {value:d}")
def step_uow_pool_size(context: Context, value: int) -> None:
"""Assert UnitOfWork.pool_size via read-only property."""
assert context.uow.pool_size == value
@then("the UnitOfWork max_overflow should be {value:d}")
def step_uow_max_overflow(context: Context, value: int) -> None:
"""Assert UnitOfWork.max_overflow via read-only property."""
assert context.uow.max_overflow == value
@then("the UnitOfWork pool_recycle should be {value:d}")
def step_uow_pool_recycle(context: Context, value: int) -> None:
"""Assert UnitOfWork.pool_recycle via read-only property."""
assert context.uow.pool_recycle == value
# ---------------------------------------------------------------------------
# ORM Model dialect-agnostic checks
# ---------------------------------------------------------------------------
# Standard SQLAlchemy types that are portable across all dialects.
_PORTABLE_TYPE_NAMES = frozenset(
{
"VARCHAR",
"STRING",
"TEXT",
"INTEGER",
"BIGINTEGER",
"SMALLINTEGER",
"FLOAT",
"NUMERIC",
"BOOLEAN",
"DATE",
"DATETIME",
"TIME",
"TIMESTAMP",
"JSON",
"JSONB",
"BLOB",
"LARGEBINARY",
"ENUM",
"UUID",
"INTERVAL",
"ARRAY",
}
)
@given("I inspect all ORM models from the Base metadata")
def step_inspect_orm_models(context: Context) -> None:
"""Collect all tables from the ORM metadata."""
from cleveragents.infrastructure.database.models import Base
context.metadata = Base.metadata
context.tables = list(Base.metadata.tables.values())
@then("no column should use a SQLite-specific type")
def step_no_sqlite_specific_types(context: Context) -> None:
"""Assert no column uses a SQLite-only type."""
for table in context.tables:
for col in table.columns:
type_name = type(col.type).__name__.upper()
# Known SQLite-only types that are NOT portable
sqlite_only = {"SQLITEINTEGER", "SQLITETEXT", "SQLITEBLOB"}
assert type_name not in sqlite_only, (
f"Column {table.name}.{col.name} uses SQLite-specific type {type_name}"
)
@then("all columns should use standard SQLAlchemy types")
def step_all_standard_types(context: Context) -> None:
"""Assert all columns use portable SQLAlchemy types."""
for table in context.tables:
for col in table.columns:
type_name = type(col.type).__name__.upper()
# Allow common base types and their variants
assert type_name in _PORTABLE_TYPE_NAMES or any(
base in type_name for base in _PORTABLE_TYPE_NAMES
), f"Column {table.name}.{col.name} uses non-standard type {type_name}"
@then("all table names should be valid SQL identifiers")
def step_valid_table_names(context: Context) -> None:
"""Assert all table names are valid (lowercase) SQL identifiers."""
for table in context.tables:
assert _VALID_SQL_IDENTIFIER.match(table.name), (
f"Table name '{table.name}' is not a valid SQL identifier"
)
@given("I read the models module source code")
def step_read_models_source(context: Context) -> None:
"""Read the models.py source for textual analysis."""
from cleveragents.infrastructure.database import models
context.models_source = inspect.getsource(models)
@then('the source should not contain "{forbidden}"')
def step_source_not_contain(context: Context, forbidden: str) -> None:
"""Assert the source code does not contain a forbidden string."""
assert forbidden not in context.models_source, (
f"models.py should not contain '{forbidden}'"
)
# ---------------------------------------------------------------------------
# Migration runner
# ---------------------------------------------------------------------------
@given('I create a MigrationRunner with URL "{url}"')
def step_create_migration_runner(context: Context, url: str) -> None:
"""Create a MigrationRunner instance."""
from cleveragents.infrastructure.database.migration_runner import MigrationRunner
context.migration_runner = MigrationRunner(url)
@then('the migration runner database_url should start with "{prefix}"')
def step_runner_url_starts_with(context: Context, prefix: str) -> None:
"""Assert migration runner database_url prefix."""
url = context.migration_runner.database_url
assert url.startswith(prefix), f"Expected '{prefix}', got '{url}'"
+184 -20
View File
@@ -53,20 +53,31 @@ def step_uow_non_sqlite_url(context: Context) -> None:
@when("I access the engine property with mocked create_engine")
def step_access_engine_property_mocked(context: Context) -> None:
"""Access the engine property, mocking _ensure_database_initialized and create_engine."""
import sys as _sys
from sqlalchemy.engine import Engine
mock_engine = MagicMock(spec=Engine)
with (
patch.object(context.uow, "_ensure_database_initialized"),
patch(
"cleveragents.infrastructure.database.unit_of_work.create_engine",
return_value=mock_engine,
) as mock_create,
):
engine = context.uow.engine
context.result_engine = engine
context.mock_create_engine = mock_create
# Ensure psycopg2 can be imported (it may not be installed in test env)
_needs_psycopg2_stub = "psycopg2" not in _sys.modules
if _needs_psycopg2_stub:
_sys.modules["psycopg2"] = MagicMock()
try:
with (
patch.object(context.uow, "_ensure_database_initialized"),
patch(
"cleveragents.infrastructure.database.unit_of_work.create_engine",
return_value=mock_engine,
) as mock_create,
):
engine = context.uow.engine
context.result_engine = engine
context.mock_create_engine = mock_create
finally:
if _needs_psycopg2_stub:
_sys.modules.pop("psycopg2", None)
@then("the engine should be created via the non-SQLite code path")
@@ -232,20 +243,31 @@ def step_uow_postgresql_url(context: Context) -> None:
@when("I trigger engine creation with mocked dependencies")
def step_trigger_engine_creation_mocked(context: Context) -> None:
"""Access engine property with mocked create_engine and migration runner."""
import sys as _sys
from sqlalchemy.engine import Engine
mock_engine = MagicMock(spec=Engine)
with (
patch.object(context.uow, "_ensure_database_initialized"),
patch(
"cleveragents.infrastructure.database.unit_of_work.create_engine",
return_value=mock_engine,
) as mock_create,
):
engine = context.uow.engine
context.result_engine = engine
context.mock_create_engine = mock_create
# Ensure psycopg2 can be imported (it may not be installed in test env)
_needs_psycopg2_stub = "psycopg2" not in _sys.modules
if _needs_psycopg2_stub:
_sys.modules["psycopg2"] = MagicMock()
try:
with (
patch.object(context.uow, "_ensure_database_initialized"),
patch(
"cleveragents.infrastructure.database.unit_of_work.create_engine",
return_value=mock_engine,
) as mock_create,
):
engine = context.uow.engine
context.result_engine = engine
context.mock_create_engine = mock_create
finally:
if _needs_psycopg2_stub:
_sys.modules.pop("psycopg2", None)
@then("create_engine should have been called without isolation_level")
@@ -299,3 +321,145 @@ def step_second_engine_same_as_first(context: Context) -> None:
if hasattr(context, "saved_memory_engines"):
MEMORY_ENGINES.clear()
MEMORY_ENGINES.update(context.saved_memory_engines)
# =========================================================================
# Argument validation in __init__ (lines 68-75)
# =========================================================================
@when("I try to create a UnitOfWork with an empty database_url")
def step_uow_empty_database_url(context: Context) -> None:
"""Try to create UnitOfWork with empty database_url; capture ValueError."""
try:
UnitOfWork(database_url="")
context.last_error = None
except ValueError as exc:
context.last_error = exc
@when("I try to create a UnitOfWork with pool_size set to {value:d}")
def step_uow_invalid_pool_size(context: Context, value: int) -> None:
"""Try to create UnitOfWork with invalid pool_size; capture ValueError."""
try:
UnitOfWork(database_url="sqlite:///test.db", pool_size=value)
context.last_error = None
except ValueError as exc:
context.last_error = exc
@when("I try to create a UnitOfWork with max_overflow set to {value:d}")
def step_uow_invalid_max_overflow(context: Context, value: int) -> None:
"""Try to create UnitOfWork with invalid max_overflow; capture ValueError."""
try:
UnitOfWork(database_url="sqlite:///test.db", max_overflow=value)
context.last_error = None
except ValueError as exc:
context.last_error = exc
@when("I try to create a UnitOfWork with pool_recycle set to {value:d}")
def step_uow_invalid_pool_recycle(context: Context, value: int) -> None:
"""Try to create UnitOfWork with invalid pool_recycle; capture ValueError."""
try:
UnitOfWork(database_url="sqlite:///test.db", pool_recycle=value)
context.last_error = None
except ValueError as exc:
context.last_error = exc
# =========================================================================
# dispose() method (lines 196-205)
# =========================================================================
@given("a UnitOfWork with a mocked non-memory engine")
def step_uow_mocked_non_memory_engine(context: Context) -> None:
"""Create a UnitOfWork with a pre-set mocked engine to test dispose()."""
from sqlalchemy.engine import Engine
context.uow = UnitOfWork(
database_url="postgresql://user:pass@localhost:5432/testdb"
)
context.mock_engine = MagicMock(spec=Engine)
context.uow._engine = context.mock_engine
@when("I call dispose on the UnitOfWork")
def step_call_dispose_on_uow(context: Context) -> None:
"""Call dispose() on the UnitOfWork stored in context."""
context.uow.dispose()
@then("the internal engine should have been disposed and cleared")
def step_engine_disposed_and_cleared(context: Context) -> None:
"""Verify dispose() called engine.dispose() and cleared internal references."""
context.mock_engine.dispose.assert_called_once()
assert context.uow._engine is None, (
f"Expected _engine to be None after dispose(), got {context.uow._engine}"
)
assert context.uow._session_factory is None, (
"Expected _session_factory to be None after dispose()"
)
@then("the dispose call should complete without error")
def step_dispose_no_error(context: Context) -> None:
"""Verify dispose() completes without raising (False branch — engine is None)."""
assert context.uow._engine is None, (
"Expected _engine to remain None when dispose() is a no-op"
)
# =========================================================================
# Context manager protocol (__enter__ / __exit__) (lines 207-213)
# =========================================================================
@when("I use the UnitOfWork as a context manager")
def step_use_uow_as_context_manager(context: Context) -> None:
"""Use the UnitOfWork with a with-statement to exercise __enter__/__exit__."""
with context.uow as entered_uow:
context.entered_uow = entered_uow
@then("the engine should be disposed on context manager exit")
def step_engine_disposed_on_context_exit(context: Context) -> None:
"""Verify __enter__ returned self and __exit__ triggered dispose()."""
assert context.entered_uow is context.uow, "__enter__ should return self"
context.mock_engine.dispose.assert_called_once()
# =========================================================================
# psycopg2 ImportError hint (lines 142-148)
# =========================================================================
@when("I access the engine with psycopg2 forcibly unavailable")
def step_access_engine_no_psycopg2(context: Context) -> None:
"""Access engine property while psycopg2 is blocked via sys.modules."""
import sys as _sys
try:
with (
patch.dict(_sys.modules, {"psycopg2": None}),
patch.object(context.uow, "_ensure_database_initialized"),
):
_ = context.uow.engine
context.raised_exception = None
except ImportError as exc:
context.raised_exception = exc
@then("an ImportError should be raised mentioning the server extra")
def step_import_error_server_extra(context: Context) -> None:
"""Verify ImportError was raised containing the install-hint message."""
assert context.raised_exception is not None, (
"Expected an ImportError to be raised but no exception was captured"
)
assert isinstance(context.raised_exception, ImportError), (
f"Expected ImportError, got {type(context.raised_exception)}"
)
assert "server" in str(context.raised_exception).lower(), (
f"Expected 'server' in exception message: '{context.raised_exception}'"
)
+60
View File
@@ -74,3 +74,63 @@ Feature: UnitOfWork coverage boost for uncovered code paths
And a first UnitOfWork has already created an in-memory engine
When a second UnitOfWork accesses its engine property
Then the second engine should be the same object as the first
# ---------------------------------------------------------------------------
# Argument validation in __init__ (ValueError raises)
# ---------------------------------------------------------------------------
@uow_validate_empty_url
Scenario: UnitOfWork rejects an empty database_url
When I try to create a UnitOfWork with an empty database_url
Then a ValueError should be raised containing "database_url must be a non-empty string"
@uow_validate_pool_size
Scenario: UnitOfWork rejects pool_size less than 1
When I try to create a UnitOfWork with pool_size set to 0
Then a ValueError should be raised containing "pool_size must be >= 1"
@uow_validate_max_overflow
Scenario: UnitOfWork rejects negative max_overflow
When I try to create a UnitOfWork with max_overflow set to -1
Then a ValueError should be raised containing "max_overflow must be >= 0"
@uow_validate_pool_recycle
Scenario: UnitOfWork rejects pool_recycle below -1
When I try to create a UnitOfWork with pool_recycle set to -2
Then a ValueError should be raised containing "pool_recycle must be >= -1"
# ---------------------------------------------------------------------------
# dispose() method branches (lines 196-205)
# ---------------------------------------------------------------------------
@uow_dispose_active_engine
Scenario: UnitOfWork dispose clears a non-memory engine
Given a UnitOfWork with a mocked non-memory engine
When I call dispose on the UnitOfWork
Then the internal engine should have been disposed and cleared
@uow_dispose_no_engine
Scenario: UnitOfWork dispose is a no-op when engine has not been created
Given a UnitOfWork configured with a non-SQLite database URL
When I call dispose on the UnitOfWork
Then the dispose call should complete without error
# ---------------------------------------------------------------------------
# Context manager protocol (__enter__ / __exit__) (lines 207-213)
# ---------------------------------------------------------------------------
@uow_context_manager_protocol
Scenario: UnitOfWork context manager disposes engine on exit
Given a UnitOfWork with a mocked non-memory engine
When I use the UnitOfWork as a context manager
Then the engine should be disposed on context manager exit
# ---------------------------------------------------------------------------
# psycopg2 ImportError hint (lines 142-148)
# ---------------------------------------------------------------------------
@uow_psycopg2_missing
Scenario: UnitOfWork raises ImportError with install hint when psycopg2 is unavailable
Given a UnitOfWork configured with a non-SQLite database URL
When I access the engine with psycopg2 forcibly unavailable
Then an ImportError should be raised mentioning the server extra
+3
View File
@@ -52,6 +52,9 @@ dependencies = [
]
[project.optional-dependencies]
server = [
"psycopg2-binary>=2.9.9", # PostgreSQL adapter for server-mode storage
]
tui = [
"textual>=1.0.0,<2.0.0",
]
+166
View File
@@ -0,0 +1,166 @@
*** Settings ***
Documentation Integration tests for PostgreSQL storage backend abstraction layer.
... Tests the dialect-agnostic ORM models, connection configuration,
... and database URL selection logic using SQLite as the test backend.
... Tests tagged @requires_postgresql are excluded from default CI runs.
Library OperatingSystem
Library String
Library Collections
Library Process
Library indentation_library.py
Resource ${CURDIR}/common.resource
*** Variables ***
${PYTHON} python
*** Test Cases ***
Settings Resolve Database URL Local Mode
[Documentation] Verify resolve_database_url returns SQLite in local mode
${script}= Catenate SEPARATOR=\n
... import os
... os.environ.pop('CLEVERAGENTS_SERVER_MODE', None)
... os.environ.pop('CLEVERAGENTS_DATABASE_URL', None)
... from cleveragents.config.settings import Settings
... s = Settings()
... print(s.resolve_database_url())
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Start With ${result.stdout.strip()} sqlite
Settings Resolve Database URL Server Mode
[Documentation] Verify resolve_database_url returns PostgreSQL when server_mode is on
... and CLEVERAGENTS_DATABASE_URL points to a PostgreSQL instance.
${script}= Catenate SEPARATOR=\n
... import os
... os.environ['CLEVERAGENTS_SERVER_MODE'] = 'true'
... os.environ['CLEVERAGENTS_DATABASE_URL'] = 'postgresql://localhost/cleveragents'
... from cleveragents.config.settings import Settings
... s = Settings()
... print(s.resolve_database_url())
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Start With ${result.stdout.strip()} postgresql
Settings Is PostgreSQL Returns True For PG URL
[Documentation] Verify is_postgresql detects PostgreSQL URLs
${script}= Catenate SEPARATOR=\n
... import os
... os.environ['CLEVERAGENTS_SERVER_MODE'] = 'true'
... os.environ['CLEVERAGENTS_DATABASE_URL'] = 'postgresql://localhost/db'
... from cleveragents.config.settings import Settings
... s = Settings()
... print(s.is_postgresql())
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Be Equal As Strings ${result.stdout.strip()} True
Settings Is PostgreSQL Returns False For SQLite URL
[Documentation] Verify is_postgresql returns False for SQLite URLs
${script}= Catenate SEPARATOR=\n
... import os
... os.environ.pop('CLEVERAGENTS_SERVER_MODE', None)
... os.environ.pop('CLEVERAGENTS_DATABASE_URL', None)
... from cleveragents.config.settings import Settings
... s = Settings()
... print(s.is_postgresql())
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Be Equal As Strings ${result.stdout.strip()} False
UnitOfWork Creates SQLite Engine With Pool Parameters
[Documentation] UnitOfWork stores pool parameters and creates an SQLite engine
${result}= Run Process ${PYTHON} -c
... from cleveragents.infrastructure.database.unit_of_work import UnitOfWork; u \= UnitOfWork('sqlite:///:memory:', prompt_for_migration\=lambda m: True, pool_size\=8, max_overflow\=15, pool_recycle\=900); print(u._pool_size, u._max_overflow, u._pool_recycle, u.engine.dialect.name)
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout.strip()} 8 15 900 sqlite
ORM Models Use Dialect-Agnostic Types
[Documentation] All ORM column types are portable across SQLite and PostgreSQL
${script}= Catenate SEPARATOR=\n
... from cleveragents.infrastructure.database.models import Base
... PORTABLE = {'VARCHAR','STRING','TEXT','INTEGER','BIGINTEGER','SMALLINTEGER',
... 'FLOAT','NUMERIC','BOOLEAN','DATE','DATETIME','TIME','TIMESTAMP','JSON',
... 'BLOB','LARGEBINARY','ENUM'}
... for t in Base.metadata.tables.values():
... ${SPACE}${SPACE}${SPACE}${SPACE}for c in t.columns:
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}tn = type(c.type).__name__.upper()
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}if tn not in PORTABLE and not any(b in tn for b in PORTABLE):
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise AssertionError(f'{t.name}.{c.name} has non-portable type {tn}')
... print('ALL_PORTABLE')
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout.strip()} ALL_PORTABLE
ORM Models Source Contains No SQLite Specific Code
[Documentation] models.py should not import sqlite3 or use PRAGMAs
${script}= Catenate SEPARATOR=\n
... import inspect
... from cleveragents.infrastructure.database import models
... src = inspect.getsource(models)
... for forbidden in ['sqlite3', 'PRAGMA']:
... ${SPACE}${SPACE}${SPACE}${SPACE}assert forbidden not in src, f'Found {forbidden}'
... print('CLEAN')
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout.strip()} CLEAN
Table Names Are Valid PostgreSQL Identifiers
[Documentation] All table names are lowercase SQL identifiers
${script}= Catenate SEPARATOR=\n
... import re
... from cleveragents.infrastructure.database.models import Base
... pat = re.compile(r'^[a-z_][a-z0-9_]*$')
... for t in Base.metadata.tables.values():
... ${SPACE}${SPACE}${SPACE}${SPACE}assert pat.match(t.name), f'Invalid table: {t.name}'
... print('VALID')
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout.strip()} VALID
Docker Compose File Exists For PostgreSQL Development
[Documentation] docker-compose.yml exists at project root
File Should Exist ${CURDIR}/../docker-compose.yml
Pool Defaults Are Multi-User Appropriate
[Documentation] Default pool settings: pool_size=5, max_overflow=10, pool_recycle=1800
${script}= Catenate SEPARATOR=\n
... import os
... os.environ['CLEVERAGENTS_SERVER_MODE'] = 'true'
... os.environ.pop('CLEVERAGENTS_DATABASE_URL', None)
... from cleveragents.config.settings import Settings
... s = Settings()
... print(s.db_pool_size, s.db_max_overflow, s.db_pool_recycle)
${result}= Run Process ${PYTHON} -c ${script}
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Be Equal As Strings ${result.stdout.strip()} 5 10 1800
Migration Runner Accepts PostgreSQL URL
[Documentation] MigrationRunner can be constructed with a PostgreSQL URL
${result}= Run Process ${PYTHON} -c
... from cleveragents.infrastructure.database.migration_runner import MigrationRunner; r \= MigrationRunner('postgresql://localhost/test'); print(r.database_url)
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Start With ${result.stdout.strip()} postgresql
PostgreSQL Live Connection Test
[Documentation] Connect to a real PostgreSQL instance and run migrations.
... Requires a running PostgreSQL server and CLEVERAGENTS_DATABASE_URL
... pointing to it. Excluded from default CI runs.
[Tags] requires_postgresql
${pg_url}= Get Environment Variable CLEVERAGENTS_DATABASE_URL default=SKIP
Skip If '${pg_url}' == 'SKIP' PostgreSQL not available
${result}= Run Process ${PYTHON} -c
... from cleveragents.infrastructure.database.migration_runner import MigrationRunner; r \= MigrationRunner('${pg_url}'); r.init_or_upgrade(); print('OK')
... cwd=${CURDIR}/..
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout.strip()} OK
@@ -251,6 +251,9 @@ def get_database_url() -> str:
relative to ``CLEVERAGENTS_HOME`` (if set) rather than CWD, ensuring
test isolation and correct data placement.
.. todo:: (#878) Use ``Settings.resolve_database_url()`` once
PostgreSQL wiring is complete so that *server_mode* is honoured.
Returns:
SQLAlchemy database URL with absolute path for SQLite
"""
+1
View File
@@ -37,6 +37,7 @@ def _get_runner() -> MigrationRunner:
from cleveragents.application.container import get_database_url
from cleveragents.infrastructure.database.migration_runner import MigrationRunner
# TODO(#878): Use Settings.resolve_database_url() once PostgreSQL wiring is complete
return MigrationRunner(get_database_url())
+1
View File
@@ -126,6 +126,7 @@ def _store_project_extras(
from cleveragents.application.container import get_database_url
# TODO(#878): Use Settings.resolve_database_url() once PostgreSQL wiring is complete
db_url = get_database_url()
engine = create_engine(db_url, echo=False)
session = sessionmaker(bind=engine, expire_on_commit=False)()
+2
View File
@@ -413,6 +413,8 @@ def _check_stale_locks() -> dict[str, Any]:
try:
from sqlalchemy.orm import Session, sessionmaker
# TODO(#878): Use Settings.resolve_database_url()
# once PostgreSQL wiring is complete.
db_url = get_database_url()
engine = create_engine(db_url, echo=False)
+68
View File
@@ -386,6 +386,38 @@ class Settings(BaseSettings):
validation_alias=AliasChoices("CLEVERAGENTS_TEST_DATABASE_URL"),
)
# Server mode toggle — when True, the application operates as a
# collaborative multi-user server, which selects a PostgreSQL-compatible
# database URL by default instead of SQLite.
server_mode: bool = Field(
default=False,
validation_alias=AliasChoices("CLEVERAGENTS_SERVER_MODE"),
description="Enable server mode for multi-user collaborative access.",
)
# PostgreSQL connection-pool tuning (server mode)
db_pool_size: int = Field(
default=5,
ge=1,
validation_alias=AliasChoices("CLEVERAGENTS_DB_POOL_SIZE"),
description="SQLAlchemy connection-pool size for PostgreSQL.",
)
db_max_overflow: int = Field(
default=10,
ge=0,
validation_alias=AliasChoices("CLEVERAGENTS_DB_MAX_OVERFLOW"),
description="Extra connections above pool_size allowed under load.",
)
db_pool_recycle: int = Field(
default=1800,
ge=-1,
validation_alias=AliasChoices("CLEVERAGENTS_DB_POOL_RECYCLE"),
description=(
"Seconds after which a pooled connection is recycled. "
"-1 disables recycling."
),
)
# Context tier budgets (ACMS #208)
context_max_tokens_hot: int = Field(
default=32000,
@@ -893,6 +925,42 @@ class Settings(BaseSettings):
return self._derive_test_database_url(self.database_url)
return self.database_url
def resolve_database_url(self, *, test: bool = False) -> str:
"""Return the database URL, honouring *server_mode*.
In **server mode** the database URL must point to a PostgreSQL
instance. If the caller has not provided an explicit PostgreSQL
URL via ``CLEVERAGENTS_DATABASE_URL``, a ``ValueError`` is
raised to prevent silent fallback to a possibly unreachable
default.
In **local mode** the existing SQLite URL is returned unchanged.
Args:
test: When True, derive or return the test database URL.
Returns:
A SQLAlchemy-compatible database URL string.
Raises:
ValueError: When *server_mode* is True but no explicit
PostgreSQL URL has been configured.
"""
base_url = self.get_database_url(test=test)
if self.server_mode and base_url.startswith("sqlite"):
raise ValueError(
"server_mode=True requires an explicit database URL. "
"Set CLEVERAGENTS_DATABASE_URL to a PostgreSQL connection string."
)
return base_url
def is_postgresql(self) -> bool:
"""Return True when the active database URL targets PostgreSQL."""
try:
return self.resolve_database_url().startswith("postgresql")
except ValueError:
return False
def is_production(self) -> bool:
"""Whether the runtime matches production settings."""
return (
@@ -167,9 +167,12 @@ class MigrationRunner:
)
else:
engine = create_engine(self.database_url)
with engine.connect() as connection:
context = MigrationContext.configure(connection)
return context.get_current_revision()
try:
with engine.connect() as connection:
context = MigrationContext.configure(connection)
return context.get_current_revision()
finally:
engine.dispose()
def get_pending_migrations(self) -> list[str]:
"""Get list of pending migrations.
@@ -241,7 +244,7 @@ class MigrationRunner:
)
# For SQLite, ensure the parent directory exists
# Skip this for in-memory databases (:memory:)
# Skip this for in-memory databases (:memory:) and non-SQLite backends
if (
self.database_url.startswith("sqlite")
and ":memory:" not in self.database_url
@@ -255,8 +258,10 @@ class MigrationRunner:
db_file = Path(db_path)
db_file.parent.mkdir(parents=True, exist_ok=True)
# Create or retrieve engine with proper settings for SQLite
# For in-memory databases, use cached engines to ensure schema persists
# Create or retrieve engine with proper settings.
# SQLite uses cached engines for in-memory databases; PostgreSQL
# (and other non-SQLite backends) use a simple disposable engine
# since the migration runner only needs a short-lived connection.
if self.database_url.startswith("sqlite"):
if self.database_url == "sqlite:///:memory:":
# For in-memory databases, reuse cached engine to ensure
@@ -276,6 +281,7 @@ class MigrationRunner:
)
should_dispose = True
else:
# PostgreSQL / other RDBMS — no SQLite-specific connect_args
engine = create_engine(self.database_url)
should_dispose = True
@@ -77,6 +77,7 @@ def run_migrations_online() -> None:
In this scenario we need to create an Engine
and associate a connection with the context.
Supports both SQLite (local mode) and PostgreSQL (server mode).
"""
if config is None:
msg = "Config is required for online migrations"
@@ -94,14 +95,22 @@ def run_migrations_online() -> None:
)
# We created the engine, so we should manage the connection lifecycle
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
else:
# Connection was provided externally, use it but don't close it
# (the caller is responsible for cleanup)
context.configure(connection=connectable, target_metadata=target_metadata)
context.configure(
connection=connectable,
target_metadata=target_metadata,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
@@ -43,6 +43,13 @@ class UnitOfWork:
database_url: str,
prompt_for_migration: Callable[[str], bool] | None = None,
require_confirmation: bool = True,
# Pool defaults are intentionally duplicated from Settings.db_pool_*
Outdated
Review

[VALIDATION] Missing fail-fast argument validation. Per CONTRIBUTING.md, all public methods must validate arguments as the first step. database_url should be checked for empty/None, and pool parameters should be range-checked. This class is explicitly designed to be usable without a Settings instance (per the comment above), so it cannot rely on Pydantic validation upstream.

**[VALIDATION]** Missing fail-fast argument validation. Per CONTRIBUTING.md, all public methods must validate arguments as the first step. `database_url` should be checked for empty/None, and pool parameters should be range-checked. This class is explicitly designed to be usable without a Settings instance (per the comment above), so it cannot rely on Pydantic validation upstream.
# so that UnitOfWork remains usable without a Settings instance
# (e.g. in tests or standalone scripts). When constructed via the
# DI container the caller should forward Settings values explicitly.
pool_size: int = 5,
max_overflow: int = 10,
pool_recycle: int = 1800,
) -> None:
"""Initialize Unit of Work with database connection.
@@ -53,13 +60,44 @@ class UnitOfWork:
require_confirmation: When False, migrations are applied without
prompting the user. Set to False when the caller has
already obtained blanket approval (e.g. ``--yes`` flag).
pool_size: Connection-pool size for PostgreSQL backends.
max_overflow: Extra connections above *pool_size* under load.
pool_recycle: Seconds before a pooled connection is recycled.
"""
# Validate arguments per CONTRIBUTING.md fail-fast principles
if not database_url:
raise ValueError("database_url must be a non-empty string")
if pool_size < 1:
raise ValueError(f"pool_size must be >= 1, got {pool_size}")
if max_overflow < 0:
raise ValueError(f"max_overflow must be >= 0, got {max_overflow}")
if pool_recycle < -1:
raise ValueError(f"pool_recycle must be >= -1, got {pool_recycle}")
self.database_url = database_url
self._engine: Engine | None = None
self._session_factory: sessionmaker[Session] | None = None
self._database_initialized = False
self._prompt_for_migration = prompt_for_migration
self._require_confirmation = require_confirmation
self._pool_size = pool_size
self._max_overflow = max_overflow
self._pool_recycle = pool_recycle
@property
def pool_size(self) -> int:
"""Read-only access to the configured connection-pool size."""
return self._pool_size
@property
def max_overflow(self) -> int:
"""Read-only access to the configured max overflow."""
return self._max_overflow
@property
def pool_recycle(self) -> int:
"""Read-only access to the configured pool recycle interval."""
return self._pool_recycle
@property
def engine(self) -> Engine:
1
@@ -93,10 +131,29 @@ class UnitOfWork:
connect_args={"check_same_thread": False},
)
else:
# PostgreSQL (or other non-SQLite): use connection pooling
# suitable for multi-user server mode.
# Use READ COMMITTED isolation level for concurrent multi-user access.
# This allows non-repeatable reads and phantom reads, which is
# acceptable for the concurrent plan execution use case where
# individual plan steps are atomic operations. For stronger
# consistency, callers can use explicit locking or application-
# level coordination.
try:
import psycopg2 # noqa: F401
except ImportError as exc:
raise ImportError(
"PostgreSQL support requires the 'server' extra: "
"pip install cleveragents[server]"
) from exc
self._engine = create_engine(
self.database_url,
echo=False, # Set to True for SQL debugging
future=True, # Use SQLAlchemy 2.0 style
pool_size=self._pool_size,
max_overflow=self._max_overflow,
pool_recycle=self._pool_recycle,
pool_pre_ping=True,
)
return self._engine
@@ -136,6 +193,25 @@ class UnitOfWork:
finally:
session.close()
def dispose(self) -> None:
"""Release all database connections and dispose the engine.
Must be called when the UnitOfWork is no longer needed, especially
for PostgreSQL backends where the connection pool holds TCP connections.
"""
if self._engine is not None and self.database_url != "sqlite:///:memory:":
self._engine.dispose()
self._engine = None
self._session_factory = None
def __enter__(self) -> UnitOfWork:
"""Enter context manager."""
return self
def __exit__(self, *args: object) -> None:
"""Exit context manager and dispose resources."""
self.dispose()
def _ensure_database_initialized(self, force: bool = False) -> None:
"""Run migrations if needed before database access."""
if self._database_initialized and not force: