freemo
d3cc7d30d7
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 19s
CI / security (pull_request) Successful in 34s
CI / typecheck (pull_request) Successful in 36s
CI / unit_tests (pull_request) Successful in 3m45s
CI / docker (pull_request) Successful in 40s
CI / integration_tests (pull_request) Successful in 4m34s
CI / coverage (pull_request) Successful in 4m39s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 17s
CI / security (push) Successful in 33s
CI / typecheck (push) Successful in 34s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m21s
CI / integration_tests (push) Successful in 3m9s
CI / docker (push) Successful in 1m0s
CI / coverage (push) Successful in 4m37s
CI / benchmark-publish (push) Has been cancelled
CI / benchmark-regression (pull_request) Successful in 30m5s
ToolRegistryRepository.create(), .update(), and .delete() called session.flush() but never session.commit(). The CLI factory creates a raw sessionmaker without a UnitOfWork wrapper, so the transaction was never committed and SQLAlchemy performed an implicit rollback when the session was garbage-collected. The same bug existed in ValidationAttachmentRepository.attach() and .detach(). Changes: - Add session.commit() after session.flush() in all five mutating methods across ToolRegistryRepository and ValidationAttachmentRepository. - Add finally: session.close() to guarantee session cleanup regardless of success or failure. - Update class docstrings to reflect the new commit-on-write semantics. - Add Behave BDD feature (tool_add_persist.feature) with scenarios for single-tool round-trip, multi-tool persistence, and duplicate rejection, using file-based SQLite to reproduce the cross-session issue. - Add Robot Framework integration test (tool_add_persist.robot) with add-then-list and fresh-list-empty scenarios. - Add ASV benchmark (tool_add_persist_bench.py) with track_list_after_add_count metric. Key decisions: - File-based SQLite (not in-memory) is used in tests because the bug only manifests when the session/engine is fully disposed between add and list, simulating separate CLI invocations. - Step patterns are prefixed with "tool-persist" to avoid AmbiguousStep collisions with existing tool_registry_steps.py. - The commit-in-repository approach was chosen over adding a UnitOfWork to the CLI factory because the CLI commands are simple CRUD operations that should auto-persist without requiring callers to remember to commit. ISSUES CLOSED: #621
99 lines
3.2 KiB
Python
99 lines
3.2 KiB
Python
"""ASV benchmarks for tool-add persistence (issue #621).
|
|
|
|
Measures the round-trip: create a tool ➜ list from a fresh session.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
# Force-reload so ASV picks up the source tree version.
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from sqlalchemy import create_engine # noqa: E402
|
|
from sqlalchemy.engine import Engine # noqa: E402
|
|
from sqlalchemy.orm import Session, sessionmaker # noqa: E402
|
|
|
|
from cleveragents.infrastructure.database.models import Base # noqa: E402
|
|
from cleveragents.infrastructure.database.repositories import ( # noqa: E402
|
|
ToolRegistryRepository,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_tool(name: str) -> dict[str, Any]:
|
|
now_iso: str = datetime.now().isoformat()
|
|
ns: str = name.split("/", 1)[0] if "/" in name else ""
|
|
short: str = name.split("/", 1)[1] if "/" in name else name
|
|
return {
|
|
"name": name,
|
|
"namespace": ns,
|
|
"short_name": short,
|
|
"description": f"Bench tool {name}",
|
|
"tool_type": "tool",
|
|
"source": "custom",
|
|
"timeout": 300,
|
|
"created_at": now_iso,
|
|
"updated_at": now_iso,
|
|
"resource_bindings": [],
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Benchmark suite
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class ToolAddPersistSuite:
|
|
"""Benchmark the add-then-list persistence round-trip."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
fd, self._db_path = tempfile.mkstemp(suffix=".db", prefix="tool_bench_")
|
|
os.close(fd)
|
|
engine: Engine = create_engine(f"sqlite:///{self._db_path}", echo=False)
|
|
Base.metadata.create_all(engine)
|
|
engine.dispose()
|
|
|
|
def teardown(self) -> None:
|
|
if os.path.exists(self._db_path):
|
|
os.unlink(self._db_path)
|
|
|
|
def _repo(self) -> ToolRegistryRepository:
|
|
engine: Engine = create_engine(f"sqlite:///{self._db_path}", echo=False)
|
|
factory: sessionmaker[Session] = sessionmaker(
|
|
bind=engine, expire_on_commit=False
|
|
)
|
|
return ToolRegistryRepository(session_factory=factory)
|
|
|
|
# -- tracking benchmark ------------------------------------------------
|
|
|
|
def track_list_after_add_count(self) -> int:
|
|
"""Returns the count of tools after a single add (should be 1)."""
|
|
repo: ToolRegistryRepository = self._repo()
|
|
repo.create(_make_tool("bench/persist-check"))
|
|
# Open a new repo (fresh engine) to prove persistence
|
|
repo2: ToolRegistryRepository = self._repo()
|
|
tools: list[Any] = repo2.list_all()
|
|
return len(tools)
|
|
|
|
track_list_after_add_count.unit = "tools" # type: ignore[attr-defined]
|