brent.edwards
122af46305
- Fix project_repository_steps session mismatch: use shared session for repos so context.pr_session.commit() commits flushed data - Fix cli_format_bench setup/teardown to accept fmt parameter for parameterized ASV benchmarks - Fix plan_model_bench PlanPhase.APPLIED -> PlanPhase.APPLY - Fix benchmark idempotency: use batch counters to generate unique IDs across repeated ASV iterations (plan_phase_migration_bench, project_migration_bench, resource_registry_migration_bench) - Fix uow_lifecycle_bench FK constraint by reusing pre-seeded action - Fix plan_lifecycle_persistence_bench to create fresh plan per iteration
141 lines
4.0 KiB
Python
141 lines
4.0 KiB
Python
"""ASV benchmarks for plan lifecycle persistence overhead.
|
|
|
|
Measures the cost of persisting actions and plans through
|
|
PlanLifecycleService with a UnitOfWork backing store compared
|
|
to pure in-memory operation.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from sqlalchemy import create_engine, event
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from cleveragents.application.services.plan_lifecycle_service import (
|
|
PlanLifecycleService,
|
|
)
|
|
from cleveragents.config.settings import Settings
|
|
from cleveragents.infrastructure.database.models import Base
|
|
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
|
|
|
|
_CB32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
|
_bench_counter = 7000
|
|
|
|
|
|
def _bench_ulid() -> str:
|
|
global _bench_counter
|
|
_bench_counter += 1
|
|
n = _bench_counter
|
|
suffix = ""
|
|
for _ in range(8):
|
|
suffix = _CB32[n % 32] + suffix
|
|
n //= 32
|
|
return f"01HGZ6FE0AQDYTR4BX{suffix}"
|
|
|
|
|
|
def _build_uow() -> tuple[UnitOfWork, Any]:
|
|
engine = create_engine(
|
|
"sqlite:///:memory:",
|
|
echo=False,
|
|
future=True,
|
|
connect_args={"check_same_thread": False},
|
|
)
|
|
|
|
@event.listens_for(engine, "connect")
|
|
def _fk(dbapi_conn: Any, _rec: Any) -> None:
|
|
cursor = dbapi_conn.cursor()
|
|
cursor.execute("PRAGMA foreign_keys=ON")
|
|
cursor.close()
|
|
|
|
Base.metadata.create_all(engine)
|
|
sf: sessionmaker[Session] = sessionmaker(
|
|
bind=engine,
|
|
expire_on_commit=False,
|
|
autoflush=False,
|
|
autocommit=False,
|
|
class_=Session,
|
|
)
|
|
|
|
uow = UnitOfWork.__new__(UnitOfWork)
|
|
uow.database_url = "sqlite:///:memory:"
|
|
uow._engine = engine
|
|
uow._session_factory = sf
|
|
uow._database_initialized = True
|
|
uow._prompt_for_migration = None
|
|
|
|
return uow, engine
|
|
|
|
|
|
class TimeCreateActionPersisted:
|
|
"""Benchmark creating an action with persistence."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
self._counter = 0
|
|
self.uow, _ = _build_uow()
|
|
self.settings = Settings()
|
|
self.svc = PlanLifecycleService(settings=self.settings, unit_of_work=self.uow)
|
|
|
|
def time_create_action(self) -> None:
|
|
self._counter += 1
|
|
self.svc.create_action(
|
|
name=f"local/bench-act-{self._counter}",
|
|
description="Bench action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
|
|
class TimeCreatePlanPersisted:
|
|
"""Benchmark creating a plan with persistence."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
self._counter = 0
|
|
self.uow, _ = _build_uow()
|
|
self.settings = Settings()
|
|
self.svc = PlanLifecycleService(settings=self.settings, unit_of_work=self.uow)
|
|
self.svc.create_action(
|
|
name="local/bench-plan-action",
|
|
description="Bench action for plans",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
def time_create_plan(self) -> None:
|
|
self.svc.use_action(action_name="local/bench-plan-action")
|
|
|
|
|
|
class TimePhaseTransitionPersisted:
|
|
"""Benchmark a full phase transition cycle with persistence."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
self._counter = 0
|
|
self.uow, _ = _build_uow()
|
|
self.settings = Settings()
|
|
self.svc = PlanLifecycleService(settings=self.settings, unit_of_work=self.uow)
|
|
self.svc.create_action(
|
|
name="local/bench-transition-action",
|
|
description="Bench action for transitions",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
def time_execute_transition(self) -> None:
|
|
self._counter += 1
|
|
plan = self.svc.use_action(
|
|
action_name="local/bench-transition-action",
|
|
)
|
|
plan_id = plan.identity.plan_id
|
|
self.svc.start_strategize(plan_id)
|
|
self.svc.complete_strategize(plan_id)
|
|
self.svc.execute_plan(plan_id)
|