157 lines
4.3 KiB
Python
157 lines
4.3 KiB
Python
"""ASV benchmarks for Session CLI command parsing overhead.
|
|
|
|
Measures the performance of:
|
|
- Session create command parsing
|
|
- Session list command rendering
|
|
- Session show command rendering
|
|
- Session tell command parsing
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from typer.testing import CliRunner # noqa: E402
|
|
from ulid import ULID # noqa: E402
|
|
|
|
from cleveragents.cli.commands import session as session_mod # noqa: E402
|
|
from cleveragents.cli.commands.session import app as session_app # noqa: E402
|
|
from cleveragents.domain.models.core.session import ( # noqa: E402
|
|
MessageRole,
|
|
Session,
|
|
SessionMessage,
|
|
SessionTokenUsage,
|
|
)
|
|
|
|
_runner = CliRunner()
|
|
|
|
|
|
def _mock_session(
|
|
session_id: str | None = None,
|
|
actor_name: str | None = None,
|
|
messages: list[SessionMessage] | None = None,
|
|
) -> Session:
|
|
return Session(
|
|
session_id=session_id or str(ULID()),
|
|
actor_name=actor_name,
|
|
namespace="local",
|
|
messages=messages or [],
|
|
token_usage=SessionTokenUsage(),
|
|
created_at=datetime.now(),
|
|
updated_at=datetime.now(),
|
|
)
|
|
|
|
|
|
def _mock_message(
|
|
role: MessageRole = MessageRole.USER,
|
|
content: str = "Hello",
|
|
sequence: int = 0,
|
|
) -> SessionMessage:
|
|
return SessionMessage(
|
|
message_id=str(ULID()),
|
|
role=role,
|
|
content=content,
|
|
sequence=sequence,
|
|
timestamp=datetime.now(),
|
|
)
|
|
|
|
|
|
class SessionCLICreateSuite:
|
|
"""Benchmark session create command throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = MagicMock()
|
|
self._svc.create.return_value = _mock_session()
|
|
session_mod._service = self._svc
|
|
|
|
def teardown(self) -> None:
|
|
session_mod._service = None
|
|
|
|
def time_create_default(self) -> None:
|
|
"""Benchmark create with defaults."""
|
|
_runner.invoke(session_app, ["create"])
|
|
|
|
def time_create_with_actor(self) -> None:
|
|
"""Benchmark create with actor flag."""
|
|
_runner.invoke(session_app, ["create", "--actor", "openai/gpt-4"])
|
|
|
|
|
|
class SessionCLIListSuite:
|
|
"""Benchmark session list command throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = MagicMock()
|
|
self._svc.list.return_value = [
|
|
_mock_session(actor_name=f"openai/gpt-{i}") for i in range(50)
|
|
]
|
|
session_mod._service = self._svc
|
|
|
|
def teardown(self) -> None:
|
|
session_mod._service = None
|
|
|
|
def time_list_all(self) -> None:
|
|
"""Benchmark listing all sessions."""
|
|
_runner.invoke(session_app, ["list"])
|
|
|
|
def time_list_json(self) -> None:
|
|
"""Benchmark listing with JSON format."""
|
|
_runner.invoke(session_app, ["list", "--format", "json"])
|
|
|
|
|
|
class SessionCLIShowSuite:
|
|
"""Benchmark session show command throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._sid = str(ULID())
|
|
self._svc = MagicMock()
|
|
self._svc.get.return_value = _mock_session(
|
|
session_id=self._sid,
|
|
messages=[
|
|
_mock_message(MessageRole.USER, "Hello", 0),
|
|
_mock_message(MessageRole.ASSISTANT, "Hi", 1),
|
|
],
|
|
)
|
|
session_mod._service = self._svc
|
|
|
|
def teardown(self) -> None:
|
|
session_mod._service = None
|
|
|
|
def time_show(self) -> None:
|
|
"""Benchmark show command."""
|
|
_runner.invoke(session_app, ["show", self._sid])
|
|
|
|
|
|
class SessionCLITellSuite:
|
|
"""Benchmark session tell command throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._sid = str(ULID())
|
|
self._svc = MagicMock()
|
|
self._svc.append_message.side_effect = lambda **kw: _mock_message(
|
|
role=kw.get("role", MessageRole.USER),
|
|
content=kw.get("content", "msg"),
|
|
sequence=0,
|
|
)
|
|
session_mod._service = self._svc
|
|
|
|
def teardown(self) -> None:
|
|
session_mod._service = None
|
|
|
|
def time_tell(self) -> None:
|
|
"""Benchmark tell command."""
|
|
_runner.invoke(session_app, ["tell", "--session", self._sid, "Hello world"])
|