"""ASV benchmarks for Automation Profile CLI command throughput. Measures the performance of: - Add profile from YAML config - List profiles rendering - Show profile rendering - Remove profile """ from __future__ import annotations import importlib import os import sys import tempfile from pathlib import Path # Ensure the local *source* tree is importable even when ASV has an # older build of the package installed. _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) import cleveragents # noqa: E402 importlib.reload(cleveragents) from typer.testing import CliRunner # noqa: E402 from cleveragents.application.services.automation_profile_service import ( # noqa: E402 AutomationProfileService, ) from cleveragents.cli.commands.automation_profile import ( # noqa: E402 app as profile_app, ) from cleveragents.domain.models.core.automation_profile import ( # noqa: E402 AutomationProfile, ) class _InMemoryProfileRepository: """In-memory repository satisfying the duck-type contract. Provides the four methods expected by ``AutomationProfileService``: ``get_by_name``, ``list_all``, ``upsert``, and ``delete``. """ def __init__(self) -> None: self._store: dict[str, AutomationProfile] = {} def get_by_name(self, name: str) -> AutomationProfile | None: return self._store.get(name) def list_all(self) -> list[AutomationProfile]: return list(self._store.values()) def upsert(self, profile: AutomationProfile) -> None: self._store[profile.name] = profile def delete(self, name: str) -> None: self._store.pop(name, None) _VALID_YAML = """\ name: bench/test-profile description: Benchmark test profile schema_version: "1.0" decompose_task: 0.5 create_tool: 0.5 select_tool: 1.0 edit_code: 0.5 execute_command: 0.5 create_file: 0.5 delete_content: 0.5 access_network: 0.5 install_dependency: 0.5 modify_config: 0.0 approve_plan: 0.5 safety: require_sandbox: true require_checkpoints: true allow_unsafe_tools: false """ _runner = CliRunner() def _patch_service() -> None: """Patch ``_get_service`` in the CLI module to use an in-memory repo. Each call creates a fresh repo + service so benchmark iterations are isolated from each other and safe for parallel execution. """ import cleveragents.cli.commands.automation_profile as ap_mod repo = _InMemoryProfileRepository() service = AutomationProfileService(repo=repo) # type: ignore[arg-type] ap_mod._get_service = lambda: service class AutomationProfileCLIAddSuite: """Benchmark automation-profile add --config throughput.""" def setup(self) -> None: """Write a temporary YAML file and patch service.""" _patch_service() fd, self._path = tempfile.mkstemp(suffix=".yaml") with os.fdopen(fd, "w") as fh: fh.write(_VALID_YAML) def teardown(self) -> None: """Clean up.""" Path(self._path).unlink(missing_ok=True) def time_add_from_config(self) -> None: """Benchmark add --config end-to-end.""" _patch_service() _runner.invoke(profile_app, ["add", "--config", self._path]) class AutomationProfileCLIListSuite: """Benchmark automation-profile list throughput.""" def setup(self) -> None: """Set up with built-in profiles.""" _patch_service() def time_list_all(self) -> None: """Benchmark listing all profiles.""" _runner.invoke(profile_app, ["list"]) def time_list_json(self) -> None: """Benchmark listing profiles as JSON.""" _runner.invoke(profile_app, ["list", "--format", "json"]) def time_list_with_regex(self) -> None: """Benchmark listing with regex filter.""" _runner.invoke(profile_app, ["list", "caut.*"]) class AutomationProfileCLIShowSuite: """Benchmark automation-profile show throughput.""" def setup(self) -> None: """Set up.""" _patch_service() def time_show_rich(self) -> None: """Benchmark showing a profile in rich format.""" _runner.invoke(profile_app, ["show", "manual"]) def time_show_json(self) -> None: """Benchmark showing a profile in JSON format.""" _runner.invoke(profile_app, ["show", "manual", "--format", "json"]) def time_show_yaml(self) -> None: """Benchmark showing a profile in YAML format.""" _runner.invoke(profile_app, ["show", "manual", "--format", "yaml"])