cleveragents-core/benchmarks/automation_profile_cli_bench.py

"""ASV benchmarks for Automation Profile CLI command throughput.

Measures the performance of:
- Add profile from YAML config
- List profiles rendering
- Show profile rendering
- Remove profile
"""

from __future__ import annotations

import importlib
import os
import sys
import tempfile
from pathlib import Path

# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

import cleveragents  # noqa: E402

importlib.reload(cleveragents)

from typer.testing import CliRunner  # noqa: E402

from cleveragents.application.services.automation_profile_service import (  # noqa: E402
    AutomationProfileService,
)
from cleveragents.cli.commands.automation_profile import (  # noqa: E402
    app as profile_app,
)
from cleveragents.domain.models.core.automation_profile import (  # noqa: E402
    AutomationProfile,
)


class _InMemoryProfileRepository:
    """In-memory repository satisfying the duck-type contract.

    Provides the four methods expected by ``AutomationProfileService``:
    ``get_by_name``, ``list_all``, ``upsert``, and ``delete``.
    """

    def __init__(self) -> None:
        self._store: dict[str, AutomationProfile] = {}

    def get_by_name(self, name: str) -> AutomationProfile | None:
        return self._store.get(name)

    def list_all(self) -> list[AutomationProfile]:
        return list(self._store.values())

    def upsert(self, profile: AutomationProfile) -> None:
        self._store[profile.name] = profile

    def delete(self, name: str) -> None:
        self._store.pop(name, None)


_VALID_YAML = """\
name: bench/test-profile
description: Benchmark test profile
schema_version: "1.0"
auto_strategize: 0.5
auto_execute: 0.5
auto_apply: 1.0
auto_decisions_strategize: 0.5
auto_decisions_execute: 0.5
auto_validation_fix: 0.5
auto_strategy_revision: 0.5
auto_reversion_from_apply: 0.5
auto_child_plans: 0.5
auto_retry_transient: 0.0
auto_checkpoint_restore: 0.5
safety:
  require_sandbox: true
  require_checkpoints: true
  allow_unsafe_tools: false
"""

_runner = CliRunner()


def _patch_service() -> None:
    """Patch ``_get_service`` in the CLI module to use an in-memory repo.

    Each call creates a fresh repo + service so benchmark iterations
    are isolated from each other and safe for parallel execution.
    """
    import cleveragents.cli.commands.automation_profile as ap_mod

    repo = _InMemoryProfileRepository()
    service = AutomationProfileService(repo=repo)  # type: ignore[arg-type]
    ap_mod._get_service = lambda: service


class AutomationProfileCLIAddSuite:
    """Benchmark automation-profile add --config throughput."""

    def setup(self) -> None:
        """Write a temporary YAML file and patch service."""
        _patch_service()
        fd, self._path = tempfile.mkstemp(suffix=".yaml")
        with os.fdopen(fd, "w") as fh:
            fh.write(_VALID_YAML)

    def teardown(self) -> None:
        """Clean up."""
        Path(self._path).unlink(missing_ok=True)

    def time_add_from_config(self) -> None:
        """Benchmark add --config end-to-end."""
        _patch_service()
        _runner.invoke(profile_app, ["add", "--config", self._path])


class AutomationProfileCLIListSuite:
    """Benchmark automation-profile list throughput."""

    def setup(self) -> None:
        """Set up with built-in profiles."""
        _patch_service()

    def time_list_all(self) -> None:
        """Benchmark listing all profiles."""
        _runner.invoke(profile_app, ["list"])

    def time_list_json(self) -> None:
        """Benchmark listing profiles as JSON."""
        _runner.invoke(profile_app, ["list", "--format", "json"])

    def time_list_with_regex(self) -> None:
        """Benchmark listing with regex filter."""
        _runner.invoke(profile_app, ["list", "caut.*"])


class AutomationProfileCLIShowSuite:
    """Benchmark automation-profile show throughput."""

    def setup(self) -> None:
        """Set up."""
        _patch_service()

    def time_show_rich(self) -> None:
        """Benchmark showing a profile in rich format."""
        _runner.invoke(profile_app, ["show", "manual"])

    def time_show_json(self) -> None:
        """Benchmark showing a profile in JSON format."""
        _runner.invoke(profile_app, ["show", "manual", "--format", "json"])

    def time_show_yaml(self) -> None:
        """Benchmark showing a profile in YAML format."""
        _runner.invoke(profile_app, ["show", "manual", "--format", "yaml"])