cleveragents-core/benchmarks/action_cli_bench.py

"""ASV benchmarks for Action CLI command throughput.

Measures the performance of:
- Config-only create (YAML load + validate + Action.from_config)
- List command rendering
- Show command rendering
- Archive command rendering
"""

from __future__ import annotations

import importlib
import os
import sys
import tempfile
from datetime import datetime
from pathlib import Path
from unittest.mock import MagicMock, patch

# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

import cleveragents  # noqa: E402

importlib.reload(cleveragents)

from typer.testing import CliRunner  # noqa: E402

from cleveragents.cli.commands.action import app as action_app  # noqa: E402
from cleveragents.domain.models.core.action import Action, ActionState  # noqa: E402
from cleveragents.domain.models.core.plan import NamespacedName  # noqa: E402

_VALID_YAML = """\
name: local/bench-action
description: Benchmark action
strategy_actor: openai/gpt-4
execution_actor: openai/gpt-4
definition_of_done: Benchmarks pass
"""

_runner = CliRunner()


def _mock_action(name: str = "local/bench-action") -> Action:
    return Action(
        namespaced_name=NamespacedName.parse(name),
        description="Benchmark action",
        definition_of_done="Benchmarks pass",
        strategy_actor="openai/gpt-4",
        execution_actor="openai/gpt-4",
        state=ActionState.AVAILABLE,
        created_at=datetime.now(),
        updated_at=datetime.now(),
    )


class ActionCLICreateSuite:
    """Benchmark action create --config throughput."""

    def setup(self) -> None:
        """Write a temporary YAML file and set up mock service."""
        fd, self._path = tempfile.mkstemp(suffix=".yaml")
        with os.fdopen(fd, "w") as fh:
            fh.write(_VALID_YAML)
        self._mock_service = MagicMock()
        self._mock_service.create_action.return_value = _mock_action()
        self._patcher = patch(
            "cleveragents.cli.commands.action._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._patcher.start()

    def teardown(self) -> None:
        """Clean up."""
        self._patcher.stop()
        Path(self._path).unlink(missing_ok=True)

    def time_create_from_config(self) -> None:
        """Benchmark create --config end-to-end."""
        _runner.invoke(action_app, ["create", "--config", self._path])


class ActionCLIListSuite:
    """Benchmark action list throughput."""

    def setup(self) -> None:
        """Set up mock service with actions."""
        self._mock_service = MagicMock()
        self._mock_service.list_actions.return_value = [
            _mock_action(f"local/action-{i}") for i in range(50)
        ]
        self._patcher = patch(
            "cleveragents.cli.commands.action._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._patcher.start()

    def teardown(self) -> None:
        self._patcher.stop()

    def time_list_all(self) -> None:
        """Benchmark listing all actions."""
        _runner.invoke(action_app, ["list"])

    def time_list_with_namespace(self) -> None:
        """Benchmark listing with namespace filter."""
        _runner.invoke(action_app, ["list", "--namespace", "local"])

    def time_list_with_state(self) -> None:
        """Benchmark listing with state filter."""
        _runner.invoke(action_app, ["list", "--state", "available"])


class ActionCLIShowSuite:
    """Benchmark action show throughput."""

    def setup(self) -> None:
        self._mock_service = MagicMock()
        self._mock_service.get_action_by_name.return_value = _mock_action()
        self._patcher = patch(
            "cleveragents.cli.commands.action._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._patcher.start()

    def teardown(self) -> None:
        self._patcher.stop()

    def time_show(self) -> None:
        """Benchmark showing an action."""
        _runner.invoke(action_app, ["show", "local/bench-action"])