"""ASV benchmarks for Tool CLI command throughput. Measures the performance of: - Config-only add (YAML load + validate + Tool.from_config) - List command rendering - Show command rendering - Remove command rendering """ from __future__ import annotations import importlib import os import sys import tempfile from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch # Ensure the local *source* tree is importable even when ASV has an # older build of the package installed. _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) import cleveragents # noqa: E402 importlib.reload(cleveragents) from typer.testing import CliRunner # noqa: E402 from cleveragents.cli.commands.tool import app as tool_app # noqa: E402 from cleveragents.cli.commands.validation import app as validation_app # noqa: E402 _TOOL_YAML = """\ name: local/bench-tool description: Benchmark tool source: custom code: | def run(inputs): return {"result": True} """ _VALIDATION_YAML = """\ name: local/bench-val description: Benchmark validation source: custom mode: required code: | def run(inputs): return {"passed": True} """ _runner = CliRunner() def _mock_tool( name: str = "local/bench-tool", tool_type: str = "tool", ) -> dict[str, Any]: ns, sn = name.split("/", 1) return { "name": name, "description": f"Benchmark {tool_type}", "source": "custom", "tool_type": tool_type, "namespace": ns, "short_name": sn, "capability": {"read_only": False, "writes": False, "checkpointable": False}, "timeout": 300, } class ToolCLIAddSuite: """Benchmark tool add --config throughput.""" def setup(self) -> None: """Write a temporary YAML file and set up mock service.""" fd, self._path = tempfile.mkstemp(suffix=".yaml") with os.fdopen(fd, "w") as fh: fh.write(_TOOL_YAML) self._mock_service = MagicMock() self._mock_service.register_tool.return_value = _mock_tool() self._mock_service.get_tool.return_value = None self._patcher = patch( "cleveragents.cli.commands.tool._get_tool_registry_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: """Clean up.""" self._patcher.stop() Path(self._path).unlink(missing_ok=True) def time_add_from_config(self) -> None: """Benchmark add --config end-to-end.""" _runner.invoke(tool_app, ["add", "--config", self._path]) class ToolCLIListSuite: """Benchmark tool list throughput.""" def setup(self) -> None: """Set up mock service with tools.""" self._mock_service = MagicMock() self._mock_service.list_tools.return_value = [ _mock_tool(f"local/tool-{i}") for i in range(50) ] self._patcher = patch( "cleveragents.cli.commands.tool._get_tool_registry_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_list_all(self) -> None: """Benchmark listing all tools.""" _runner.invoke(tool_app, ["list"]) def time_list_with_namespace(self) -> None: """Benchmark listing with namespace filter.""" _runner.invoke(tool_app, ["list", "--namespace", "local"]) def time_list_with_type(self) -> None: """Benchmark listing with type filter.""" _runner.invoke(tool_app, ["list", "--type", "tool"]) class ToolCLIShowSuite: """Benchmark tool show throughput.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_tool.return_value = _mock_tool() self._patcher = patch( "cleveragents.cli.commands.tool._get_tool_registry_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_show(self) -> None: """Benchmark showing a tool.""" _runner.invoke(tool_app, ["show", "local/bench-tool"]) class ValidationCLIAddSuite: """Benchmark validation add --config throughput.""" def setup(self) -> None: fd, self._path = tempfile.mkstemp(suffix=".yaml") with os.fdopen(fd, "w") as fh: fh.write(_VALIDATION_YAML) self._mock_service = MagicMock() self._mock_service.register_tool.return_value = _mock_tool( "local/bench-val", "validation" ) self._mock_service.get_tool.return_value = None self._patcher = patch( "cleveragents.cli.commands.validation._get_tool_registry_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() Path(self._path).unlink(missing_ok=True) def time_add_from_config(self) -> None: """Benchmark validation add --config end-to-end.""" _runner.invoke(validation_app, ["add", "--config", self._path])