177 lines
5.1 KiB
Python
177 lines
5.1 KiB
Python
"""ASV benchmarks for Tool CLI command throughput.
|
|
|
|
Measures the performance of:
|
|
- Config-only add (YAML load + validate + Tool.from_config)
|
|
- List command rendering
|
|
- Show command rendering
|
|
- Remove command rendering
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from typer.testing import CliRunner # noqa: E402
|
|
|
|
from cleveragents.cli.commands.tool import app as tool_app # noqa: E402
|
|
from cleveragents.cli.commands.validation import app as validation_app # noqa: E402
|
|
|
|
_TOOL_YAML = """\
|
|
name: local/bench-tool
|
|
description: Benchmark tool
|
|
source: custom
|
|
code: |
|
|
def run(inputs):
|
|
return {"result": True}
|
|
"""
|
|
|
|
_VALIDATION_YAML = """\
|
|
name: local/bench-val
|
|
description: Benchmark validation
|
|
source: custom
|
|
mode: required
|
|
code: |
|
|
def run(inputs):
|
|
return {"passed": True}
|
|
"""
|
|
|
|
_runner = CliRunner()
|
|
|
|
|
|
def _mock_tool(
|
|
name: str = "local/bench-tool",
|
|
tool_type: str = "tool",
|
|
) -> dict[str, Any]:
|
|
ns, sn = name.split("/", 1)
|
|
return {
|
|
"name": name,
|
|
"description": f"Benchmark {tool_type}",
|
|
"source": "custom",
|
|
"tool_type": tool_type,
|
|
"namespace": ns,
|
|
"short_name": sn,
|
|
"capability": {"read_only": False, "writes": False, "checkpointable": False},
|
|
"timeout": 300,
|
|
}
|
|
|
|
|
|
class ToolCLIAddSuite:
|
|
"""Benchmark tool add --config throughput."""
|
|
|
|
def setup(self) -> None:
|
|
"""Write a temporary YAML file and set up mock service."""
|
|
fd, self._path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(_TOOL_YAML)
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.register_tool.return_value = _mock_tool()
|
|
self._mock_service.get_tool.return_value = None
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.tool._get_tool_registry_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
"""Clean up."""
|
|
self._patcher.stop()
|
|
Path(self._path).unlink(missing_ok=True)
|
|
|
|
def time_add_from_config(self) -> None:
|
|
"""Benchmark add --config end-to-end."""
|
|
_runner.invoke(tool_app, ["add", "--config", self._path])
|
|
|
|
|
|
class ToolCLIListSuite:
|
|
"""Benchmark tool list throughput."""
|
|
|
|
def setup(self) -> None:
|
|
"""Set up mock service with tools."""
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.list_tools.return_value = [
|
|
_mock_tool(f"local/tool-{i}") for i in range(50)
|
|
]
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.tool._get_tool_registry_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
|
|
def time_list_all(self) -> None:
|
|
"""Benchmark listing all tools."""
|
|
_runner.invoke(tool_app, ["list"])
|
|
|
|
def time_list_with_namespace(self) -> None:
|
|
"""Benchmark listing with namespace filter."""
|
|
_runner.invoke(tool_app, ["list", "--namespace", "local"])
|
|
|
|
def time_list_with_type(self) -> None:
|
|
"""Benchmark listing with type filter."""
|
|
_runner.invoke(tool_app, ["list", "--type", "tool"])
|
|
|
|
|
|
class ToolCLIShowSuite:
|
|
"""Benchmark tool show throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.get_tool.return_value = _mock_tool()
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.tool._get_tool_registry_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
|
|
def time_show(self) -> None:
|
|
"""Benchmark showing a tool."""
|
|
_runner.invoke(tool_app, ["show", "local/bench-tool"])
|
|
|
|
|
|
class ValidationCLIAddSuite:
|
|
"""Benchmark validation add --config throughput."""
|
|
|
|
def setup(self) -> None:
|
|
fd, self._path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(_VALIDATION_YAML)
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.register_tool.return_value = _mock_tool(
|
|
"local/bench-val", "validation"
|
|
)
|
|
self._mock_service.get_tool.return_value = None
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.validation._get_tool_registry_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
Path(self._path).unlink(missing_ok=True)
|
|
|
|
def time_add_from_config(self) -> None:
|
|
"""Benchmark validation add --config end-to-end."""
|
|
_runner.invoke(validation_app, ["add", "--config", self._path])
|