119 lines
3.5 KiB
Python
119 lines
3.5 KiB
Python
"""ASV benchmarks for inline tool executor operations.
|
|
|
|
Measures the performance of:
|
|
- InlineToolExecutor creation
|
|
- Simple inline tool execution overhead
|
|
- InlineToolResult construction
|
|
- Tool validation
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.domain.models.core.skill import SkillInlineTool
|
|
from cleveragents.domain.models.core.tool import ToolCapability, ToolSource
|
|
from cleveragents.skills.context import SkillContext
|
|
from cleveragents.skills.inline_executor import InlineToolExecutor, InlineToolResult
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.domain.models.core.skill import SkillInlineTool
|
|
from cleveragents.domain.models.core.tool import ToolCapability, ToolSource
|
|
from cleveragents.skills.context import SkillContext
|
|
from cleveragents.skills.inline_executor import InlineToolExecutor, InlineToolResult
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_SANDBOX = Path("/tmp/bench-inline-sandbox")
|
|
|
|
_SIMPLE_TOOL = SkillInlineTool(
|
|
description="Benchmark print tool",
|
|
source=ToolSource.CUSTOM,
|
|
code='print("bench")',
|
|
)
|
|
|
|
_NOOP_TOOL = SkillInlineTool(
|
|
description="Benchmark no-op tool",
|
|
source=ToolSource.CUSTOM,
|
|
code="pass",
|
|
)
|
|
|
|
_WRITE_TOOL = SkillInlineTool(
|
|
description="Benchmark write tool",
|
|
source=ToolSource.CUSTOM,
|
|
code='print("write")',
|
|
capability=ToolCapability(writes=True),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Executor Benchmarks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TimeInlineExecutorCreation:
|
|
"""Benchmark InlineToolExecutor creation throughput."""
|
|
|
|
timeout = 60
|
|
|
|
def time_create_executor(self) -> None:
|
|
"""Time basic InlineToolExecutor creation."""
|
|
InlineToolExecutor()
|
|
|
|
def time_create_executor_custom(self) -> None:
|
|
"""Time InlineToolExecutor creation with custom limits."""
|
|
InlineToolExecutor(max_runtime_seconds=10.0, max_output_bytes=512)
|
|
|
|
|
|
class TimeInlineToolExecution:
|
|
"""Benchmark inline tool execution overhead."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
"""Create executor and context for benchmarking."""
|
|
self.executor = InlineToolExecutor()
|
|
self.ctx = SkillContext(
|
|
plan_id="plan-bench-inline",
|
|
project_id="proj-bench-inline",
|
|
sandbox_path=_SANDBOX,
|
|
)
|
|
|
|
def time_execute_noop(self) -> None:
|
|
"""Time no-op inline tool execution."""
|
|
self.executor.execute(_NOOP_TOOL, self.ctx, {})
|
|
|
|
def time_execute_print(self) -> None:
|
|
"""Time simple print inline tool execution."""
|
|
self.executor.execute(_SIMPLE_TOOL, self.ctx, {})
|
|
|
|
def time_validate_tool(self) -> None:
|
|
"""Time tool validation."""
|
|
self.executor.validate_tool(_SIMPLE_TOOL)
|
|
|
|
|
|
class TimeInlineToolResult:
|
|
"""Benchmark InlineToolResult construction."""
|
|
|
|
timeout = 60
|
|
|
|
def time_create_success_result(self) -> None:
|
|
"""Time successful result creation."""
|
|
InlineToolResult(
|
|
success=True,
|
|
output="benchmark output",
|
|
duration_ms=1.5,
|
|
)
|
|
|
|
def time_create_error_result(self) -> None:
|
|
"""Time error result creation."""
|
|
InlineToolResult(
|
|
success=False,
|
|
error_message="benchmark error",
|
|
duration_ms=2.0,
|
|
)
|