"""ASV benchmarks for inline tool executor operations. Measures the performance of: - InlineToolExecutor creation - Simple inline tool execution overhead - InlineToolResult construction - Tool validation """ from __future__ import annotations import sys from pathlib import Path try: from cleveragents.domain.models.core.skill import SkillInlineTool from cleveragents.domain.models.core.tool import ToolCapability, ToolSource from cleveragents.skills.context import SkillContext from cleveragents.skills.inline_executor import InlineToolExecutor, InlineToolResult except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.domain.models.core.skill import SkillInlineTool from cleveragents.domain.models.core.tool import ToolCapability, ToolSource from cleveragents.skills.context import SkillContext from cleveragents.skills.inline_executor import InlineToolExecutor, InlineToolResult # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- _SANDBOX = Path("/tmp/bench-inline-sandbox") _SIMPLE_TOOL = SkillInlineTool( description="Benchmark print tool", source=ToolSource.CUSTOM, code='print("bench")', ) _NOOP_TOOL = SkillInlineTool( description="Benchmark no-op tool", source=ToolSource.CUSTOM, code="pass", ) _WRITE_TOOL = SkillInlineTool( description="Benchmark write tool", source=ToolSource.CUSTOM, code='print("write")', capability=ToolCapability(writes=True), ) # --------------------------------------------------------------------------- # Executor Benchmarks # --------------------------------------------------------------------------- class TimeInlineExecutorCreation: """Benchmark InlineToolExecutor creation throughput.""" timeout = 60 def time_create_executor(self) -> None: """Time basic InlineToolExecutor creation.""" InlineToolExecutor() def time_create_executor_custom(self) -> None: """Time InlineToolExecutor creation with custom limits.""" InlineToolExecutor(max_runtime_seconds=10.0, max_output_bytes=512) class TimeInlineToolExecution: """Benchmark inline tool execution overhead.""" timeout = 60 def setup(self) -> None: """Create executor and context for benchmarking.""" self.executor = InlineToolExecutor() self.ctx = SkillContext( plan_id="plan-bench-inline", project_id="proj-bench-inline", sandbox_path=_SANDBOX, ) def time_execute_noop(self) -> None: """Time no-op inline tool execution.""" self.executor.execute(_NOOP_TOOL, self.ctx, {}) def time_execute_print(self) -> None: """Time simple print inline tool execution.""" self.executor.execute(_SIMPLE_TOOL, self.ctx, {}) def time_validate_tool(self) -> None: """Time tool validation.""" self.executor.validate_tool(_SIMPLE_TOOL) class TimeInlineToolResult: """Benchmark InlineToolResult construction.""" timeout = 60 def time_create_success_result(self) -> None: """Time successful result creation.""" InlineToolResult( success=True, output="benchmark output", duration_ms=1.5, ) def time_create_error_result(self) -> None: """Time error result creation.""" InlineToolResult( success=False, error_message="benchmark error", duration_ms=2.0, )