154 lines
4.7 KiB
Python
154 lines
4.7 KiB
Python
"""ASV benchmarks for container tool execution overhead (#515).
|
|
|
|
Measures the performance cost of path mapping, input/output
|
|
translation, and container metadata creation — the operations
|
|
that happen on every container-routed tool call.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from cleveragents.tool.container_executor import (
|
|
ContainerConfig,
|
|
ContainerMetadata,
|
|
ContainerToolExecutor,
|
|
)
|
|
from cleveragents.tool.path_mapper import PathMapper
|
|
|
|
|
|
class PathMapperBench:
|
|
"""Benchmark PathMapper host-to-container and container-to-host translations."""
|
|
|
|
def setup(self) -> None:
|
|
self.mapper = PathMapper(
|
|
host_root="/tmp/sandbox/project-abc",
|
|
container_root="/workspace",
|
|
)
|
|
self.host_path = "/tmp/sandbox/project-abc/src/cleveragents/tool/runner.py"
|
|
self.container_path = "/workspace/src/cleveragents/tool/runner.py"
|
|
self.external_path = (
|
|
"/usr/local/lib/python3.13/site-packages/pydantic/__init__.py"
|
|
)
|
|
|
|
def time_host_to_container(self) -> None:
|
|
self.mapper.host_to_container(self.host_path)
|
|
|
|
def time_container_to_host(self) -> None:
|
|
self.mapper.container_to_host(self.container_path)
|
|
|
|
def time_external_path_passthrough(self) -> None:
|
|
self.mapper.host_to_container(self.external_path)
|
|
|
|
def time_is_host_path_check(self) -> None:
|
|
self.mapper.is_host_path(self.host_path)
|
|
|
|
def time_is_container_path_check(self) -> None:
|
|
self.mapper.is_container_path(self.container_path)
|
|
|
|
|
|
class InputPathMappingBench:
|
|
"""Benchmark input path mapping with various nesting depths."""
|
|
|
|
def setup(self) -> None:
|
|
config = ContainerConfig(
|
|
workspace_folder="/workspace",
|
|
host_sandbox_path="/tmp/sandbox",
|
|
container_id="bench-ctr",
|
|
)
|
|
self.executor = ContainerToolExecutor(config)
|
|
self.flat_inputs = {
|
|
"path": "/tmp/sandbox/src/main.py",
|
|
"output_dir": "/tmp/sandbox/build",
|
|
"name": "my-tool",
|
|
"count": 42,
|
|
}
|
|
self.nested_inputs = {
|
|
"paths": {
|
|
"source": "/tmp/sandbox/src/main.py",
|
|
"config": "/tmp/sandbox/.config/settings.json",
|
|
"nested": {
|
|
"deep": "/tmp/sandbox/deep/path.txt",
|
|
},
|
|
},
|
|
"list_paths": [
|
|
"/tmp/sandbox/a.py",
|
|
"/tmp/sandbox/b.py",
|
|
"/tmp/sandbox/c.py",
|
|
],
|
|
}
|
|
|
|
def time_flat_input_mapping(self) -> None:
|
|
self.executor._map_input_paths(self.flat_inputs)
|
|
|
|
def time_nested_input_mapping(self) -> None:
|
|
self.executor._map_input_paths(self.nested_inputs)
|
|
|
|
|
|
class OutputPathMappingBench:
|
|
"""Benchmark output path mapping."""
|
|
|
|
def setup(self) -> None:
|
|
config = ContainerConfig(
|
|
workspace_folder="/workspace",
|
|
host_sandbox_path="/tmp/sandbox",
|
|
container_id="bench-ctr",
|
|
)
|
|
self.executor = ContainerToolExecutor(config)
|
|
self.output = {
|
|
"created_file": "/workspace/build/output.txt",
|
|
"log_file": "/workspace/logs/run.log",
|
|
"status": "success",
|
|
"count": 10,
|
|
}
|
|
|
|
def time_output_mapping(self) -> None:
|
|
self.executor._map_output_paths(self.output)
|
|
|
|
|
|
class ContainerMetadataBench:
|
|
"""Benchmark container metadata creation and serialisation."""
|
|
|
|
def setup(self) -> None:
|
|
self.metadata = ContainerMetadata(
|
|
container_id="abc123def456",
|
|
image="ghcr.io/org/devimage:latest",
|
|
workspace_folder="/workspace",
|
|
exec_time_ms=1500.0,
|
|
exit_code=0,
|
|
timed_out=False,
|
|
)
|
|
|
|
def time_metadata_creation(self) -> None:
|
|
ContainerMetadata(
|
|
container_id="abc123def456",
|
|
image="ghcr.io/org/devimage:latest",
|
|
workspace_folder="/workspace",
|
|
exec_time_ms=1500.0,
|
|
exit_code=0,
|
|
timed_out=False,
|
|
)
|
|
|
|
def time_metadata_to_dict(self) -> None:
|
|
self.metadata.model_dump()
|
|
|
|
|
|
class OutputParsingBench:
|
|
"""Benchmark output parsing (JSON vs plain text)."""
|
|
|
|
def setup(self) -> None:
|
|
self.json_output = json.dumps(
|
|
{"files": [f"file_{i}.py" for i in range(50)], "errors": 0}
|
|
)
|
|
self.plain_output = "Build completed successfully\n" * 20
|
|
self.empty_output = ""
|
|
|
|
def time_parse_json_output(self) -> None:
|
|
ContainerToolExecutor._parse_output(self.json_output)
|
|
|
|
def time_parse_plain_output(self) -> None:
|
|
ContainerToolExecutor._parse_output(self.plain_output)
|
|
|
|
def time_parse_empty_output(self) -> None:
|
|
ContainerToolExecutor._parse_output(self.empty_output)
|