127 lines
3.5 KiB
Python
127 lines
3.5 KiB
Python
"""ASV benchmarks for ChangeSet capture overhead.
|
|
|
|
Measures the cost of creating entries, recording to a store,
|
|
computing summaries, and wrapping file tools with capture.
|
|
"""
|
|
|
|
import hashlib
|
|
import tempfile
|
|
from typing import ClassVar
|
|
|
|
from cleveragents.domain.models.core.change import (
|
|
ChangeEntry,
|
|
ChangeOperation,
|
|
InMemoryChangeSetStore,
|
|
SpecChangeSet,
|
|
)
|
|
from cleveragents.tool.builtins.changeset import (
|
|
ChangeSetCapture,
|
|
)
|
|
from cleveragents.tool.builtins.file_tools import (
|
|
FILE_WRITE_SPEC,
|
|
)
|
|
|
|
|
|
class ChangeEntryCreation:
|
|
"""Benchmark ChangeEntry instantiation."""
|
|
|
|
def setup(self):
|
|
self.kwargs = {
|
|
"plan_id": "plan-bench",
|
|
"resource_id": "res-bench",
|
|
"tool_name": "builtin/file-write",
|
|
"operation": ChangeOperation.CREATE,
|
|
"path": "bench/file.py",
|
|
"after_hash": hashlib.sha256(b"content").hexdigest(),
|
|
}
|
|
|
|
def time_create_entry(self):
|
|
ChangeEntry(**self.kwargs)
|
|
|
|
|
|
class SpecChangeSetSummary:
|
|
"""Benchmark SpecChangeSet summary computation."""
|
|
|
|
params: ClassVar[list[int]] = [10, 100, 1000]
|
|
param_names: ClassVar[list[str]] = ["num_entries"]
|
|
|
|
def setup(self, num_entries):
|
|
entries = []
|
|
ops = list(ChangeOperation)
|
|
for i in range(num_entries):
|
|
entries.append(
|
|
ChangeEntry(
|
|
plan_id="plan-bench",
|
|
resource_id=f"res-{i % 5}",
|
|
tool_name="builtin/file-write",
|
|
operation=ops[i % len(ops)],
|
|
path=f"file_{i}.py",
|
|
)
|
|
)
|
|
self.cs = SpecChangeSet(plan_id="plan-bench", entries=entries)
|
|
|
|
def time_summary(self, num_entries):
|
|
self.cs.summary()
|
|
|
|
def time_paths_changed(self, num_entries):
|
|
_ = self.cs.paths_changed
|
|
|
|
def time_resources_involved(self, num_entries):
|
|
_ = self.cs.resources_involved
|
|
|
|
|
|
class InMemoryStoreRecording:
|
|
"""Benchmark InMemoryChangeSetStore operations."""
|
|
|
|
params: ClassVar[list[int]] = [10, 100, 1000]
|
|
param_names: ClassVar[list[str]] = ["num_entries"]
|
|
|
|
def setup(self, num_entries):
|
|
self.store = InMemoryChangeSetStore()
|
|
self.cs_id = self.store.start("plan-bench")
|
|
self.entries = [
|
|
ChangeEntry(
|
|
plan_id="plan-bench",
|
|
resource_id="res-1",
|
|
tool_name="builtin/file-write",
|
|
operation=ChangeOperation.CREATE,
|
|
path=f"file_{i}.py",
|
|
)
|
|
for i in range(num_entries)
|
|
]
|
|
|
|
def time_record_entries(self, num_entries):
|
|
store = InMemoryChangeSetStore()
|
|
cs_id = store.start("plan-bench")
|
|
for entry in self.entries:
|
|
store.record(cs_id, entry)
|
|
|
|
def time_summarize(self, num_entries):
|
|
self.store.summarize(self.cs_id)
|
|
|
|
|
|
class CaptureWrapOverhead:
|
|
"""Benchmark wrapping a tool with capture."""
|
|
|
|
def setup(self):
|
|
self.tmpdir = tempfile.mkdtemp()
|
|
self.capture = ChangeSetCapture(
|
|
plan_id="plan-bench",
|
|
resource_id="res-bench",
|
|
sandbox_root=self.tmpdir,
|
|
)
|
|
|
|
def time_wrap_tool(self):
|
|
self.capture.wrap_tool(FILE_WRITE_SPEC)
|
|
|
|
def time_wrapped_execution(self):
|
|
wrapped = self.capture.wrap_tool(FILE_WRITE_SPEC)
|
|
wrapped.handler(
|
|
{
|
|
"path": "bench_out.txt",
|
|
"content": "benchmark",
|
|
"sandbox_root": self.tmpdir,
|
|
}
|
|
)
|
|
self.capture.clear()
|