"""ASV benchmarks for read-only enforcement layer throughput. Measures the performance of: - ToolRuntime._enforce_capabilities (allow and deny paths) - SkillContext.enforce_write_guard (allow and deny paths) - ChangeSetCapture.wrap_tool (allow and deny paths) """ from __future__ import annotations import contextlib import importlib import sys from pathlib import Path # Ensure the local *source* tree is importable even when ASV has an # older build of the package installed. _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) import cleveragents # noqa: E402 importlib.reload(cleveragents) from cleveragents.domain.models.core.tool import ( # noqa: E402 Tool, ToolCapability, ToolSource, ToolType, ) from cleveragents.skills.context import ( # noqa: E402 SkillContext, SkillExecutionError, ) from cleveragents.tool.builtins.changeset import ( # noqa: E402 ChangeSetCapture, ReadOnlyViolationError, ) from cleveragents.tool.context import ToolExecutionContext # noqa: E402 from cleveragents.tool.lifecycle import ( # noqa: E402 ToolAccessDeniedError, ToolRuntime, ) from cleveragents.tool.runtime import ToolSpec # noqa: E402 # --------------------------------------------------------------------------- # Fixtures shared across suites # --------------------------------------------------------------------------- _WRITE_TOOL = Tool( name="bench/writer", description="Benchmark write tool", source=ToolSource.BUILTIN, tool_type=ToolType.TOOL, timeout=300, capability=ToolCapability(writes=True, read_only=False), ) _READ_TOOL = Tool( name="bench/reader", description="Benchmark read tool", source=ToolSource.BUILTIN, tool_type=ToolType.TOOL, timeout=300, capability=ToolCapability(writes=False, read_only=True), ) _RO_CTX = ToolExecutionContext(plan_id="bench-ro", plan_read_only=True) _RW_CTX = ToolExecutionContext(plan_id="bench-rw", plan_read_only=False) _WRITE_SPEC = ToolSpec( name="bench/writer", description="Benchmark write spec", capabilities=ToolCapability(writes=True, read_only=False), handler=lambda inputs: {"ok": True}, ) _READ_SPEC = ToolSpec( name="bench/reader", description="Benchmark read spec", capabilities=ToolCapability(writes=False, read_only=True), handler=lambda inputs: {"ok": True}, ) # --------------------------------------------------------------------------- # ToolRuntime._enforce_capabilities # --------------------------------------------------------------------------- class EnforceCapabilitiesSuite: """Benchmark ToolRuntime._enforce_capabilities.""" def time_allow_read_tool_on_readonly_plan(self) -> None: """Read-only tool on read-only plan (fast-path: no raise).""" ToolRuntime._enforce_capabilities(_READ_TOOL, _RO_CTX) def time_allow_write_tool_on_rw_plan(self) -> None: """Write tool on writable plan (fast-path: no raise).""" ToolRuntime._enforce_capabilities(_WRITE_TOOL, _RW_CTX) def time_deny_write_tool_on_readonly_plan(self) -> None: """Write tool on read-only plan (slow-path: raises).""" with contextlib.suppress(ToolAccessDeniedError): ToolRuntime._enforce_capabilities(_WRITE_TOOL, _RO_CTX) # --------------------------------------------------------------------------- # SkillContext.enforce_write_guard # --------------------------------------------------------------------------- class EnforceWriteGuardSuite: """Benchmark SkillContext.enforce_write_guard.""" def setup(self) -> None: self.ro_ctx = SkillContext( plan_id="bench-ro", project_id="proj", sandbox_path=Path("/tmp/sandbox"), read_only=True, ) self.rw_ctx = SkillContext( plan_id="bench-rw", project_id="proj", sandbox_path=Path("/tmp/sandbox"), read_only=False, ) def time_allow_write_guard(self) -> None: """Writable context allows write (fast-path).""" self.rw_ctx.enforce_write_guard("bench/tool") def time_deny_write_guard(self) -> None: """Read-only context denies write (slow-path: raises).""" with contextlib.suppress(SkillExecutionError): self.ro_ctx.enforce_write_guard("bench/tool") # --------------------------------------------------------------------------- # ChangeSetCapture.wrap_tool # --------------------------------------------------------------------------- class ChangeSetCaptureSuite: """Benchmark ChangeSetCapture.wrap_tool.""" def setup(self) -> None: self.ro_capture = ChangeSetCapture(plan_id="bench-ro", read_only=True) self.rw_capture = ChangeSetCapture(plan_id="bench-rw", read_only=False) def time_pass_through_read_spec(self) -> None: """Read-only spec passes through (no wrapping).""" self.ro_capture.wrap_tool(_READ_SPEC) def time_wrap_write_spec_on_rw(self) -> None: """Write spec wraps on writable capture.""" self.rw_capture.wrap_tool(_WRITE_SPEC) def time_deny_write_spec_on_ro(self) -> None: """Write spec denied on read-only capture (raises).""" with contextlib.suppress(ReadOnlyViolationError): self.ro_capture.wrap_tool(_WRITE_SPEC)