cleveragents-core/benchmarks/security_readonly_bench.py

"""ASV benchmarks for read-only enforcement layer throughput.

Measures the performance of:
- ToolRuntime._enforce_capabilities (allow and deny paths)
- SkillContext.enforce_write_guard (allow and deny paths)
- ChangeSetCapture.wrap_tool (allow and deny paths)
"""

from __future__ import annotations

import contextlib
import importlib
import sys
from pathlib import Path

# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

import cleveragents  # noqa: E402

importlib.reload(cleveragents)

from cleveragents.domain.models.core.tool import (  # noqa: E402
    Tool,
    ToolCapability,
    ToolSource,
    ToolType,
)
from cleveragents.skills.context import (  # noqa: E402
    SkillContext,
    SkillExecutionError,
)
from cleveragents.tool.builtins.changeset import (  # noqa: E402
    ChangeSetCapture,
    ReadOnlyViolationError,
)
from cleveragents.tool.context import ToolExecutionContext  # noqa: E402
from cleveragents.tool.lifecycle import (  # noqa: E402
    ToolAccessDeniedError,
    ToolRuntime,
)
from cleveragents.tool.runtime import ToolSpec  # noqa: E402

# ---------------------------------------------------------------------------
# Fixtures shared across suites
# ---------------------------------------------------------------------------

_WRITE_TOOL = Tool(
    name="bench/writer",
    description="Benchmark write tool",
    source=ToolSource.BUILTIN,
    tool_type=ToolType.TOOL,
    timeout=300,
    capability=ToolCapability(writes=True, read_only=False),
)

_READ_TOOL = Tool(
    name="bench/reader",
    description="Benchmark read tool",
    source=ToolSource.BUILTIN,
    tool_type=ToolType.TOOL,
    timeout=300,
    capability=ToolCapability(writes=False, read_only=True),
)

_RO_CTX = ToolExecutionContext(plan_id="bench-ro", plan_read_only=True)
_RW_CTX = ToolExecutionContext(plan_id="bench-rw", plan_read_only=False)

_WRITE_SPEC = ToolSpec(
    name="bench/writer",
    description="Benchmark write spec",
    capabilities=ToolCapability(writes=True, read_only=False),
    handler=lambda inputs: {"ok": True},
)

_READ_SPEC = ToolSpec(
    name="bench/reader",
    description="Benchmark read spec",
    capabilities=ToolCapability(writes=False, read_only=True),
    handler=lambda inputs: {"ok": True},
)


# ---------------------------------------------------------------------------
# ToolRuntime._enforce_capabilities
# ---------------------------------------------------------------------------


class EnforceCapabilitiesSuite:
    """Benchmark ToolRuntime._enforce_capabilities."""

    def time_allow_read_tool_on_readonly_plan(self) -> None:
        """Read-only tool on read-only plan (fast-path: no raise)."""
        ToolRuntime._enforce_capabilities(_READ_TOOL, _RO_CTX)

    def time_allow_write_tool_on_rw_plan(self) -> None:
        """Write tool on writable plan (fast-path: no raise)."""
        ToolRuntime._enforce_capabilities(_WRITE_TOOL, _RW_CTX)

    def time_deny_write_tool_on_readonly_plan(self) -> None:
        """Write tool on read-only plan (slow-path: raises)."""
        with contextlib.suppress(ToolAccessDeniedError):
            ToolRuntime._enforce_capabilities(_WRITE_TOOL, _RO_CTX)


# ---------------------------------------------------------------------------
# SkillContext.enforce_write_guard
# ---------------------------------------------------------------------------


class EnforceWriteGuardSuite:
    """Benchmark SkillContext.enforce_write_guard."""

    def setup(self) -> None:
        self.ro_ctx = SkillContext(
            plan_id="bench-ro",
            project_id="proj",
            sandbox_path=Path("/tmp/sandbox"),
            read_only=True,
        )
        self.rw_ctx = SkillContext(
            plan_id="bench-rw",
            project_id="proj",
            sandbox_path=Path("/tmp/sandbox"),
            read_only=False,
        )

    def time_allow_write_guard(self) -> None:
        """Writable context allows write (fast-path)."""
        self.rw_ctx.enforce_write_guard("bench/tool")

    def time_deny_write_guard(self) -> None:
        """Read-only context denies write (slow-path: raises)."""
        with contextlib.suppress(SkillExecutionError):
            self.ro_ctx.enforce_write_guard("bench/tool")


# ---------------------------------------------------------------------------
# ChangeSetCapture.wrap_tool
# ---------------------------------------------------------------------------


class ChangeSetCaptureSuite:
    """Benchmark ChangeSetCapture.wrap_tool."""

    def setup(self) -> None:
        self.ro_capture = ChangeSetCapture(plan_id="bench-ro", read_only=True)
        self.rw_capture = ChangeSetCapture(plan_id="bench-rw", read_only=False)

    def time_pass_through_read_spec(self) -> None:
        """Read-only spec passes through (no wrapping)."""
        self.ro_capture.wrap_tool(_READ_SPEC)

    def time_wrap_write_spec_on_rw(self) -> None:
        """Write spec wraps on writable capture."""
        self.rw_capture.wrap_tool(_WRITE_SPEC)

    def time_deny_write_spec_on_ro(self) -> None:
        """Write spec denied on read-only capture (raises)."""
        with contextlib.suppress(ReadOnlyViolationError):
            self.ro_capture.wrap_tool(_WRITE_SPEC)