cleveragents-core/benchmarks/binding_resolution_bench.py

"""ASV benchmarks for BindingResolutionService.

Measures resolve latency for contextual, static, and parameter
bindings across varying numbers of linked resources.
"""

from __future__ import annotations

from unittest.mock import MagicMock

from cleveragents.application.services.binding_resolution_service import (
    BindingResolutionService,
)
from cleveragents.core.exceptions import NotFoundError
from cleveragents.domain.models.core.project import (
    LinkedResource,
    NamespacedProject,
)
from cleveragents.domain.models.core.resource import PhysVirt, Resource
from cleveragents.domain.models.core.resource_type import (
    ResourceKind,
    ResourceTypeSpec,
    SandboxStrategy,
)
from cleveragents.domain.models.core.tool import (
    BindingMode,
    ResourceAccessMode,
    ResourceSlot,
    Tool,
    ToolSource,
    ToolType,
)

_CB32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
_bench_counter = 0


def _bench_ulid() -> str:
    global _bench_counter
    _bench_counter += 1
    n = _bench_counter
    suffix = ""
    for _ in range(8):
        suffix = _CB32[n % 32] + suffix
        n //= 32
    return f"01HGZ6FE0AQDYTR4BX{suffix}"


def _make_registry(
    resources: dict[str, Resource],
    types: dict[str, ResourceTypeSpec],
) -> MagicMock:
    registry = MagicMock()

    def show_resource(name_or_id: str) -> Resource:
        res = resources.get(name_or_id)
        if res is None:
            raise NotFoundError(
                resource_type="resource",
                resource_id=name_or_id,
            )
        return res

    def show_type(name: str) -> ResourceTypeSpec:
        spec = types.get(name)
        if spec is None:
            raise NotFoundError(
                resource_type="resource_type",
                resource_id=name,
            )
        return spec

    registry.show_resource = show_resource
    registry.show_type = show_type
    return registry


class BindingResolutionContextualBench:
    """Benchmark contextual binding resolution latency."""

    timeout = 60
    params: list[int] = [1, 10, 50]
    param_names: list[str] = ["num_resources"]

    def setup(self, num_resources: int) -> None:
        global _bench_counter
        _bench_counter = 0

        resources: dict[str, Resource] = {}
        links: list[LinkedResource] = []
        for i in range(num_resources):
            rid = _bench_ulid()
            res = Resource(
                resource_id=rid,
                name=None,
                resource_type_name="git-checkout",
                classification=PhysVirt.PHYSICAL,
            )
            resources[rid] = res
            alias = "repo" if i == 0 else f"res-{i}"
            links.append(LinkedResource(resource_id=rid, alias=alias))

        types: dict[str, ResourceTypeSpec] = {
            "git-checkout": ResourceTypeSpec(
                name="git-checkout",
                resource_kind=ResourceKind.PHYSICAL,
                sandbox_strategy=SandboxStrategy.NONE,
                built_in=True,
            ),
        }

        self.registry = _make_registry(resources, types)
        self.project = NamespacedProject(
            name="bench",
            namespace="local",
            linked_resources=links,
        )
        self.tool = Tool(
            name="bench/reader",
            description="Benchmark tool",
            source=ToolSource.BUILTIN,
            tool_type=ToolType.TOOL,
            resource_slots=[
                ResourceSlot(
                    name="repo",
                    resource_type="git-checkout",
                    access=ResourceAccessMode.READ_WRITE,
                    binding=BindingMode.CONTEXTUAL,
                ),
            ],
        )
        self.service = BindingResolutionService(self.registry)

    def time_resolve_contextual(self, num_resources: int) -> None:
        self.service.resolve(self.tool, self.project)


class BindingResolutionStaticBench:
    """Benchmark static binding resolution latency."""

    timeout = 60

    def setup(self) -> None:
        global _bench_counter
        _bench_counter = 0

        rid = _bench_ulid()
        res = Resource(
            resource_id=rid,
            name="bench/config",
            resource_type_name="fs-directory",
            classification=PhysVirt.PHYSICAL,
        )
        resources = {
            "bench/config": res,
            rid: res,
        }
        types: dict[str, ResourceTypeSpec] = {
            "fs-directory": ResourceTypeSpec(
                name="fs-directory",
                resource_kind=ResourceKind.PHYSICAL,
                sandbox_strategy=SandboxStrategy.NONE,
                built_in=True,
            ),
        }

        self.registry = _make_registry(resources, types)
        self.project = NamespacedProject(name="bench", namespace="local")
        self.tool = Tool(
            name="bench/static",
            description="Benchmark static tool",
            source=ToolSource.BUILTIN,
            tool_type=ToolType.TOOL,
            resource_slots=[
                ResourceSlot(
                    name="config",
                    resource_type="fs-directory",
                    access=ResourceAccessMode.READ_ONLY,
                    binding=BindingMode.STATIC,
                    static_resource="bench/config",
                ),
            ],
        )
        self.service = BindingResolutionService(self.registry)

    def time_resolve_static(self) -> None:
        self.service.resolve(self.tool, self.project)


class BindingResolutionParameterBench:
    """Benchmark parameter binding (deferred) resolution."""

    timeout = 60

    def setup(self) -> None:
        registry = MagicMock()
        self.project = NamespacedProject(name="bench", namespace="local")
        self.tool = Tool(
            name="bench/param",
            description="Benchmark param tool",
            source=ToolSource.BUILTIN,
            tool_type=ToolType.TOOL,
            resource_slots=[
                ResourceSlot(
                    name="target",
                    resource_type="fs-directory",
                    access=ResourceAccessMode.READ_WRITE,
                    binding=BindingMode.PARAMETER,
                ),
            ],
        )
        self.service = BindingResolutionService(registry)

    def time_resolve_parameter_deferred(self) -> None:
        self.service.resolve(self.tool, self.project)