"""ASV benchmarks for BindingResolutionService. Measures resolve latency for contextual, static, and parameter bindings across varying numbers of linked resources. """ from __future__ import annotations import typing from unittest.mock import MagicMock from cleveragents.application.services.binding_resolution_service import ( BindingResolutionService, ) from cleveragents.core.exceptions import NotFoundError from cleveragents.domain.models.core.project import ( LinkedResource, NamespacedProject, ) from cleveragents.domain.models.core.resource import PhysVirt, Resource from cleveragents.domain.models.core.resource_type import ( ResourceKind, ResourceTypeSpec, SandboxStrategy, ) from cleveragents.domain.models.core.tool import ( BindingMode, ResourceAccessMode, ResourceSlot, Tool, ToolSource, ToolType, ) _CB32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" _bench_counter = 0 def _bench_ulid() -> str: global _bench_counter _bench_counter += 1 n = _bench_counter suffix = "" for _ in range(8): suffix = _CB32[n % 32] + suffix n //= 32 return f"01HGZ6FE0AQDYTR4BX{suffix}" def _make_registry( resources: dict[str, Resource], types: dict[str, ResourceTypeSpec], ) -> MagicMock: registry = MagicMock() def show_resource(name_or_id: str) -> Resource: res = resources.get(name_or_id) if res is None: raise NotFoundError( resource_type="resource", resource_id=name_or_id, ) return res def show_type(name: str) -> ResourceTypeSpec: spec = types.get(name) if spec is None: raise NotFoundError( resource_type="resource_type", resource_id=name, ) return spec registry.show_resource = show_resource registry.show_type = show_type return registry class BindingResolutionContextualBench: """Benchmark contextual binding resolution latency.""" timeout = 60 params: typing.ClassVar[list[int]] = [1, 10, 50] param_names: typing.ClassVar[list[str]] = ["num_resources"] def setup(self, num_resources: int) -> None: global _bench_counter _bench_counter = 0 resources: dict[str, Resource] = {} links: list[LinkedResource] = [] for i in range(num_resources): rid = _bench_ulid() res = Resource( resource_id=rid, name=None, resource_type_name="git-checkout", classification=PhysVirt.PHYSICAL, ) resources[rid] = res alias = "repo" if i == 0 else f"res-{i}" links.append(LinkedResource(resource_id=rid, alias=alias)) types: dict[str, ResourceTypeSpec] = { "git-checkout": ResourceTypeSpec( name="git-checkout", resource_kind=ResourceKind.PHYSICAL, sandbox_strategy=SandboxStrategy.NONE, built_in=True, ), } self.registry = _make_registry(resources, types) self.project = NamespacedProject( name="bench", namespace="local", linked_resources=links, ) self.tool = Tool( name="bench/reader", description="Benchmark tool", source=ToolSource.BUILTIN, tool_type=ToolType.TOOL, resource_slots=[ ResourceSlot( name="repo", resource_type="git-checkout", access=ResourceAccessMode.READ_WRITE, binding=BindingMode.CONTEXTUAL, ), ], ) self.service = BindingResolutionService(self.registry) def time_resolve_contextual(self, num_resources: int) -> None: self.service.resolve(self.tool, self.project) class BindingResolutionStaticBench: """Benchmark static binding resolution latency.""" timeout = 60 def setup(self) -> None: global _bench_counter _bench_counter = 0 rid = _bench_ulid() res = Resource( resource_id=rid, name="bench/config", resource_type_name="fs-directory", classification=PhysVirt.PHYSICAL, ) resources = { "bench/config": res, rid: res, } types: dict[str, ResourceTypeSpec] = { "fs-directory": ResourceTypeSpec( name="fs-directory", resource_kind=ResourceKind.PHYSICAL, sandbox_strategy=SandboxStrategy.NONE, built_in=True, ), } self.registry = _make_registry(resources, types) self.project = NamespacedProject(name="bench", namespace="local") self.tool = Tool( name="bench/static", description="Benchmark static tool", source=ToolSource.BUILTIN, tool_type=ToolType.TOOL, resource_slots=[ ResourceSlot( name="config", resource_type="fs-directory", access=ResourceAccessMode.READ_ONLY, binding=BindingMode.STATIC, static_resource="bench/config", ), ], ) self.service = BindingResolutionService(self.registry) def time_resolve_static(self) -> None: self.service.resolve(self.tool, self.project) class BindingResolutionParameterBench: """Benchmark parameter binding (deferred) resolution.""" timeout = 60 def setup(self) -> None: registry = MagicMock() self.project = NamespacedProject(name="bench", namespace="local") self.tool = Tool( name="bench/param", description="Benchmark param tool", source=ToolSource.BUILTIN, tool_type=ToolType.TOOL, resource_slots=[ ResourceSlot( name="target", resource_type="fs-directory", access=ResourceAccessMode.READ_WRITE, binding=BindingMode.PARAMETER, ), ], ) self.service = BindingResolutionService(registry) def time_resolve_parameter_deferred(self) -> None: self.service.resolve(self.tool, self.project)