cleveragents-core/benchmarks/resource_handler_bench.py

"""ASV benchmarks for resource handler resolution overhead.

Measures the performance of:
- Handler resolver (import + cache lookup)
- GitCheckoutHandler.resolve() with mock sandbox
- FsDirectoryHandler.resolve() with mock sandbox
- ResourceHandlerService.resolve_binding() end-to-end
"""

from __future__ import annotations

import sys
from datetime import datetime
from pathlib import Path
from unittest.mock import MagicMock, PropertyMock

try:
    from cleveragents.domain.models.core.resource import (
        PhysVirt,
        Resource,
        ResourceCapabilities,
    )
    from cleveragents.domain.models.core.resource_slot import BindingResult
    from cleveragents.domain.models.core.resource_type import (
        ResourceKind,
        ResourceTypeSpec,
    )
    from cleveragents.domain.models.core.resource_type import (
        SandboxStrategy as TypeSandboxStrategy,
    )
    from cleveragents.infrastructure.sandbox.factory import SandboxFactory
    from cleveragents.infrastructure.sandbox.manager import SandboxManager
    from cleveragents.infrastructure.sandbox.protocol import (
        SandboxContext,
        SandboxStatus,
    )
    from cleveragents.resource.handlers.fs_directory import FsDirectoryHandler
    from cleveragents.resource.handlers.git_checkout import GitCheckoutHandler
    from cleveragents.resource.handlers.resolver import (
        clear_handler_cache,
        resolve_handler,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.domain.models.core.resource import (
        PhysVirt,
        Resource,
        ResourceCapabilities,
    )
    from cleveragents.domain.models.core.resource_slot import BindingResult
    from cleveragents.domain.models.core.resource_type import (
        ResourceKind,
        ResourceTypeSpec,
    )
    from cleveragents.domain.models.core.resource_type import (
        SandboxStrategy as TypeSandboxStrategy,
    )
    from cleveragents.infrastructure.sandbox.factory import SandboxFactory
    from cleveragents.infrastructure.sandbox.manager import SandboxManager
    from cleveragents.infrastructure.sandbox.protocol import (
        SandboxContext,
        SandboxStatus,
    )
    from cleveragents.resource.handlers.fs_directory import FsDirectoryHandler
    from cleveragents.resource.handlers.git_checkout import GitCheckoutHandler
    from cleveragents.resource.handlers.resolver import (
        clear_handler_cache,
        resolve_handler,
    )


def _make_resource(rtype: str, location: str) -> Resource:
    return Resource(
        resource_id="01KJ5C5TPMP8GGX3QC83E2MAQS",
        resource_type_name=rtype,
        classification=PhysVirt.PHYSICAL,
        location=location,
        capabilities=ResourceCapabilities(
            readable=True, writable=True, sandboxable=True
        ),
    )


def _make_mock_manager() -> SandboxManager:
    mock_factory = MagicMock(spec=SandboxFactory)
    mock_sandbox = MagicMock()
    mock_sandbox.sandbox_id = "sb-bench-001"
    type(mock_sandbox).status = PropertyMock(return_value=SandboxStatus.CREATED)
    mock_sandbox.context = SandboxContext(
        sandbox_id="sb-bench-001",
        sandbox_path="/tmp/sandbox/sb-bench-001",
        original_path="/tmp/original",
        resource_id="res-bench",
        plan_id="plan-bench",
        created_at=datetime.now(),
    )
    mock_sandbox.create.return_value = mock_sandbox.context
    mock_factory.create_sandbox.return_value = mock_sandbox
    return SandboxManager(factory=mock_factory, cleanup_on_exit=False)


class HandlerResolverSuite:
    """Benchmark handler resolver import and cache performance."""

    def time_resolve_git_handler_cold(self) -> None:
        """Benchmark cold resolve (no cache)."""
        clear_handler_cache()
        resolve_handler(
            "cleveragents.resource.handlers.git_checkout:GitCheckoutHandler"
        )

    def time_resolve_git_handler_cached(self) -> None:
        """Benchmark cached resolve."""
        resolve_handler(
            "cleveragents.resource.handlers.git_checkout:GitCheckoutHandler"
        )

    def time_resolve_fs_handler_cold(self) -> None:
        """Benchmark cold resolve for fs-directory."""
        clear_handler_cache()
        resolve_handler(
            "cleveragents.resource.handlers.fs_directory:FsDirectoryHandler"
        )


class GitCheckoutHandlerSuite:
    """Benchmark GitCheckoutHandler.resolve() overhead."""

    def setup(self) -> None:
        self.resource = _make_resource("git-checkout", "/tmp/bench-repo")
        self.manager = _make_mock_manager()
        self.handler = GitCheckoutHandler()

    def time_resolve_git_checkout(self) -> None:
        """Benchmark single git-checkout resolution."""
        self.handler.resolve(
            resource=self.resource,
            plan_id="PLAN_BENCH",
            slot_name="repo",
            sandbox_manager=self.manager,
        )


class FsDirectoryHandlerSuite:
    """Benchmark FsDirectoryHandler.resolve() overhead."""

    def setup(self) -> None:
        self.resource = _make_resource("fs-directory", "/tmp/bench-dir")
        self.manager = _make_mock_manager()
        self.handler = FsDirectoryHandler()

    def time_resolve_fs_directory(self) -> None:
        """Benchmark single fs-directory resolution."""
        self.handler.resolve(
            resource=self.resource,
            plan_id="PLAN_BENCH",
            slot_name="workdir",
            sandbox_manager=self.manager,
        )


class ResourceHandlerServiceSuite:
    """Benchmark ResourceHandlerService end-to-end resolution."""

    def setup(self) -> None:
        from cleveragents.application.services.resource_handler_service import (
            ResourceHandlerService,
        )

        self.resource = _make_resource("git-checkout", "/tmp/bench-repo")
        self.type_spec = ResourceTypeSpec(
            name="git-checkout",
            description="Test git-checkout type",
            resource_kind=ResourceKind.PHYSICAL,
            sandbox_strategy=TypeSandboxStrategy.GIT_WORKTREE,
            user_addable=True,
            built_in=True,
            handler="cleveragents.resource.handlers.git_checkout:GitCheckoutHandler",
        )
        self.manager = _make_mock_manager()
        self.service = ResourceHandlerService(
            sandbox_manager=self.manager,
            resource_lookup=lambda _: self.resource,
            type_lookup=lambda _: self.type_spec,
        )
        self.binding = BindingResult(
            slot_name="repo",
            resource_id="01KJ5C5TPMP8GGX3QC83E2MAQS",
            resource_name="test-repo",
            binding_mode="contextual",
        )

    def time_resolve_binding(self) -> None:
        """Benchmark single binding resolution."""
        self.service.resolve_binding(
            binding=self.binding,
            plan_id="PLAN_BENCH",
        )

    def time_resolve_bindings_batch(self) -> None:
        """Benchmark batch binding resolution (5 bindings)."""
        bindings = [
            BindingResult(
                slot_name=f"slot-{i}",
                resource_id="01KJ5C5TPMP8GGX3QC83E2MAQS",
                resource_name=f"res-{i}",
                binding_mode="contextual",
            )
            for i in range(5)
        ]
        self.service.resolve_bindings(
            bindings=bindings,
            plan_id="PLAN_BENCH",
        )