cleveragents-core/benchmarks/tool_model_bench.py

"""ASV benchmarks for Tool and Validation domain model validation and serialization.

Measures the performance of:
- Tool model construction (Pydantic validation)
- Tool.as_cli_dict() serialization
- Tool.from_config() factory
- Validation model construction with forced constraints
- ResourceSlot construction
- ToolCapability construction
"""

from __future__ import annotations

import sys
from pathlib import Path

try:
    from cleveragents.domain.models.core.tool import (
        BindingMode,
        ResourceAccessMode,
        ResourceSlot,
        Tool,
        ToolCapability,
        ToolLifecycle,
        ToolSource,
        Validation,
        ValidationMode,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.domain.models.core.tool import (
        BindingMode,
        ResourceAccessMode,
        ResourceSlot,
        Tool,
        ToolCapability,
        ToolLifecycle,
        ToolSource,
        Validation,
        ValidationMode,
    )


def _make_tool() -> Tool:
    """Create a fully-populated Tool for benchmarking."""
    return Tool(
        name="bench/tool-test",
        description="Benchmark tool for validation and serialization",
        source=ToolSource.CUSTOM,
        code="return {'result': True}",
        input_schema={"type": "object", "properties": {"file": {"type": "string"}}},
        output_schema={"type": "object", "properties": {"result": {"type": "boolean"}}},
        capability=ToolCapability(
            writes=True,
            write_scope="file",
            checkpointable=True,
            side_effects=["filesystem"],
            idempotent=False,
        ),
        resource_slots=[
            ResourceSlot(
                name="repo",
                resource_type="git-checkout",
                access=ResourceAccessMode.READ_ONLY,
                binding=BindingMode.CONTEXTUAL,
            ),
        ],
        lifecycle=ToolLifecycle(
            discover="bench.discover",
            activate="bench.activate",
        ),
        timeout=120,
    )


def _make_validation() -> Validation:
    """Create a fully-populated Validation for benchmarking."""
    return Validation(
        name="bench/val-test",
        description="Benchmark validation",
        source=ToolSource.WRAPPED,
        wraps="bench/tool-test",
        transform="def transform(r): return {'passed': True}",
        argument_mapping={"strict": True},
        mode=ValidationMode.REQUIRED,
    )


class ToolValidationSuite:
    """Benchmark Tool model construction (Pydantic validation)."""

    def time_tool_construction(self) -> None:
        """Benchmark creating a fully-populated Tool object."""
        _make_tool()

    def time_tool_minimal_construction(self) -> None:
        """Benchmark creating a minimal Tool object."""
        Tool(
            name="bench/minimal",
            description="Minimal tool",
            source=ToolSource.BUILTIN,
        )

    def time_validation_construction(self) -> None:
        """Benchmark creating a Validation with forced constraints."""
        _make_validation()

    def time_resource_slot_construction(self) -> None:
        """Benchmark ResourceSlot construction."""
        ResourceSlot(
            name="repo",
            resource_type="git-checkout",
            access=ResourceAccessMode.READ_ONLY,
            binding=BindingMode.CONTEXTUAL,
        )

    def time_capability_construction(self) -> None:
        """Benchmark ToolCapability construction."""
        ToolCapability(
            read_only=True,
            writes=False,
            checkpointable=False,
            idempotent=True,
        )


class ToolSerializationSuite:
    """Benchmark Tool serialization methods."""

    def setup(self) -> None:
        """Create objects for serialization benchmarks."""
        self.tool = _make_tool()
        self.validation = _make_validation()

    def time_tool_as_cli_dict(self) -> None:
        """Benchmark Tool.as_cli_dict() ordered dict generation."""
        self.tool.as_cli_dict()

    def time_validation_as_cli_dict(self) -> None:
        """Benchmark Validation.as_cli_dict() ordered dict generation."""
        self.validation.as_cli_dict()

    def time_tool_model_dump(self) -> None:
        """Benchmark Pydantic model_dump() serialization."""
        self.tool.model_dump()

    def time_tool_model_dump_json(self) -> None:
        """Benchmark Pydantic model_dump_json() JSON serialization."""
        self.tool.model_dump_json()


class ToolFromConfigSuite:
    """Benchmark Tool.from_config() factory."""

    def setup(self) -> None:
        """Prepare config dicts for benchmarks."""
        self.tool_config = {
            "name": "bench/from-config",
            "description": "Config benchmark",
            "source": "builtin",
            "capability": {"read_only": True},
            "resource_slots": [
                {
                    "name": "repo",
                    "resource_type": "git-checkout",
                    "access": "read_only",
                },
            ],
        }
        self.validation_config = {
            "name": "bench/val-config",
            "description": "Config benchmark validation",
            "wraps": "bench/tool-test",
            "transform": "def transform(r): return r",
            "mode": "required",
        }

    def time_tool_from_config(self) -> None:
        """Benchmark Tool.from_config()."""
        Tool.from_config(self.tool_config)

    def time_validation_from_config(self) -> None:
        """Benchmark Validation.from_config()."""
        Validation.from_config(self.validation_config)