cleveragents-core/benchmarks/skill_protocol_bench.py

"""ASV benchmarks for Skill protocol type validation throughput.

Measures the performance of:
- SkillMetadata.from_skill() factory
- SkillError creation
- SkillResult creation
- SkillDefinition creation with schema validation
- map_tool_error() error mapping
"""

from __future__ import annotations

import sys
from pathlib import Path

try:
    from cleveragents.domain.models.core.skill import (
        Skill,
        SkillInlineTool,
        SkillResolver,
    )
    from cleveragents.domain.models.core.tool import (
        ToolCapability,
        ToolSource,
    )
    from cleveragents.skills.protocol import (
        SkillDefinition,
        SkillError,
        SkillErrorType,
        SkillMetadata,
        SkillResult,
        map_tool_error,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.domain.models.core.skill import (
        Skill,
        SkillInlineTool,
        SkillResolver,
    )
    from cleveragents.domain.models.core.tool import (
        ToolCapability,
        ToolSource,
    )
    from cleveragents.skills.protocol import (
        SkillDefinition,
        SkillError,
        SkillErrorType,
        SkillMetadata,
        SkillResult,
        map_tool_error,
    )


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------

_BASIC_SKILL = Skill(
    name="local/bench-skill",
    description="Benchmark skill",
    tool_refs=["local/tool-a", "local/tool-b"],
)

_INLINE_SKILL = Skill(
    name="local/bench-inline",
    description="Benchmark inline skill",
    anonymous_tools=[
        SkillInlineTool(
            description="Inline bench tool",
            source=ToolSource.CUSTOM,
            code="x = 1",
            capability=ToolCapability(read_only=True),
        ),
    ],
)

_RESOLVER = SkillResolver()
_RESOLVED_BASIC = _RESOLVER.resolve_tools(_BASIC_SKILL, {})
_RESOLVED_INLINE = _RESOLVER.resolve_tools(_INLINE_SKILL, {})

_METADATA_BASIC = SkillMetadata.from_skill(_BASIC_SKILL, resolved=_RESOLVED_BASIC)


# ---------------------------------------------------------------------------
# Benchmarks
# ---------------------------------------------------------------------------


class TimeSkillMetadataFromSkill:
    """Benchmark SkillMetadata.from_skill() throughput."""

    timeout = 60

    def time_from_skill_basic(self) -> None:
        """Time metadata creation from a basic skill."""
        SkillMetadata.from_skill(_BASIC_SKILL, resolved=_RESOLVED_BASIC)

    def time_from_skill_inline(self) -> None:
        """Time metadata creation from a skill with inline tools."""
        SkillMetadata.from_skill(_INLINE_SKILL, resolved=_RESOLVED_INLINE)


class TimeSkillErrorCreation:
    """Benchmark SkillError creation throughput."""

    timeout = 60

    def time_create_error(self) -> None:
        """Time SkillError instantiation."""
        SkillError(
            error_type=SkillErrorType.TOOL_EXECUTION_FAILURE,
            message="benchmark error",
            skill_name="local/bench-skill",
            details={"trace": "abc"},
        )


class TimeSkillResultCreation:
    """Benchmark SkillResult creation throughput."""

    timeout = 60

    def time_create_success_result(self) -> None:
        """Time successful SkillResult instantiation."""
        SkillResult(
            skill_name="local/bench-skill",
            tool_name="local/bench-tool",
            success=True,
            output_data={"value": 42},
            duration_ms=10.5,
        )

    def time_create_failure_result(self) -> None:
        """Time failed SkillResult instantiation."""
        err = SkillError(
            error_type=SkillErrorType.TOOL_EXECUTION_FAILURE,
            message="benchmark error",
            skill_name="local/bench-skill",
        )
        SkillResult(
            skill_name="local/bench-skill",
            tool_name="local/bench-tool",
            success=False,
            error=err,
        )


class TimeSkillDefinitionCreation:
    """Benchmark SkillDefinition creation throughput."""

    timeout = 60

    def time_create_definition(self) -> None:
        """Time SkillDefinition instantiation."""
        SkillDefinition(
            skill=_BASIC_SKILL,
            resolved_tools=_RESOLVED_BASIC,
            metadata=_METADATA_BASIC,
        )

    def time_create_definition_with_schemas(self) -> None:
        """Time SkillDefinition with input/output schemas."""
        SkillDefinition(
            skill=_BASIC_SKILL,
            resolved_tools=_RESOLVED_BASIC,
            metadata=_METADATA_BASIC,
            input_schema={"type": "object", "properties": {"q": {"type": "string"}}},
            output_schema={"type": "object", "properties": {"r": {"type": "string"}}},
        )


class TimeMapToolError:
    """Benchmark map_tool_error() throughput."""

    timeout = 60

    def time_map_value_error(self) -> None:
        """Time mapping a ValueError."""
        map_tool_error(ValueError("not found"), skill_name="local/bench-skill")

    def time_map_permission_error(self) -> None:
        """Time mapping a PermissionError."""
        map_tool_error(PermissionError("denied"), skill_name="local/bench-skill")

    def time_map_runtime_error(self) -> None:
        """Time mapping a generic RuntimeError."""
        map_tool_error(
            RuntimeError("crash"),
            skill_name="local/bench-skill",
            tool_name="local/bench-tool",
        )