feat(skill): add skill protocol and metadata

2026-02-20 03:59:51 +00:00
parent 3bd02a7c6e
commit b18bace06b
9 changed files with 1957 additions and 24 deletions
@@ -0,0 +1,192 @@
+"""ASV benchmarks for Skill protocol type validation throughput.
+
+Measures the performance of:
+- SkillMetadata.from_skill() factory
+- SkillError creation
+- SkillResult creation
+- SkillDefinition creation with schema validation
+- map_tool_error() error mapping
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+try:
+    from cleveragents.domain.models.core.skill import (
+        Skill,
+        SkillInlineTool,
+        SkillResolver,
+    )
+    from cleveragents.domain.models.core.tool import (
+        ToolCapability,
+        ToolSource,
+    )
+    from cleveragents.skills.protocol import (
+        SkillDefinition,
+        SkillError,
+        SkillErrorType,
+        SkillMetadata,
+        SkillResult,
+        map_tool_error,
+    )
+except ModuleNotFoundError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+    from cleveragents.domain.models.core.skill import (
+        Skill,
+        SkillInlineTool,
+        SkillResolver,
+    )
+    from cleveragents.domain.models.core.tool import (
+        ToolCapability,
+        ToolSource,
+    )
+    from cleveragents.skills.protocol import (
+        SkillDefinition,
+        SkillError,
+        SkillErrorType,
+        SkillMetadata,
+        SkillResult,
+        map_tool_error,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+_BASIC_SKILL = Skill(
+    name="local/bench-skill",
+    description="Benchmark skill",
+    tool_refs=["local/tool-a", "local/tool-b"],
+)
+
+_INLINE_SKILL = Skill(
+    name="local/bench-inline",
+    description="Benchmark inline skill",
+    anonymous_tools=[
+        SkillInlineTool(
+            description="Inline bench tool",
+            source=ToolSource.CUSTOM,
+            code="x = 1",
+            capability=ToolCapability(read_only=True),
+        ),
+    ],
+)
+
+_RESOLVER = SkillResolver()
+_RESOLVED_BASIC = _RESOLVER.resolve_tools(_BASIC_SKILL, {})
+_RESOLVED_INLINE = _RESOLVER.resolve_tools(_INLINE_SKILL, {})
+
+_METADATA_BASIC = SkillMetadata.from_skill(_BASIC_SKILL, resolved=_RESOLVED_BASIC)
+
+
+# ---------------------------------------------------------------------------
+# Benchmarks
+# ---------------------------------------------------------------------------
+
+
+class TimeSkillMetadataFromSkill:
+    """Benchmark SkillMetadata.from_skill() throughput."""
+
+    timeout = 60
+
+    def time_from_skill_basic(self) -> None:
+        """Time metadata creation from a basic skill."""
+        SkillMetadata.from_skill(_BASIC_SKILL, resolved=_RESOLVED_BASIC)
+
+    def time_from_skill_inline(self) -> None:
+        """Time metadata creation from a skill with inline tools."""
+        SkillMetadata.from_skill(_INLINE_SKILL, resolved=_RESOLVED_INLINE)
+
+
+class TimeSkillErrorCreation:
+    """Benchmark SkillError creation throughput."""
+
+    timeout = 60
+
+    def time_create_error(self) -> None:
+        """Time SkillError instantiation."""
+        SkillError(
+            error_type=SkillErrorType.TOOL_EXECUTION_FAILURE,
+            message="benchmark error",
+            skill_name="local/bench-skill",
+            details={"trace": "abc"},
+        )
+
+
+class TimeSkillResultCreation:
+    """Benchmark SkillResult creation throughput."""
+
+    timeout = 60
+
+    def time_create_success_result(self) -> None:
+        """Time successful SkillResult instantiation."""
+        SkillResult(
+            skill_name="local/bench-skill",
+            tool_name="local/bench-tool",
+            success=True,
+            output_data={"value": 42},
+            duration_ms=10.5,
+        )
+
+    def time_create_failure_result(self) -> None:
+        """Time failed SkillResult instantiation."""
+        err = SkillError(
+            error_type=SkillErrorType.TOOL_EXECUTION_FAILURE,
+            message="benchmark error",
+            skill_name="local/bench-skill",
+        )
+        SkillResult(
+            skill_name="local/bench-skill",
+            tool_name="local/bench-tool",
+            success=False,
+            error=err,
+        )
+
+
+class TimeSkillDefinitionCreation:
+    """Benchmark SkillDefinition creation throughput."""
+
+    timeout = 60
+
+    def time_create_definition(self) -> None:
+        """Time SkillDefinition instantiation."""
+        SkillDefinition(
+            skill=_BASIC_SKILL,
+            resolved_tools=_RESOLVED_BASIC,
+            metadata=_METADATA_BASIC,
+        )
+
+    def time_create_definition_with_schemas(self) -> None:
+        """Time SkillDefinition with input/output schemas."""
+        SkillDefinition(
+            skill=_BASIC_SKILL,
+            resolved_tools=_RESOLVED_BASIC,
+            metadata=_METADATA_BASIC,
+            input_schema={"type": "object", "properties": {"q": {"type": "string"}}},
+            output_schema={"type": "object", "properties": {"r": {"type": "string"}}},
+        )
+
+
+class TimeMapToolError:
+    """Benchmark map_tool_error() throughput."""
+
+    timeout = 60
+
+    def time_map_value_error(self) -> None:
+        """Time mapping a ValueError."""
+        map_tool_error(ValueError("not found"), skill_name="local/bench-skill")
+
+    def time_map_permission_error(self) -> None:
+        """Time mapping a PermissionError."""
+        map_tool_error(PermissionError("denied"), skill_name="local/bench-skill")
+
+    def time_map_runtime_error(self) -> None:
+        """Time mapping a generic RuntimeError."""
+        map_tool_error(
+            RuntimeError("crash"),
+            skill_name="local/bench-skill",
+            tool_name="local/bench-tool",
+        )