"""ASV benchmarks for Skill protocol type validation throughput. Measures the performance of: - SkillMetadata.from_skill() factory - SkillError creation - SkillResult creation - SkillDefinition creation with schema validation - map_tool_error() error mapping """ from __future__ import annotations import sys from pathlib import Path try: from cleveragents.domain.models.core.skill import ( Skill, SkillInlineTool, SkillResolver, ) from cleveragents.domain.models.core.tool import ( ToolCapability, ToolSource, ) from cleveragents.skills.protocol import ( SkillDefinition, SkillError, SkillErrorType, SkillMetadata, SkillResult, map_tool_error, ) except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.domain.models.core.skill import ( Skill, SkillInlineTool, SkillResolver, ) from cleveragents.domain.models.core.tool import ( ToolCapability, ToolSource, ) from cleveragents.skills.protocol import ( SkillDefinition, SkillError, SkillErrorType, SkillMetadata, SkillResult, map_tool_error, ) # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- _BASIC_SKILL = Skill( name="local/bench-skill", description="Benchmark skill", tool_refs=["local/tool-a", "local/tool-b"], ) _INLINE_SKILL = Skill( name="local/bench-inline", description="Benchmark inline skill", anonymous_tools=[ SkillInlineTool( description="Inline bench tool", source=ToolSource.CUSTOM, code="x = 1", capability=ToolCapability(read_only=True), ), ], ) _RESOLVER = SkillResolver() _RESOLVED_BASIC = _RESOLVER.resolve_tools(_BASIC_SKILL, {}) _RESOLVED_INLINE = _RESOLVER.resolve_tools(_INLINE_SKILL, {}) _METADATA_BASIC = SkillMetadata.from_skill(_BASIC_SKILL, resolved=_RESOLVED_BASIC) # --------------------------------------------------------------------------- # Benchmarks # --------------------------------------------------------------------------- class TimeSkillMetadataFromSkill: """Benchmark SkillMetadata.from_skill() throughput.""" timeout = 60 def time_from_skill_basic(self) -> None: """Time metadata creation from a basic skill.""" SkillMetadata.from_skill(_BASIC_SKILL, resolved=_RESOLVED_BASIC) def time_from_skill_inline(self) -> None: """Time metadata creation from a skill with inline tools.""" SkillMetadata.from_skill(_INLINE_SKILL, resolved=_RESOLVED_INLINE) class TimeSkillErrorCreation: """Benchmark SkillError creation throughput.""" timeout = 60 def time_create_error(self) -> None: """Time SkillError instantiation.""" SkillError( error_type=SkillErrorType.TOOL_EXECUTION_FAILURE, message="benchmark error", skill_name="local/bench-skill", details={"trace": "abc"}, ) class TimeSkillResultCreation: """Benchmark SkillResult creation throughput.""" timeout = 60 def time_create_success_result(self) -> None: """Time successful SkillResult instantiation.""" SkillResult( skill_name="local/bench-skill", tool_name="local/bench-tool", success=True, output_data={"value": 42}, duration_ms=10.5, ) def time_create_failure_result(self) -> None: """Time failed SkillResult instantiation.""" err = SkillError( error_type=SkillErrorType.TOOL_EXECUTION_FAILURE, message="benchmark error", skill_name="local/bench-skill", ) SkillResult( skill_name="local/bench-skill", tool_name="local/bench-tool", success=False, error=err, ) class TimeSkillDefinitionCreation: """Benchmark SkillDefinition creation throughput.""" timeout = 60 def time_create_definition(self) -> None: """Time SkillDefinition instantiation.""" SkillDefinition( skill=_BASIC_SKILL, resolved_tools=_RESOLVED_BASIC, metadata=_METADATA_BASIC, ) def time_create_definition_with_schemas(self) -> None: """Time SkillDefinition with input/output schemas.""" SkillDefinition( skill=_BASIC_SKILL, resolved_tools=_RESOLVED_BASIC, metadata=_METADATA_BASIC, input_schema={"type": "object", "properties": {"q": {"type": "string"}}}, output_schema={"type": "object", "properties": {"r": {"type": "string"}}}, ) class TimeMapToolError: """Benchmark map_tool_error() throughput.""" timeout = 60 def time_map_value_error(self) -> None: """Time mapping a ValueError.""" map_tool_error(ValueError("not found"), skill_name="local/bench-skill") def time_map_permission_error(self) -> None: """Time mapping a PermissionError.""" map_tool_error(PermissionError("denied"), skill_name="local/bench-skill") def time_map_runtime_error(self) -> None: """Time mapping a generic RuntimeError.""" map_tool_error( RuntimeError("crash"), skill_name="local/bench-skill", tool_name="local/bench-tool", )