"""Step definitions for Skill protocol types tests.""" from typing import Any from behave import then, when from behave.runner import Context from pydantic import ValidationError from cleveragents.domain.models.core.skill import ( Skill, SkillAgentSource, SkillInlineTool, SkillMcpSource, SkillResolver, ) from cleveragents.domain.models.core.tool import ( ToolCapability, ToolSource, ) from cleveragents.skills.protocol import ( SkillDefinition, SkillError, SkillErrorType, SkillMetadata, SkillResult, map_tool_error, ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_skill(**overrides: Any) -> Skill: """Create a Skill with sensible defaults, allowing overrides.""" defaults: dict[str, Any] = { "name": "local/test-skill", "description": "A test skill", } defaults.update(overrides) return Skill(**defaults) def _make_metadata_skill_with_write_tools() -> Skill: """Create a skill with inline write-capable tools.""" return Skill( name="local/write-skill", description="Skill with writes", anonymous_tools=[ SkillInlineTool( description="Write tool", source=ToolSource.CUSTOM, code="x = 1", capability=ToolCapability( writes=True, side_effects=["filesystem"], ), ), ], ) def _make_metadata_skill_with_read_only_tools() -> Skill: """Create a skill with only read-only inline tools.""" return Skill( name="local/readonly-skill", description="Read only skill", anonymous_tools=[ SkillInlineTool( description="Read tool", source=ToolSource.CUSTOM, code="x = 1", capability=ToolCapability(read_only=True), ), ], ) # --------------------------------------------------------------------------- # SkillErrorType Enum # --------------------------------------------------------------------------- @when("I list all skill_protocol error types") def skill_protocol_list_error_types(context: Context) -> None: """Collect all SkillErrorType members.""" context.skill_protocol_error_types = list(SkillErrorType) @then('the skill_protocol error types should include "{value}"') def skill_protocol_check_error_type_member(context: Context, value: str) -> None: """Check that a specific error type is in the list.""" values = [e.value for e in context.skill_protocol_error_types] assert value in values, f"Expected '{value}' in {values}" @then("there should be {count:d} skill_protocol error types") def skill_protocol_check_error_type_count(context: Context, count: int) -> None: """Check total number of error types.""" actual = len(context.skill_protocol_error_types) assert actual == count, f"Expected {count} error types, got {actual}" # --------------------------------------------------------------------------- # SkillError Creation # --------------------------------------------------------------------------- @when( 'I create a skill_protocol error with type "{err_type}" and message "{msg}" for skill "{skill_name}"' ) def skill_protocol_create_error( context: Context, err_type: str, msg: str, skill_name: str ) -> None: """Create a SkillError.""" context.skill_protocol_error = SkillError( error_type=SkillErrorType(err_type), message=msg, skill_name=skill_name, ) @when('I create a skill_protocol error with tool_name "{tool_name}"') def skill_protocol_create_error_with_tool(context: Context, tool_name: str) -> None: """Create a SkillError with tool_name.""" context.skill_protocol_error = SkillError( error_type=SkillErrorType.TOOL_EXECUTION_FAILURE, message="Tool failed", skill_name="local/test-skill", tool_name=tool_name, ) @when("I create a skill_protocol error with details") def skill_protocol_create_error_with_details(context: Context) -> None: """Create a SkillError with details dict.""" context.skill_protocol_error = SkillError( error_type=SkillErrorType.TOOL_EXECUTION_FAILURE, message="Tool failed", skill_name="local/test-skill", details={"trace_id": "abc-123", "attempt": 1}, ) @then("the skill_protocol error should be created") def skill_protocol_check_error_created(context: Context) -> None: """Verify error was created.""" assert context.skill_protocol_error is not None @then('the skill_protocol error type should be "{expected}"') def skill_protocol_check_error_type(context: Context, expected: str) -> None: """Check error type value.""" assert context.skill_protocol_error.error_type.value == expected @then('the skill_protocol error message should be "{expected}"') def skill_protocol_check_error_message(context: Context, expected: str) -> None: """Check error message.""" assert context.skill_protocol_error.message == expected @then('the skill_protocol error skill_name should be "{expected}"') def skill_protocol_check_error_skill_name(context: Context, expected: str) -> None: """Check error skill_name.""" assert context.skill_protocol_error.skill_name == expected @then("the skill_protocol error tool_name should be none") def skill_protocol_check_error_tool_none(context: Context) -> None: """Check error tool_name is None.""" assert context.skill_protocol_error.tool_name is None @then('the skill_protocol error tool_name should be "{expected}"') def skill_protocol_check_error_tool_name(context: Context, expected: str) -> None: """Check error tool_name value.""" assert context.skill_protocol_error.tool_name == expected @then('the skill_protocol error details should have key "{key}"') def skill_protocol_check_error_details_key(context: Context, key: str) -> None: """Check error details contains a key.""" assert key in context.skill_protocol_error.details @then("the skill_protocol error should be immutable") def skill_protocol_check_error_immutable(context: Context) -> None: """Verify SkillError is frozen.""" try: context.skill_protocol_error.message = "changed" raise AssertionError("Expected frozen model to reject assignment") except ValidationError: pass # --------------------------------------------------------------------------- # SkillMetadata Creation # --------------------------------------------------------------------------- @when("I create a skill_protocol metadata from a basic skill") def skill_protocol_create_metadata_basic(context: Context) -> None: """Create SkillMetadata from a basic skill.""" skill = _make_skill() context.skill_protocol_metadata = SkillMetadata.from_skill(skill) @when("I create a skill_protocol metadata from a skill with tool refs") def skill_protocol_create_metadata_refs(context: Context) -> None: """Create SkillMetadata from a skill with tool refs.""" skill = _make_skill(tool_refs=["local/edit-file", "local/read-file"]) context.skill_protocol_metadata = SkillMetadata.from_skill(skill) @when("I create a skill_protocol metadata from a skill with write tools") def skill_protocol_create_metadata_write(context: Context) -> None: """Create SkillMetadata from a skill with write tools.""" skill = _make_metadata_skill_with_write_tools() context.skill_protocol_metadata = SkillMetadata.from_skill(skill) @when("I create a skill_protocol metadata from a skill with mcp sources") def skill_protocol_create_metadata_mcp(context: Context) -> None: """Create SkillMetadata from a skill with MCP sources.""" skill = Skill( name="local/mcp-skill", description="MCP skill", mcp_servers=[SkillMcpSource(server="npx @mcp/fs", tools=["read"])], ) context.skill_protocol_metadata = SkillMetadata.from_skill(skill) @when("I create a skill_protocol metadata from a skill with agent skill sources") def skill_protocol_create_metadata_agent(context: Context) -> None: """Create SkillMetadata from a skill with agent skill sources.""" skill = Skill( name="local/agent-skill", description="Agent skill", agent_skills=[SkillAgentSource(path="./skills/review")], ) context.skill_protocol_metadata = SkillMetadata.from_skill(skill) @when('I create a skill_protocol metadata with version "{version}"') def skill_protocol_create_metadata_version(context: Context, version: str) -> None: """Create SkillMetadata with a specific version.""" skill = _make_skill() context.skill_protocol_metadata = SkillMetadata.from_skill(skill, version=version) @when("I create a skill_protocol metadata from a skill with read_only tools") def skill_protocol_create_metadata_readonly(context: Context) -> None: """Create SkillMetadata from a skill with read-only tools.""" skill = _make_metadata_skill_with_read_only_tools() context.skill_protocol_metadata = SkillMetadata.from_skill(skill) @when("I create a skill_protocol metadata from an empty skill") def skill_protocol_create_metadata_empty(context: Context) -> None: """Create SkillMetadata from an empty skill.""" skill = _make_skill() context.skill_protocol_metadata = SkillMetadata.from_skill(skill) @when("I try to create skill_protocol metadata from None") def skill_protocol_create_metadata_none(context: Context) -> None: """Try to create SkillMetadata from None.""" context.skill_protocol_value_error = None try: SkillMetadata.from_skill(None) # type: ignore[arg-type] except ValueError as e: context.skill_protocol_value_error = e @then("the skill_protocol metadata should be created") def skill_protocol_check_metadata_created(context: Context) -> None: """Verify metadata was created.""" assert context.skill_protocol_metadata is not None @then('the skill_protocol metadata name should be "{expected}"') def skill_protocol_check_metadata_name(context: Context, expected: str) -> None: """Check metadata name.""" assert context.skill_protocol_metadata.name == expected @then('the skill_protocol metadata description should be "{expected}"') def skill_protocol_check_metadata_desc(context: Context, expected: str) -> None: """Check metadata description.""" assert context.skill_protocol_metadata.description == expected @then('the skill_protocol metadata version should be "{expected}"') def skill_protocol_check_metadata_version(context: Context, expected: str) -> None: """Check metadata version.""" assert context.skill_protocol_metadata.version == expected @then("the skill_protocol metadata read_only should be true") def skill_protocol_check_metadata_readonly_true(context: Context) -> None: """Check metadata read_only is True.""" assert context.skill_protocol_metadata.read_only is True @then("the skill_protocol metadata read_only should be false") def skill_protocol_check_metadata_readonly_false(context: Context) -> None: """Check metadata read_only is False.""" assert context.skill_protocol_metadata.read_only is False @then("the skill_protocol metadata writes should be true") def skill_protocol_check_metadata_writes_true(context: Context) -> None: """Check metadata writes is True.""" assert context.skill_protocol_metadata.writes is True @then("the skill_protocol metadata writes should be false") def skill_protocol_check_metadata_writes_false(context: Context) -> None: """Check metadata writes is False.""" assert context.skill_protocol_metadata.writes is False @then("the skill_protocol metadata tool_count should be {expected:d}") def skill_protocol_check_metadata_tool_count(context: Context, expected: int) -> None: """Check metadata tool_count.""" actual = context.skill_protocol_metadata.tool_count assert actual == expected, f"Expected {expected} tools, got {actual}" @then('the skill_protocol metadata source_types should contain "{expected}"') def skill_protocol_check_metadata_source_type(context: Context, expected: str) -> None: """Check metadata source_types contains a value.""" assert expected in context.skill_protocol_metadata.source_types, ( f"Expected '{expected}' in {context.skill_protocol_metadata.source_types}" ) @then("the skill_protocol metadata should be immutable") def skill_protocol_check_metadata_immutable(context: Context) -> None: """Verify SkillMetadata is frozen.""" try: context.skill_protocol_metadata.name = "changed" raise AssertionError("Expected frozen model to reject assignment") except ValidationError: pass # --------------------------------------------------------------------------- # SkillDefinition # --------------------------------------------------------------------------- @when("I create a skill_protocol definition with resolved tools") def skill_protocol_create_definition(context: Context) -> None: """Create a SkillDefinition with resolved tools.""" skill = Skill( name="local/def-skill", description="Definition skill", tool_refs=["local/my-tool"], ) resolver = SkillResolver() resolved = resolver.resolve_tools(skill, {}) metadata = SkillMetadata.from_skill(skill, resolved=resolved) context.skill_protocol_definition = SkillDefinition( skill=skill, resolved_tools=resolved, metadata=metadata, ) @when("I create a skill_protocol definition with input and output schemas") def skill_protocol_create_definition_schemas(context: Context) -> None: """Create a SkillDefinition with schemas.""" skill = Skill( name="local/schema-skill", description="Schema skill", tool_refs=["local/my-tool"], ) resolver = SkillResolver() resolved = resolver.resolve_tools(skill, {}) metadata = SkillMetadata.from_skill(skill, resolved=resolved) context.skill_protocol_definition = SkillDefinition( skill=skill, resolved_tools=resolved, metadata=metadata, input_schema={"type": "object", "properties": {"query": {"type": "string"}}}, output_schema={"type": "object", "properties": {"result": {"type": "string"}}}, ) @when("I try to create a skill_protocol definition with invalid input_schema") def skill_protocol_create_definition_invalid_input(context: Context) -> None: """Try to create a SkillDefinition with invalid input_schema.""" skill = _make_skill() metadata = SkillMetadata.from_skill(skill) context.skill_protocol_validation_error = None try: SkillDefinition( skill=skill, resolved_tools=[], metadata=metadata, input_schema={"properties": {"x": {"type": "string"}}}, # missing "type" ) except ValidationError as e: context.skill_protocol_validation_error = e @when("I try to create a skill_protocol definition with invalid output_schema") def skill_protocol_create_definition_invalid_output(context: Context) -> None: """Try to create a SkillDefinition with invalid output_schema.""" skill = _make_skill() metadata = SkillMetadata.from_skill(skill) context.skill_protocol_validation_error = None try: SkillDefinition( skill=skill, resolved_tools=[], metadata=metadata, output_schema={"description": "no type key"}, ) except ValidationError as e: context.skill_protocol_validation_error = e @when("I try to create a skill_protocol definition with writes inconsistency") def skill_protocol_create_definition_writes_inconsistency(context: Context) -> None: """Try to create a SkillDefinition where metadata says read_only but tools have writes.""" skill = Skill( name="local/inconsistent", description="Inconsistent skill", anonymous_tools=[ SkillInlineTool( description="Writer", source=ToolSource.CUSTOM, code="x = 1", capability=ToolCapability(writes=True, side_effects=["fs"]), ), ], ) # Build resolved tools -- they include a write-capable inline tool resolver = SkillResolver() resolved = resolver.resolve_tools(skill, {}) # Force metadata to claim read_only=True (bypassing from_skill) metadata = SkillMetadata( name=skill.name, description=skill.description, tool_count=len(resolved), writes=False, read_only=True, ) context.skill_protocol_validation_error = None try: SkillDefinition( skill=skill, resolved_tools=resolved, metadata=metadata, ) except ValidationError as e: context.skill_protocol_validation_error = e @then("the skill_protocol definition should be created") def skill_protocol_check_definition_created(context: Context) -> None: """Verify definition was created.""" assert context.skill_protocol_definition is not None @then('the skill_protocol definition skill name should be "{expected}"') def skill_protocol_check_definition_name(context: Context, expected: str) -> None: """Check definition skill name.""" assert context.skill_protocol_definition.skill.name == expected @then("the skill_protocol definition should have {count:d} resolved tools") def skill_protocol_check_definition_tool_count(context: Context, count: int) -> None: """Check definition resolved_tools count.""" actual = len(context.skill_protocol_definition.resolved_tools) assert actual == count, f"Expected {count}, got {actual}" @then('the skill_protocol definition input_schema should have key "{key}"') def skill_protocol_check_definition_input_key(context: Context, key: str) -> None: """Check definition input_schema has key.""" assert context.skill_protocol_definition.input_schema is not None assert key in context.skill_protocol_definition.input_schema @then('the skill_protocol definition output_schema should have key "{key}"') def skill_protocol_check_definition_output_key(context: Context, key: str) -> None: """Check definition output_schema has key.""" assert context.skill_protocol_definition.output_schema is not None assert key in context.skill_protocol_definition.output_schema @then("a skill_protocol validation error should be raised") def skill_protocol_check_validation_error(context: Context) -> None: """Verify a validation error was raised.""" assert context.skill_protocol_validation_error is not None, ( "Expected a validation error" ) @then("the skill_protocol definition should be immutable") def skill_protocol_check_definition_immutable(context: Context) -> None: """Verify SkillDefinition is frozen.""" try: context.skill_protocol_definition.input_schema = {"type": "object"} raise AssertionError("Expected frozen model to reject assignment") except ValidationError: pass # --------------------------------------------------------------------------- # SkillResult # --------------------------------------------------------------------------- @when("I create a skill_protocol result with success") def skill_protocol_create_result_success(context: Context) -> None: """Create a successful SkillResult.""" context.skill_protocol_result = SkillResult( skill_name="local/my-skill", tool_name="local/my-tool", success=True, output_data="hello", ) @when("I create a skill_protocol result with failure") def skill_protocol_create_result_failure(context: Context) -> None: """Create a failed SkillResult.""" err = SkillError( error_type=SkillErrorType.TOOL_EXECUTION_FAILURE, message="Tool timed out", skill_name="local/my-skill", tool_name="local/my-tool", ) context.skill_protocol_result = SkillResult( skill_name="local/my-skill", tool_name="local/my-tool", success=False, error=err, ) @when("I create a skill_protocol result with duration and resource changes") def skill_protocol_create_result_duration(context: Context) -> None: """Create a SkillResult with duration and resource changes.""" context.skill_protocol_result = SkillResult( skill_name="local/my-skill", tool_name="local/my-tool", success=True, duration_ms=150.5, resource_changes=[{"type": "file_modified", "path": "/tmp/test.txt"}], ) @then("the skill_protocol result should be created") def skill_protocol_check_result_created(context: Context) -> None: """Verify result was created.""" assert context.skill_protocol_result is not None @then("the skill_protocol result success should be true") def skill_protocol_check_result_success_true(context: Context) -> None: """Check result success is True.""" assert context.skill_protocol_result.success is True @then("the skill_protocol result success should be false") def skill_protocol_check_result_success_false(context: Context) -> None: """Check result success is False.""" assert context.skill_protocol_result.success is False @then('the skill_protocol result skill_name should be "{expected}"') def skill_protocol_check_result_skill(context: Context, expected: str) -> None: """Check result skill_name.""" assert context.skill_protocol_result.skill_name == expected @then('the skill_protocol result tool_name should be "{expected}"') def skill_protocol_check_result_tool(context: Context, expected: str) -> None: """Check result tool_name.""" assert context.skill_protocol_result.tool_name == expected @then("the skill_protocol result error should be none") def skill_protocol_check_result_error_none(context: Context) -> None: """Check result error is None.""" assert context.skill_protocol_result.error is None @then("the skill_protocol result error should not be none") def skill_protocol_check_result_error_present(context: Context) -> None: """Check result error is not None.""" assert context.skill_protocol_result.error is not None @then('the skill_protocol result error type should be "{expected}"') def skill_protocol_check_result_error_type(context: Context, expected: str) -> None: """Check result error type.""" assert context.skill_protocol_result.error is not None assert context.skill_protocol_result.error.error_type.value == expected @then('the skill_protocol result output_data should be "{expected}"') def skill_protocol_check_result_output(context: Context, expected: str) -> None: """Check result output_data.""" assert context.skill_protocol_result.output_data == expected @then("the skill_protocol result duration_ms should be {expected:g}") def skill_protocol_check_result_duration(context: Context, expected: float) -> None: """Check result duration_ms.""" assert context.skill_protocol_result.duration_ms == expected @then("the skill_protocol result should have {count:d} resource changes") def skill_protocol_check_result_changes(context: Context, count: int) -> None: """Check result resource_changes count.""" actual = len(context.skill_protocol_result.resource_changes) assert actual == count, f"Expected {count}, got {actual}" @then("the skill_protocol result should be immutable") def skill_protocol_check_result_immutable(context: Context) -> None: """Verify SkillResult is frozen.""" try: context.skill_protocol_result.success = False raise AssertionError("Expected frozen model to reject assignment") except ValidationError: pass # --------------------------------------------------------------------------- # Error Mapping # --------------------------------------------------------------------------- @when('I map a skill_protocol ValueError with message "{msg}"') def skill_protocol_map_value_error(context: Context, msg: str) -> None: """Map a ValueError through map_tool_error.""" context.skill_protocol_mapped_error = map_tool_error( ValueError(msg), skill_name="local/test-skill" ) @when("I map a skill_protocol PermissionError") def skill_protocol_map_permission_error(context: Context) -> None: """Map a PermissionError through map_tool_error.""" context.skill_protocol_mapped_error = map_tool_error( PermissionError("Access denied"), skill_name="local/test-skill" ) @when('I map a skill_protocol RuntimeError with message "{msg}"') def skill_protocol_map_runtime_error(context: Context, msg: str) -> None: """Map a RuntimeError through map_tool_error.""" context.skill_protocol_mapped_error = map_tool_error( RuntimeError(msg), skill_name="local/test-skill" ) @when('I map a skill_protocol error with tool_name "{tool_name}"') def skill_protocol_map_with_tool(context: Context, tool_name: str) -> None: """Map an error with tool_name.""" context.skill_protocol_mapped_error = map_tool_error( RuntimeError("fail"), skill_name="local/test-skill", tool_name=tool_name, ) @when("I try to map a skill_protocol error with empty skill_name") def skill_protocol_map_empty_skill_name(context: Context) -> None: """Try to map an error with empty skill_name.""" context.skill_protocol_value_error = None try: map_tool_error(RuntimeError("fail"), skill_name="") except ValueError as e: context.skill_protocol_value_error = e @then('the skill_protocol mapped error type should be "{expected}"') def skill_protocol_check_mapped_type(context: Context, expected: str) -> None: """Check mapped error type.""" assert context.skill_protocol_mapped_error.error_type.value == expected @then('the skill_protocol mapped error tool_name should be "{expected}"') def skill_protocol_check_mapped_tool(context: Context, expected: str) -> None: """Check mapped error tool_name.""" assert context.skill_protocol_mapped_error.tool_name == expected @then('the skill_protocol mapped error details should have key "{key}"') def skill_protocol_check_mapped_details(context: Context, key: str) -> None: """Check mapped error details has key.""" assert key in context.skill_protocol_mapped_error.details @then("a skill_protocol value error should be raised") def skill_protocol_check_value_error(context: Context) -> None: """Verify a ValueError was raised.""" assert context.skill_protocol_value_error is not None, "Expected a ValueError"