"""Step definitions for Tool and Validation domain model tests.""" from typing import Any from behave import then, when from behave.runner import Context from pydantic import ValidationError from cleveragents.domain.models.core.tool import ( BindingMode, CheckpointScope, ResourceAccessMode, ResourceSlot, Tool, ToolCapability, ToolLifecycle, ToolSource, ToolType, Validation, ValidationMode, ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_tool(**overrides: Any) -> Tool: """Create a Tool with sensible defaults, allowing overrides.""" defaults: dict[str, Any] = { "name": "local/test-tool", "description": "A test tool", "source": ToolSource.BUILTIN, } defaults.update(overrides) return Tool(**defaults) def _tool_config(**overrides: Any) -> dict[str, Any]: """Return a minimal valid config dict for Tool.from_config.""" cfg: dict[str, Any] = { "name": "local/my-tool", "description": "A tool from config", "source": "builtin", } cfg.update(overrides) return cfg def _validation_config(**overrides: Any) -> dict[str, Any]: """Return a minimal valid config dict for Validation.from_config.""" cfg: dict[str, Any] = { "name": "qa/my-validation", "description": "A validation from config", "source": "builtin", } cfg.update(overrides) return cfg # --------------------------------------------------------------------------- # Tool Creation Steps # --------------------------------------------------------------------------- @when('I create a tool with name "{name}" source "{source}" and code "{code}"') def step_create_tool_custom( context: Context, name: str, source: str, code: str ) -> None: """Create a custom tool with inline code.""" context.tool_model = Tool( name=name, description="A custom test tool", source=ToolSource(source), code=code, ) context.tool_model_error = None @when('I create a tool with name "{name}" and source "{source}"') def step_create_tool_builtin(context: Context, name: str, source: str) -> None: """Create a tool with a given source.""" context.tool_model = _make_tool(name=name, source=ToolSource(source)) context.tool_model_error = None @when( 'I create an MCP tool with name "{name}" server "{server}" ' 'and tool name "{tool_name}"' ) def step_create_mcp_tool( context: Context, name: str, server: str, tool_name: str ) -> None: """Create an MCP tool with required fields.""" context.tool_model = Tool( name=name, description="An MCP tool", source=ToolSource.MCP, mcp_server=server, mcp_tool_name=tool_name, ) context.tool_model_error = None @when('I create an agent skill tool with name "{name}" and path "{path}"') def step_create_agent_skill_tool(context: Context, name: str, path: str) -> None: """Create an agent_skill tool.""" context.tool_model = Tool( name=name, description="An agent skill tool", source=ToolSource.AGENT_SKILL, agent_skill_path=path, ) context.tool_model_error = None # --------------------------------------------------------------------------- # Tool assertions # --------------------------------------------------------------------------- @then("the tool model should be created") def step_check_tool_created(context: Context) -> None: """Verify the tool was created.""" assert context.tool_model is not None, "Tool should be created" @then('the tool model name should be "{expected}"') def step_check_tool_name(context: Context, expected: str) -> None: """Check the tool name.""" assert context.tool_model.name == expected, ( f"Expected name '{expected}', got '{context.tool_model.name}'" ) @then('the tool model source should be "{expected}"') def step_check_tool_source(context: Context, expected: str) -> None: """Check the tool source.""" assert context.tool_model.source.value == expected, ( f"Expected source '{expected}', got '{context.tool_model.source.value}'" ) @then('the tool model type should be "{expected}"') def step_check_tool_type(context: Context, expected: str) -> None: """Check the tool type.""" assert context.tool_model.tool_type.value == expected, ( f"Expected type '{expected}', got '{context.tool_model.tool_type.value}'" ) @then('the tool model namespace should be "{expected}"') def step_check_tool_namespace(context: Context, expected: str) -> None: """Check the tool namespace property.""" assert context.tool_model.namespace == expected, ( f"Expected namespace '{expected}', got '{context.tool_model.namespace}'" ) @then('the tool model short name should be "{expected}"') def step_check_tool_short_name(context: Context, expected: str) -> None: """Check the tool short_name property.""" assert context.tool_model.short_name == expected, ( f"Expected short_name '{expected}', got '{context.tool_model.short_name}'" ) # --------------------------------------------------------------------------- # Name validation # --------------------------------------------------------------------------- @when('I try to create a tool with invalid name "{name}"') def step_try_create_tool_invalid_name(context: Context, name: str) -> None: """Attempt to create a tool with an invalid name.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = _make_tool(name=name) except ValidationError as e: context.tool_model_error = e @then("a tool model validation error should be raised") def step_check_tool_validation_error(context: Context) -> None: """Verify that a validation error was raised.""" assert context.tool_model_error is not None, ( "Expected a validation error to be raised" ) @then('the tool model error should mention "{text}"') def step_check_tool_error_message(context: Context, text: str) -> None: """Check the error message contains expected text.""" error_str = str(context.tool_model_error) assert text in error_str, f"Expected error to mention '{text}', got: {error_str}" # --------------------------------------------------------------------------- # Source-conditional field requirements # --------------------------------------------------------------------------- @when('I try to create a tool with name "{name}" source "{source}" without code') def step_try_create_custom_without_code( context: Context, name: str, source: str ) -> None: """Attempt to create a custom tool without code.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Tool( name=name, description="Missing code", source=ToolSource(source), ) except ValidationError as e: context.tool_model_error = e @when("I try to create an MCP tool without mcp_server") def step_try_create_mcp_without_server(context: Context) -> None: """Attempt to create an MCP tool without mcp_server.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Tool( name="local/test", description="Missing server", source=ToolSource.MCP, mcp_tool_name="some_tool", ) except ValidationError as e: context.tool_model_error = e @when("I try to create an MCP tool without mcp_tool_name") def step_try_create_mcp_without_tool_name(context: Context) -> None: """Attempt to create an MCP tool without mcp_tool_name.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Tool( name="local/test", description="Missing tool name", source=ToolSource.MCP, mcp_server="some_server", ) except ValidationError as e: context.tool_model_error = e @when("I try to create an agent skill tool without path") def step_try_create_agent_skill_without_path(context: Context) -> None: """Attempt to create an agent_skill tool without agent_skill_path.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Tool( name="local/test", description="Missing path", source=ToolSource.AGENT_SKILL, ) except ValidationError as e: context.tool_model_error = e # --------------------------------------------------------------------------- # Capability constraint enforcement # --------------------------------------------------------------------------- @when("I try to create a tool capability with read_only true and writes true") def step_try_capability_readonly_writes(context: Context) -> None: """Attempt to create read_only capability with writes=True.""" context.tool_model_error = None try: ToolCapability(read_only=True, writes=True) except ValidationError as e: context.tool_model_error = e @when("I try to create a tool capability with read_only true and checkpointable true") def step_try_capability_readonly_checkpoint(context: Context) -> None: """Attempt to create read_only capability with checkpointable=True.""" context.tool_model_error = None try: ToolCapability(read_only=True, checkpointable=True) except ValidationError as e: context.tool_model_error = e @when("I create a tool capability with writes true") def step_create_capability_with_writes(context: Context) -> None: """Create a capability with writes enabled.""" context.tool_capability = ToolCapability(writes=True) @then("the tool capability should have writes true") def step_check_capability_writes(context: Context) -> None: """Check capability writes is True.""" assert context.tool_capability.writes is True # --------------------------------------------------------------------------- # ResourceSlot binding validation # --------------------------------------------------------------------------- @when("I try to create a resource slot with static binding and no static resource") def step_try_slot_static_no_resource(context: Context) -> None: """Attempt static binding without static_resource.""" context.tool_model_error = None try: ResourceSlot( name="repo", resource_type="git-checkout", access=ResourceAccessMode.READ_ONLY, binding=BindingMode.STATIC, ) except ValidationError as e: context.tool_model_error = e @when("I try to create a resource slot with contextual binding and static resource set") def step_try_slot_contextual_with_resource(context: Context) -> None: """Attempt contextual binding with static_resource.""" context.tool_model_error = None try: ResourceSlot( name="repo", resource_type="git-checkout", access=ResourceAccessMode.READ_ONLY, binding=BindingMode.CONTEXTUAL, static_resource="my-repo", ) except ValidationError as e: context.tool_model_error = e @when('I create a resource slot with static binding and static resource "{resource}"') def step_create_slot_static(context: Context, resource: str) -> None: """Create a valid static resource slot.""" context.tool_resource_slot = ResourceSlot( name="repo", resource_type="git-checkout", access=ResourceAccessMode.READ_ONLY, binding=BindingMode.STATIC, static_resource=resource, ) @then("the tool resource slot should be created") def step_check_slot_created(context: Context) -> None: """Verify slot was created.""" assert context.tool_resource_slot is not None @when('I try to create a resource slot with invalid name "{name}"') def step_try_slot_invalid_name(context: Context, name: str) -> None: """Attempt to create a slot with invalid name.""" context.tool_model_error = None try: ResourceSlot( name=name, resource_type="git-checkout", access=ResourceAccessMode.READ_ONLY, ) except ValidationError as e: context.tool_model_error = e # --------------------------------------------------------------------------- # Validation model creation # --------------------------------------------------------------------------- @when('I create a validation with name "{name}" source "{source}" and code "{code}"') def step_create_validation_custom( context: Context, name: str, source: str, code: str ) -> None: """Create a custom validation.""" context.tool_model = Validation( name=name, description="A test validation", source=ToolSource(source), code=code, ) context.tool_model_error = None @when('I create a validation with name "{name}" source "{source}"') def step_create_validation_builtin(context: Context, name: str, source: str) -> None: """Create a builtin validation.""" context.tool_model = Validation( name=name, description="A test validation", source=ToolSource(source), ) context.tool_model_error = None @when('I create an informational validation with name "{name}" source "{source}"') def step_create_informational_validation( context: Context, name: str, source: str ) -> None: """Create an informational validation.""" context.tool_model = Validation( name=name, description="An info validation", source=ToolSource(source), mode=ValidationMode.INFORMATIONAL, ) context.tool_model_error = None @then("the tool validation capability should be read only") def step_check_validation_readonly(context: Context) -> None: """Check validation capability is read_only.""" assert context.tool_model.capability.read_only is True @then("the tool validation capability should not have writes") def step_check_validation_no_writes(context: Context) -> None: """Check validation capability has writes=False.""" assert context.tool_model.capability.writes is False @then("the tool validation capability should not be checkpointable") def step_check_validation_no_checkpoint(context: Context) -> None: """Check validation capability has checkpointable=False.""" assert context.tool_model.capability.checkpointable is False @then('the tool validation mode should be "{expected}"') def step_check_validation_mode(context: Context, expected: str) -> None: """Check validation mode.""" assert context.tool_model.mode.value == expected, ( f"Expected mode '{expected}', got '{context.tool_model.mode.value}'" ) # --------------------------------------------------------------------------- # Validation wraps/transform # --------------------------------------------------------------------------- @when("I try to create a wrapped validation without transform") def step_try_wrapped_validation_no_transform(context: Context) -> None: """Attempt wrapped validation without transform.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Validation( name="qa/bad-wrap", description="Missing transform", source=ToolSource.WRAPPED, wraps="devops/run-linter", ) except ValidationError as e: context.tool_model_error = e @when("I try to create a wrapped validation with code") def step_try_wrapped_validation_with_code(context: Context) -> None: """Attempt wrapped validation with code (mutually exclusive).""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Validation( name="qa/bad-wrap", description="Code conflict", source=ToolSource.WRAPPED, wraps="devops/run-linter", transform="def transform(r): return r", code="print('bad')", ) except ValidationError as e: context.tool_model_error = e @when("I try to create a validation with wraps and wrong source") def step_try_wrapped_validation_wrong_source(context: Context) -> None: """Attempt validation with wraps but source != wrapped.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Validation( name="qa/bad-wrap", description="Wrong source", source=ToolSource.BUILTIN, wraps="devops/run-linter", transform="def transform(r): return r", ) except ValidationError as e: context.tool_model_error = e @when("I try to create a validation with argument_mapping but no wraps") def step_try_validation_argmap_no_wraps(context: Context) -> None: """Attempt validation with argument_mapping but no wraps.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Validation( name="qa/bad-argmap", description="Argmap without wraps", source=ToolSource.BUILTIN, argument_mapping={"key": "value"}, ) except ValidationError as e: context.tool_model_error = e @when("I create a valid wrapped validation") def step_create_valid_wrapped_validation(context: Context) -> None: """Create a valid wrapped validation.""" context.tool_model = Validation( name="qa/lint-check", description="Lint validation", source=ToolSource.WRAPPED, wraps="devops/run-linter", transform="def transform(r): return {'passed': True}", argument_mapping={"strict": True}, ) context.tool_model_error = None @then('the tool validation wraps should be "{expected}"') def step_check_validation_wraps(context: Context, expected: str) -> None: """Check validation wraps field.""" assert context.tool_model.wraps == expected, ( f"Expected wraps '{expected}', got '{context.tool_model.wraps}'" ) # --------------------------------------------------------------------------- # from_config loading # --------------------------------------------------------------------------- @when('I load a tool from config with name "{name}" source "{source}"') def step_load_tool_from_config(context: Context, name: str, source: str) -> None: """Load a tool from a config dict.""" config = _tool_config(name=name, source=source) context.tool_model = Tool.from_config(config) context.tool_model_error = None @when('I try to load a tool from config missing "{field}"') def step_try_load_tool_config_missing(context: Context, field: str) -> None: """Try loading a tool config with missing field.""" config = _tool_config() del config[field] context.tool_model_config_error = None try: Tool.from_config(config) except ValueError as e: context.tool_model_config_error = e @then('a tool model config error should be raised with "{field}"') def step_check_tool_config_error(context: Context, field: str) -> None: """Check that a config error mentioning field was raised.""" assert context.tool_model_config_error is not None, ( f"Expected ValueError for missing '{field}'" ) assert field in str(context.tool_model_config_error), ( f"Expected error to mention '{field}', got: {context.tool_model_config_error}" ) @when('I load a validation from config with wraps "{wraps}"') def step_load_validation_from_config_wraps(context: Context, wraps: str) -> None: """Load a validation from config with wraps.""" config = _validation_config( wraps=wraps, transform="def transform(r): return r", ) # Remove source since wraps implies wrapped config.pop("source", None) context.tool_model = Validation.from_config(config) context.tool_model_error = None @when('I try to load a validation config missing "{field}"') def step_try_load_validation_config_missing(context: Context, field: str) -> None: """Try loading a validation config with missing field.""" config = _validation_config() del config[field] context.tool_model_config_error = None try: Validation.from_config(config) except ValueError as e: context.tool_model_config_error = e # --------------------------------------------------------------------------- # as_cli_dict # --------------------------------------------------------------------------- @when("I create a tool and call as_cli_dict") def step_create_tool_cli_dict(context: Context) -> None: """Create a tool and get CLI dict.""" context.tool_model = _make_tool() context.tool_cli_dict = context.tool_model.as_cli_dict() @when("I create a validation and call as_cli_dict") def step_create_validation_cli_dict(context: Context) -> None: """Create a validation and get CLI dict.""" context.tool_model = Validation( name="qa/test-check", description="A validation", source=ToolSource.BUILTIN, ) context.tool_cli_dict = context.tool_model.as_cli_dict() @then('the tool cli dict should have key "{key}"') def step_check_tool_cli_dict_key(context: Context, key: str) -> None: """Check CLI dict has expected key.""" assert key in context.tool_cli_dict, ( f"Expected key '{key}' in cli dict, keys: {list(context.tool_cli_dict)}" ) # --------------------------------------------------------------------------- # Timeout # --------------------------------------------------------------------------- @when("I try to create a tool with timeout {timeout:d}") def step_try_create_tool_bad_timeout(context: Context, timeout: int) -> None: """Attempt to create a tool with invalid timeout.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = _make_tool(timeout=timeout) except ValidationError as e: context.tool_model_error = e @then("the tool model timeout should be {expected:d}") def step_check_tool_timeout(context: Context, expected: int) -> None: """Check tool timeout.""" assert context.tool_model.timeout == expected, ( f"Expected timeout {expected}, got {context.tool_model.timeout}" ) # --------------------------------------------------------------------------- # ToolLifecycle # --------------------------------------------------------------------------- @when('I create a tool lifecycle with discover "{discover}"') def step_create_lifecycle(context: Context, discover: str) -> None: """Create a ToolLifecycle.""" context.tool_lifecycle = ToolLifecycle(discover=discover) @then('the tool lifecycle discover should be "{expected}"') def step_check_lifecycle_discover(context: Context, expected: str) -> None: """Check lifecycle discover hook.""" assert context.tool_lifecycle.discover == expected # --------------------------------------------------------------------------- # Enum checks # --------------------------------------------------------------------------- @then('the ToolSource enum should have values "{values}"') def step_check_tool_source_enum(context: Context, values: str) -> None: """Verify ToolSource enum values.""" expected = set(values.split(",")) actual = {e.value for e in ToolSource} assert actual == expected, f"Expected {expected}, got {actual}" @then('the ToolType enum should have values "{values}"') def step_check_tool_type_enum(context: Context, values: str) -> None: """Verify ToolType enum values.""" expected = set(values.split(",")) actual = {e.value for e in ToolType} assert actual == expected, f"Expected {expected}, got {actual}" @then('the ValidationMode enum should have values "{values}"') def step_check_validation_mode_enum(context: Context, values: str) -> None: """Verify ValidationMode enum values.""" expected = set(values.split(",")) actual = {e.value for e in ValidationMode} assert actual == expected, f"Expected {expected}, got {actual}" @then('the CheckpointScope enum should have values "{values}"') def step_check_checkpoint_scope_enum(context: Context, values: str) -> None: """Verify CheckpointScope enum values.""" expected = set(values.split(",")) actual = {e.value for e in CheckpointScope} assert actual == expected, f"Expected {expected}, got {actual}" @then('the BindingMode enum should have values "{values}"') def step_check_binding_mode_enum(context: Context, values: str) -> None: """Verify BindingMode enum values.""" expected = set(values.split(",")) actual = {e.value for e in BindingMode} assert actual == expected, f"Expected {expected}, got {actual}" @then('the ResourceAccessMode enum should have values "{values}"') def step_check_access_mode_enum(context: Context, values: str) -> None: """Verify ResourceAccessMode enum values.""" expected = set(values.split(",")) actual = {e.value for e in ResourceAccessMode} assert actual == expected, f"Expected {expected}, got {actual}" # --------------------------------------------------------------------------- # from_config with resource slots # --------------------------------------------------------------------------- @when("I load a tool from config with resource slots") def step_load_tool_config_with_slots(context: Context) -> None: """Load a tool from config that includes resource slots.""" config = _tool_config( resource_slots=[ { "name": "repo", "resource_type": "git-checkout", "access": "read_only", }, ], ) context.tool_model = Tool.from_config(config) context.tool_model_error = None @then("the tool model should have {count:d} resource slot") def step_check_tool_slot_count(context: Context, count: int) -> None: """Check tool resource slot count.""" actual = len(context.tool_model.resource_slots) assert actual == count, f"Expected {count} slots, got {actual}" # --------------------------------------------------------------------------- # Empty description # --------------------------------------------------------------------------- @when("I try to create a tool with empty description") def step_try_create_tool_empty_desc(context: Context) -> None: """Attempt to create a tool with empty description.""" context.tool_model_error = None context.tool_model = None try: context.tool_model = Tool( name="local/bad-desc", description="", source=ToolSource.BUILTIN, ) except ValidationError as e: context.tool_model_error = e # --------------------------------------------------------------------------- # as_cli_dict coverage for optional fields # --------------------------------------------------------------------------- @when("I create a custom tool and call as_cli_dict") def step_create_custom_tool_cli_dict(context: Context) -> None: """Create a custom tool and get CLI dict (covers code branch).""" context.tool_model = Tool( name="local/custom-cli", description="Custom tool for CLI dict", source=ToolSource.CUSTOM, code="return True", ) context.tool_cli_dict = context.tool_model.as_cli_dict() @when("I create an MCP tool and call as_cli_dict") def step_create_mcp_tool_cli_dict(context: Context) -> None: """Create an MCP tool and get CLI dict (covers mcp fields).""" context.tool_model = Tool( name="devops/mcp-cli", description="MCP tool for CLI dict", source=ToolSource.MCP, mcp_server="devops-mcp", mcp_tool_name="docker_build", ) context.tool_cli_dict = context.tool_model.as_cli_dict() @when("I create an agent skill tool and call as_cli_dict") def step_create_agent_skill_tool_cli_dict(context: Context) -> None: """Create an agent skill tool and get CLI dict.""" context.tool_model = Tool( name="skills/fmt-cli", description="Agent skill for CLI dict", source=ToolSource.AGENT_SKILL, agent_skill_path="/skills/fmt", ) context.tool_cli_dict = context.tool_model.as_cli_dict() @when("I create a tool with schemas and call as_cli_dict") def step_create_tool_with_schemas_cli_dict(context: Context) -> None: """Create a tool with schemas and get CLI dict.""" context.tool_model = Tool( name="local/schema-cli", description="Tool with schemas", source=ToolSource.BUILTIN, input_schema={"type": "object"}, output_schema={"type": "object"}, ) context.tool_cli_dict = context.tool_model.as_cli_dict() @when("I create a tool with side effects and cost profile and call as_cli_dict") def step_create_tool_with_effects_cli_dict(context: Context) -> None: """Create a tool with side effects and cost profile.""" context.tool_model = Tool( name="local/effects-cli", description="Tool with effects", source=ToolSource.BUILTIN, capability=ToolCapability( side_effects=["network", "filesystem"], cost_profile="high", ), ) context.tool_cli_dict = context.tool_model.as_cli_dict() @then('the tool cli dict capability should have key "{key}"') def step_check_tool_cli_dict_cap_key(context: Context, key: str) -> None: """Check CLI dict capability sub-dict has expected key.""" cap = context.tool_cli_dict["capability"] assert key in cap, f"Expected key '{key}' in capability dict, keys: {list(cap)}" @when("I create a tool with resource slots and call as_cli_dict") def step_create_tool_with_slots_cli_dict(context: Context) -> None: """Create a tool with resource slots and get CLI dict.""" context.tool_model = Tool( name="local/slots-cli", description="Tool with slots", source=ToolSource.BUILTIN, resource_slots=[ ResourceSlot( name="repo", resource_type="git-checkout", access=ResourceAccessMode.READ_ONLY, ), ], ) context.tool_cli_dict = context.tool_model.as_cli_dict() # --------------------------------------------------------------------------- # Validation as_cli_dict wraps fields # --------------------------------------------------------------------------- @when("I create a wrapped validation and call as_cli_dict") def step_create_wrapped_validation_cli_dict(context: Context) -> None: """Create a wrapped validation and get CLI dict.""" context.tool_model = Validation( name="qa/wrap-cli", description="Wrapped validation for CLI dict", source=ToolSource.WRAPPED, wraps="devops/run-linter", transform="def transform(r): return {'passed': True}", argument_mapping={"strict": True}, ) context.tool_cli_dict = context.tool_model.as_cli_dict() # --------------------------------------------------------------------------- # Validation from_config without wraps # --------------------------------------------------------------------------- @when("I try to load a validation config without source or wraps") def step_try_load_validation_no_source_no_wraps(context: Context) -> None: """Try loading a validation config with neither source nor wraps.""" config = _validation_config() del config["source"] context.tool_model_config_error = None try: Validation.from_config(config) except ValueError as e: context.tool_model_config_error = e @when('I load a validation from config with source "{source}"') def step_load_validation_from_config_source(context: Context, source: str) -> None: """Load a validation from config with explicit source.""" config = _validation_config(source=source) context.tool_model = Validation.from_config(config) context.tool_model_error = None # --------------------------------------------------------------------------- # Validation forces read_only on writable capability # --------------------------------------------------------------------------- @when("I create a validation with writes capability") def step_create_validation_with_writes(context: Context) -> None: """Create a validation that has writes in capability (will be forced).""" context.tool_model = Validation( name="qa/writable-val", description="Validation with writes cap", source=ToolSource.BUILTIN, capability=ToolCapability( writes=True, checkpointable=True, ), ) context.tool_model_error = None