"""Step definitions for structural component output validation. Tests for features/structural_validation.feature - validates all four validators covering plan tree nodes, decision CLI dicts, structured output envelopes, and the unified dispatcher. """ from __future__ import annotations from typing import Any from behave import given, then, when # type: ignore[import-untyped] from behave.runner import Context # type: ignore[import-untyped] from cleveragents.core.validation import ( ValidationError, validate_decision_dict, validate_plan_tree, validate_structured_component_output, validate_structured_output, ) # Helper: valid ULID strings for test data VALID_ULID_A = "01ARZ3NDEKTSV4XXFFJFRC889A" VALID_ULID_B = "01ARZ3NDEKTSV4XXFFJFRC889B" VALID_ULID_C = "01ARZ3NDEKTSV4XXFFJFRC889C" # ──────────────────────────────────────────────────────────── # Plan tree scenarios # ──────────────────────────────────────────────────────────── @given("the plan tree contains 1 node(s)") def step_plan_tree_1_node(ctx: Context) -> None: """Create a single-node plan tree.""" ctx.tree_nodes = [ { "decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": 0, "question": "Q", "children": [], } ] @given("the plan tree contains 5 node(s)") def step_plan_tree_5_nodes(ctx: Context) -> None: """Create a 5-node plan tree.""" ctx.tree_nodes = [ { "decision_id": f"01ARZ3NDEKTSV4XXFFJFRC{str(i).zfill(4)}", "type": "strategy_choice", "sequence": i, "question": f"Decision question #{i}", "children": [], **({"parent_decision_id": VALID_ULID_A} if i > 0 else {}), } for i in range(5) ] @given("the plan tree contains 20 node(s)") def step_plan_tree_20_nodes(ctx: Context) -> None: """Create a 20-node plan tree.""" ctx.tree_nodes = [ { "decision_id": f"01ARZ3NDEKTSV4XXFFJFRC{str(i).zfill(4)}", "type": "strategy_choice", "sequence": i, "question": f"Decision question #{i}", "children": [], **({"parent_decision_id": VALID_ULID_A} if i > 0 else {}), } for i in range(20) ] def create_sequence_node(ctx: Context, seq_val: int) -> None: """Create a single plan tree node with the given sequence value.""" ctx.tree_nodes = [ { "decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": seq_val, "question": "Q", "children": [], } ] @given('the plan tree contains a node missing "decision_id"') def step_node_missing_decision_id(ctx: Context) -> None: """Create a node missing the required decision_id field.""" ctx.tree_nodes = [ { "type": "strategy_choice", "sequence": 0, "question": "Missing ID", "children": [], } ] @given('the plan tree contains a node with invalid ULID "not-a-ulid-12345"') def step_node_invalid_ulid(ctx: Context) -> None: """Create a node with an invalid decision_id.""" ctx.tree_nodes = [ { "decision_id": "not-a-ulid-12345", "type": "strategy_choice", "sequence": 0, "question": "Invalid ULID test", "children": [], } ] @given('the plan tree contains a node with empty "children" list') def step_node_empty_children(ctx: Context) -> None: """Create a node with an empty children list - should be valid.""" ctx.tree_nodes = [ { "decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": 0, "question": "Question text", "children": [], } ] @given("two sibling nodes share the same sequence number") def step_duplicate_sequences(ctx: Context) -> None: """Create two sibling nodes with identical sequence.""" ctx.tree_nodes = [ { "decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": 5, "question": "First", "children": [], "parent_decision_id": VALID_ULID_B, }, { "decision_id": VALID_ULID_C, "type": "implementation_choice", "sequence": 5, "question": "Second", "children": [], "parent_decision_id": VALID_ULID_B, }, ] @given('a plan tree node with sequence "0"') def step_node_sequence_zero(ctx: Context) -> None: create_sequence_node(ctx, 0) @given('a plan tree node with sequence "1"') def step_node_sequence_one(ctx: Context) -> None: create_sequence_node(ctx, 1) @given('a plan tree node with sequence "100"') def step_node_sequence_100(ctx: Context) -> None: create_sequence_node(ctx, 100) @given('a plan tree node with sequence "-1"') def step_node_sequence_negative(ctx: Context) -> None: create_sequence_node(ctx, -1) # ──────────────────────────────────────────────────────────── # Decision dict scenarios # ──────────────────────────────────────────────────────────── def _parse_value(raw: str) -> Any: """Parse a behave value string into Python type.""" stripped = raw.strip().strip('"').strip("'") if stripped in ("null", "none", "None"): return None if stripped == "true": return True if stripped == "false": return False try: return int(stripped) except ValueError: pass try: return float(stripped) except ValueError: pass return stripped def _ensure_decision_dict(ctx: Context) -> dict: """Ensure ctx.decision_dict exists with defaults.""" if not hasattr(ctx, "decision_dict"): ctx.decision_dict = { "decision_id": VALID_ULID_A, "plan_id": VALID_ULID_B, "type": "strategy_choice", "sequence": 1, "question": "Which approach?", "chosen": "Option A", "confidence": 0.75, "parent": "(root)", "is_correction": False, "superseded": False, } return ctx.decision_dict # type: ignore[return-value] @given('I have a decision dict with "decision_id" set to 01ARZ3NDEKTSV4XXFFJFRC889A') def step_decision_dict_field_decision_id(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["decision_id"] = "01ARZ3NDEKTSV4XXFFJFRC889A" @given('I have a decision dict with "plan_id" set to 01ARZ3NDEKTSV4XXFFJFRC889B') def step_decision_dict_field_plan_id(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["plan_id"] = "01ARZ3NDEKTSV4XXFFJFRC889B" @given('I have a decision dict with "type" set to strategy_choice') def step_decision_dict_field_type(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["type"] = "strategy_choice" @given('I have a decision dict with "sequence" set to 5') def step_decision_dict_field_sequence(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["sequence"] = 5 @given('I have a decision dict with "question" set to Which approach?') def step_decision_dict_field_question(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["question"] = "Which approach?" @given('I have a decision dict with "chosen" set to A') def step_decision_dict_field_chosen(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["chosen"] = "A" @given('I have a decision dict with "confidence" set to 0.75') def step_decision_dict_field_confidence(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["confidence"] = 0.75 @given('I have a decision dict with "parent" set to (root)') def step_decision_dict_field_parent(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["parent"] = "(root)" @given('I have a decision dict with "is_correction" set to false') def step_decision_dict_field_is_correction(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["is_correction"] = False @given('I have a decision dict with "superseded" set to false') def step_decision_dict_field_superseded(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["superseded"] = False @given('I have a decision dict with "confidence" set to 1.5') def step_decision_dict_confidence_15(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["confidence"] = 1.5 @given('I have a decision dict with "confidence" set to null') def step_decision_dict_confidence_null(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["confidence"] = None @given('I have a decision dict with "question" set to 42') def step_decision_dict_question_42(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["question"] = 42 @given('I have a decision dict with "parent" set to "random-string"') def step_decision_dict_parent_random(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["parent"] = "random-string" @given('I have a decision dict with "is_correction" set to "yes"') def step_decision_dict_is_correction_yes(ctx: Context) -> None: d = _ensure_decision_dict(ctx) d["is_correction"] = "yes" @given("I have a valid decision dict") @when("I validate the decision dict") def step_validate_decision(ctx: Context) -> None: """Validate and store results.""" d = _ensure_decision_dict(ctx) ctx.validation_result = validate_decision_dict(d) # ──────────────────────────────────────────────────────────── # Structured output scenarios # ──────────────────────────────────────────────────────────── def _ensure_struct_output(ctx: Context) -> dict: """Ensure ctx.struct_output exists with defaults.""" if not hasattr(ctx, "struct_output"): ctx.struct_output = { "command": "agents plan list", "session_id": VALID_ULID_A, "status": "ok", "exit_code": 0, "elements": [], } return ctx.struct_output # type: ignore[return-value] @given("I have a valid structured output with all required fields") @when("I validate the structured output") def step_validate_structured(ctx: Context) -> None: """Validate and store results.""" o = _ensure_struct_output(ctx) ctx.validation_result = validate_structured_output(o) @given( 'I have a structured output with "command" set to agents plan list (and all other required fields present)' ) def step_structured_output_command(ctx: Context) -> None: o = _ensure_struct_output(ctx) o["command"] = "agents plan list" @given( 'I have a structured output with "session_id" set to 01ARZ3NDEKTSV4XXFFJFRC889A (and all other required fields present)' ) def step_structured_output_session_id(ctx: Context) -> None: o = _ensure_struct_output(ctx) o["session_id"] = "01ARZ3NDEKTSV4XXFFJFRC889A" @given( 'I have a structured output with "status" set to ok (and all other required fields present)' ) def step_structured_output_status_ok(ctx: Context) -> None: o = _ensure_struct_output(ctx) o["status"] = "ok" @given( 'I have a structured output with "exit_code" set to 0 (and all other required fields present)' ) def step_structured_output_exit_code_0(ctx: Context) -> None: o = _ensure_struct_output(ctx) o["exit_code"] = 0 @given( 'I have a structured output with "status" set to "running" (and all other required fields present)' ) def step_structured_output_status_running(ctx: Context) -> None: o = _ensure_struct_output(ctx) o["status"] = "running" @given( 'I have a structured output with "exit_code" set to "-1" (and all other required fields present)' ) def step_structured_output_exit_code_neg1(ctx: Context) -> None: o = _ensure_struct_output(ctx) o["exit_code"] = -1 @given( 'I have a structured output with empty "elements" list (and all other fields valid)' ) def step_empty_elements(ctx: Context) -> None: """Empty elements is valid.""" o = _ensure_struct_output(ctx) o["elements"] = [] @given('I have a structured output with an element missing "kind" field') def step_invalid_element(ctx: Context) -> None: """Create an invalid element.""" o = _ensure_struct_output(ctx) o["elements"] = [{"not_kind": "panel"}] # ──────────────────────────────────────────────────────────── # Dispatcher scenarios # ──────────────────────────────────────────────────────────── @given('I target validation for "plan_tree" with valid data') def step_dispatcher_target_plan_tree(ctx: Context) -> None: ctx.target_type = "plan_tree" ctx.validator_data = [ { "decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": 0, "question": "Q", "children": [], } ] @given('I target validation for "decision" with valid data') def step_dispatcher_target_decision(ctx: Context) -> None: ctx.target_type = "decision" ctx.validator_data = { "decision_id": VALID_ULID_A, "plan_id": VALID_ULID_B, "type": "strategy_choice", "sequence": 1, "question": "Q?", "chosen": "A", "confidence": 0.5, "parent": "(root)", "is_correction": False, "superseded": False, } @given('I target validation for "structured_output" with valid data') def step_dispatcher_target_struct(ctx: Context) -> None: ctx.target_type = "structured_output" ctx.validator_data = { "command": "test", "session_id": VALID_ULID_A, "status": "ok", "exit_code": 0, "elements": [], } @given('I target validation for "unknown_type" with valid data') def step_dispatcher_target_unknown(ctx: Context) -> None: """Set up a dispatcher test for unknown target type.""" ctx.target_type = "unknown_type" ctx.validator_data = {} @when("I call validate_structured_component_output") def step_call_dispatcher(ctx: Context) -> None: """Call the dispatcher.""" target = ctx.target_type if hasattr(ctx, "target_type") else "plan_tree" # type: ignore[attr-defined] data = ctx.validator_data if hasattr(ctx, "validator_data") else {} # type: ignore[attr-defined] try: ctx.dispatcher_result = validate_structured_component_output(target, data) # type: ignore[arg-type] ctx.dispatcher_error = None except ValidationError as exc: ctx.dispatcher_result = None ctx.dispatcher_error = exc # ──────────────────────────────────────────────────────────── # Shared scenarios # ──────────────────────────────────────────────────────────── @given("the plan tree has {node_count:d} valid nodes") def step_valid_tree(ctx: Context, node_count: int = 3) -> None: """Set up a default valid tree.""" if not hasattr(ctx, "tree_nodes"): ctx.tree_nodes = [ { "decision_id": f"01ARZ3NDEKTSV4XXFFJFRC{str(i).zfill(4)}", "type": "strategy_choice", "sequence": i, "question": f"Q{i}", "children": [], } for i in range(node_count) ] @given("I have a plan tree with valid nodes") def step_valid_nodes_default(ctx: Context) -> None: """Set default valid tree if no tree defined.""" @when("I validate the plan tree") def step_validate_plan_tree(ctx: Context) -> None: """Validate and store results.""" nodes = ( ctx.tree_nodes if hasattr(ctx, "tree_nodes") else [ # type: ignore[attr-defined] { "decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": i, "question": f"Q{i}", "children": [], } for i in range(3) ] ) ctx.validation_result = validate_plan_tree(nodes) @when("I validate the plan tree structure") def step_validate_structure(ctx: Context) -> None: """Run plan_tree validation.""" nodes = ctx.tree_nodes if hasattr(ctx, "tree_nodes") else [] # type: ignore[attr-defined] ctx.validation_result = validate_plan_tree(nodes) @given("I have a decision dict where whitespace may vary between fields") def step_whitespace_dict(ctx: Context) -> None: """Whitespace in string values - structure still valid.""" ctx.decision_dict = { "decision_id": VALID_ULID_A, "plan_id": VALID_ULID_B, "type": " strategy_choice ", "sequence": 1, "question": "Which approach? ", "chosen": " Option A ", "confidence": 0.75, "parent": "(root)", "is_correction": False, "superseded": False, } @given("a plan tree with parent and child node references connected by ULID") def step_parent_child_tree(ctx: Context) -> None: """Parent-decoy pattern - root plus children.""" ctx.tree_nodes = [ { "decision_id": VALID_ULID_A, "type": "prompt_definition", "sequence": 0, "question": "Root question", "children": [{"decision_id": VALID_ULID_B}, {"decision_id": VALID_ULID_C}], "parent_decision_id": None, }, { "decision_id": VALID_ULID_B, "type": "strategy_choice", "sequence": 1, "question": "Child A", "children": [], "parent_decision_id": VALID_ULID_A, }, { "decision_id": VALID_ULID_C, "type": "strategy_choice", "sequence": 2, "question": "Child B", "children": [], "parent_decision_id": VALID_ULID_A, }, ] @given("I have invalid plan tree data") def step_invalid_tree_data(ctx: Context) -> None: """Create invalid plan tree node.""" ctx.tree_nodes = [ {"not_a_key": "value"}, # missing required fields entirely ] # ──────────────────────────────────────────────────────────── # Assertions (shared / cross-scenario) # ──────────────────────────────────────────────────────────── @then("it should be structurally valid") def step_should_be_valid(ctx: Context) -> None: """Assert validation passed.""" result = getattr(ctx, "validation_result", {}) or {"valid": True} assert result["valid"], f"Expected valid but got errors: {result.get('errors', [])}" @then("it should be structurally invalid") def step_should_be_invalid(ctx: Context) -> None: """Assert validation failed.""" result = getattr(ctx, "validation_result", {}) or {"valid": True} assert not result["valid"], "Expected invalid but got valid" @then("it should report no errors") def step_no_errors(ctx: Context) -> None: """Assert no error messages.""" result = getattr(ctx, "validation_result", {}) or {} errs = result.get("errors", []) assert len(errs) == 0, f"Expected no errors but got: {errs}" @then('it should report errors including "{text}"') def step_errors_include(ctx: Context, text: str) -> None: """Assert at least one error contains the given substring.""" result = getattr(ctx, "validation_result", {}) or {} errs = result.get("errors", []) assert any(text in e for e in errs), f"'{text}' not found in errors: {errs}" @then('error message must include "{text}"') @then('error must include "{text}"') def step_error_contains(ctx: Context, text: str) -> None: """Assert an error contains substring.""" result = getattr(ctx, "validation_result", {}) or {} errs = result.get("errors", []) assert any(text in e for e in errs), f"'{text}' not found in errors: {errs}" @then('the first error message should identify "{prefix}" and the specific field issue') def step_first_error_identifies(ctx: Context, prefix: str) -> None: """Assert first error contains prefix.""" result = getattr(ctx, "validation_result", {}) or {} errs = result.get("errors", []) assert len(errs) > 0, "Expected at least one error" assert prefix in errs[0], f"First error '{errs[0]}' should contain '{prefix}'" @then('it should raise a ValidationError with message containing "{text}"') def step_raise_error_with_text(ctx: Context, text: str) -> None: """Assert dispatcher raised specific error.""" err = getattr(ctx, "dispatcher_error", None) assert err is not None, "Expected a ValidationError" assert isinstance(err, ValidationError), ( f"Expected ValidationError, got {type(err)}" ) assert text in str(err), f"'{text}' not in error message: {err}" @then("it should dispatch to the matching validator") @then("return a valid result") def step_dispatcher_valid(ctx: Context) -> None: """Assert dispatcher succeeded and returned valid.""" err = getattr(ctx, "dispatcher_error", None) res = getattr(ctx, "dispatcher_result", None) assert err is None, f"Dispatcher raised: {err}" assert res is not None, "Expected a result dict" assert res.get("valid"), f"Result not valid: {res}" @then("validation should pass (structure matched, not exact characters)") def step_structure_matched(ctx: Context) -> None: """Whitespace-tolerant structural validation passes.""" d = getattr(ctx, "decision_dict", {}) ctx.validation_result = validate_decision_dict(d) # type: ignore[attr-defined] assert ctx.validation_result["valid"], ( f"Structural validation failed: {ctx.validation_result.get('errors', [])}" ) # type: ignore[attr-defined] @then("parent-child relationships should be validly connected") def step_relationships_valid(ctx: Context) -> None: """Parent-child nodes validate without errors.""" result = ctx.validation_result if hasattr(ctx, "validation_result") else {} # type: ignore[attr-defined] assert result.get("valid", False), ( f"Relation validation failed: {result.get('errors', [])}" )