diff --git a/features/coverage_validation_error_paths.feature b/features/coverage_validation_error_paths.feature new file mode 100644 index 000000000..8c126cd79 --- /dev/null +++ b/features/coverage_validation_error_paths.feature @@ -0,0 +1,146 @@ +@coverage +Feature: Structural validation error-path coverage + Exercises uncovered defensive branches in cleveragents.core.validation + (plan-tree, decision-dict, and structured-output validators) that the + existing structural_validation.feature does not reach. Pure-function tests + routed through the public validate_structured_component_output dispatcher. + + # -- plan tree validator -- + + Scenario: Plan tree node that is not a dict + When I structurally validate "plan_tree" with the JSON payload + """ + [123] + """ + Then the structural validation fails + And the structural errors include "is not a dict" + + Scenario: Plan tree with a duplicate decision_id + When I structurally validate "plan_tree" with the JSON payload + """ + [{"decision_id": "01ARZ3NDEKTSV4XXFFJFRC889A"}, {"decision_id": "01ARZ3NDEKTSV4XXFFJFRC889A"}] + """ + Then the structural validation fails + And the structural errors include "duplicate decision_id" + + Scenario: Plan tree node with a blank type + When I structurally validate "plan_tree" with the JSON payload + """ + [{"type": " "}] + """ + Then the structural validation fails + And the structural errors include "'type' must be a non-empty string" + + Scenario: Plan tree node with a blank question + When I structurally validate "plan_tree" with the JSON payload + """ + [{"question": ""}] + """ + Then the structural validation fails + And the structural errors include "'question' must be a non-empty string" + + Scenario: Plan tree children is not a list + When I structurally validate "plan_tree" with the JSON payload + """ + [{"children": "nope"}] + """ + Then the structural validation fails + And the structural errors include "'children' must be a list" + + Scenario: Plan tree child with an invalid ULID + When I structurally validate "plan_tree" with the JSON payload + """ + [{"children": [{"decision_id": "not-a-ulid"}]}] + """ + Then the structural validation fails + And the structural errors include "must be a valid ULID" + + # -- decision dict validator -- + + Scenario: Decision dict missing all required fields + When I structurally validate "decision" with the JSON payload + """ + {} + """ + Then the structural validation fails + And the structural errors include "missing required field" + + Scenario: Decision dict with an invalid decision_id ULID + When I structurally validate "decision" with the JSON payload + """ + {"decision_id": "bad", "plan_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "type": "t", "sequence": 0, "question": "q", "chosen": "c", "confidence": 0.5, "parent": "(root)", "is_correction": false, "superseded": false} + """ + Then the structural validation fails + And the structural errors include "must be a valid ULID" + + Scenario: Decision dict with a wrong-typed confidence + When I structurally validate "decision" with the JSON payload + """ + {"decision_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "plan_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "type": "t", "sequence": 0, "question": "q", "chosen": "c", "confidence": "high", "parent": "(root)", "is_correction": false, "superseded": false} + """ + Then the structural validation fails + And the structural errors include "must be float or None" + + Scenario: Decision dict with a non-string parent + When I structurally validate "decision" with the JSON payload + """ + {"decision_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "plan_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "type": "t", "sequence": 0, "question": "q", "chosen": "c", "confidence": null, "parent": 123, "is_correction": false, "superseded": false} + """ + Then the structural validation fails + And the structural errors include "'parent' must be a string" + + Scenario: Decision dict with a non-integer sequence + When I structurally validate "decision" with the JSON payload + """ + {"decision_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "plan_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "type": "t", "sequence": "five", "question": "q", "chosen": "c", "confidence": null, "parent": "(root)", "is_correction": false, "superseded": false} + """ + Then the structural validation fails + And the structural errors include "'sequence' must be an integer" + + # -- structured output validator -- + + Scenario: Structured output with a blank command + When I structurally validate "structured_output" with the JSON payload + """ + {"command": "", "session_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "status": "ok"} + """ + Then the structural validation fails + And the structural errors include "'command' must be a non-empty string" + + Scenario: Structured output with an invalid session_id + When I structurally validate "structured_output" with the JSON payload + """ + {"command": "run", "session_id": "bad", "status": "ok"} + """ + Then the structural validation fails + And the structural errors include "'session_id' must be a valid ULID" + + Scenario: Structured output with a non-integer exit_code + When I structurally validate "structured_output" with the JSON payload + """ + {"command": "run", "session_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "status": "ok", "exit_code": "x"} + """ + Then the structural validation fails + And the structural errors include "'exit_code' must be a non-negative integer" + + Scenario: Structured output with elements that are not a list + When I structurally validate "structured_output" with the JSON payload + """ + {"command": "run", "session_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "status": "ok", "elements": "nope"} + """ + Then the structural validation fails + And the structural errors include "'elements' must be a list" + + Scenario: Structured output with a non-dict element + When I structurally validate "structured_output" with the JSON payload + """ + {"command": "run", "session_id": "01ARZ3NDEKTSV4XXFFJFRC889A", "status": "ok", "elements": [123]} + """ + Then the structural validation fails + And the structural errors include "is not a dict" + + # -- dispatcher -- + + Scenario: Unknown target type raises a ValidationError + When I structurally validate the unknown target type "totally_unknown" + Then a structural ValidationError is raised diff --git a/features/steps/coverage_validation_error_paths_steps.py b/features/steps/coverage_validation_error_paths_steps.py new file mode 100644 index 000000000..0e08b18f0 --- /dev/null +++ b/features/steps/coverage_validation_error_paths_steps.py @@ -0,0 +1,73 @@ +"""Behave steps exercising uncovered error-path branches in +``cleveragents.core.validation``. + +Targets the defensive branches in the plan-tree, decision-dict, and +structured-output validators that ``structural_validation.feature`` does not +reach (non-dict nodes, duplicate decision_id, non-list children, invalid child +ULID, missing decision fields, wrong-typed confidence/parent/sequence, +malformed structured-output fields, unknown dispatcher target). These are pure +functions -- no database, CLI runner, or mocks required. + +Step phrasing is deliberately distinct from ``structural_validation_steps.py`` +to avoid step-definition collisions. +""" + +from __future__ import annotations + +import json + +from behave import then, when +from behave.runner import Context + +from cleveragents.core.validation import ( + ValidationError, + validate_structured_component_output, +) + + +@when('I structurally validate "{target_type}" with the JSON payload') +def step_structural_validate_payload(context: Context, target_type: str) -> None: + data = json.loads(context.text) + context.cov_validation_error = None + try: + context.cov_validation_result = validate_structured_component_output( + target_type, data + ) + except ValidationError as exc: + context.cov_validation_result = None + context.cov_validation_error = exc + + +@when('I structurally validate the unknown target type "{target_type}"') +def step_structural_validate_unknown(context: Context, target_type: str) -> None: + context.cov_validation_error = None + try: + context.cov_validation_result = validate_structured_component_output( + target_type, [] + ) + except ValidationError as exc: + context.cov_validation_result = None + context.cov_validation_error = exc + + +@then("the structural validation fails") +def step_structural_validation_fails(context: Context) -> None: + assert context.cov_validation_result is not None, "expected a result dict" + assert context.cov_validation_result["valid"] is False, ( + f"expected invalid, got {context.cov_validation_result}" + ) + + +@then('the structural errors include "{substring}"') +def step_structural_errors_include(context: Context, substring: str) -> None: + errors = context.cov_validation_result["errors"] + assert any(substring in err for err in errors), ( + f"no error contained {substring!r}; errors={errors}" + ) + + +@then("a structural ValidationError is raised") +def step_structural_validation_error_raised(context: Context) -> None: + assert isinstance(context.cov_validation_error, ValidationError), ( + f"expected ValidationError, got {context.cov_validation_error!r}" + ) diff --git a/noxfile.py b/noxfile.py index 300d54cb4..b715728f8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -905,6 +905,11 @@ def coverage_report(session: nox.Session): "*/.nox/*", "src/cleveragents/discovery/*", "src/cleveragents/tui/materializer.py", + # TYPE_CHECKING-only re-export shim: 100% of its "missing" lines are + # inside an `if TYPE_CHECKING:` block (never executes at runtime), + # and slipcover has no per-line pragma to exclude them. Omit the + # whole module so those uncoverable lines stop dragging the total. + "src/cleveragents/application/services/__init__.py", ] ) diff --git a/pyproject.toml b/pyproject.toml index 976480cda..d2de7e142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,6 +202,9 @@ omit = [ "*/.nox/*", "src/cleveragents/discovery/*", "src/cleveragents/tui/materializer.py", + # TYPE_CHECKING-only re-export shim (every miss is in an `if TYPE_CHECKING:` + # block, never executes at runtime; slipcover has no per-line pragma). + "src/cleveragents/application/services/__init__.py", ] data_file = "build/.coverage"