"""Helper script for Robot Framework validation edge case tests. Self-contained Python helper that loads validation fixtures and runs edge-case checks, printing sentinel strings on success and exiting with code 1 on failure. """ from __future__ import annotations import json import sys from pathlib import Path from typing import Any # Ensure src is importable when run from workspace root sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.core.exceptions import PlanError # --------------------------------------------------------------------------- # Fixture loading # --------------------------------------------------------------------------- _FIXTURE_DIR = ( Path(__file__).resolve().parents[1] / "features" / "fixtures" / "validation" ) def _load_fixture(name: str) -> list[dict[str, Any]]: """Load a named fixture file and return its fixture list.""" path = _FIXTURE_DIR / f"{name}.json" with open(path) as fh: data = json.load(fh) return data["fixtures"] # --------------------------------------------------------------------------- # Sub-commands # --------------------------------------------------------------------------- def test_load_fixtures() -> None: """Verify all fixture files load without errors.""" fixture_names = [ "malformed_tool_output", "missing_resources", "validation_timeouts", "invalid_schema_transforms", "mixed_ordering", ] for name in fixture_names: fixtures = _load_fixture(name) assert len(fixtures) > 0, f"fixture '{name}' must have at least one case" for case in fixtures: assert "name" in case, f"fixture case in '{name}' missing 'name'" assert "description" in case, ( f"fixture case in '{name}' missing 'description'" ) assert "input" in case, f"fixture case in '{name}' missing 'input'" print("fixtures-loaded-ok") def test_malformed_output() -> None: """Validate malformed tool output detection.""" fixtures = _load_fixture("malformed_tool_output") for case in fixtures: expected = case.get("expected_error") if expected is None: continue inp = case["input"] output = inp.get("output") # Basic structural checks if "tool_name" not in inp: assert "tool_name" in expected.lower() or "missing" in expected.lower() continue if isinstance(output, str): assert "object" in expected.lower() continue if isinstance(output, dict) and "status" not in output and not output: assert "status" in expected.lower() or "contain" in expected.lower() continue print("malformed-output-ok") def test_missing_resources() -> None: """Validate missing resource reference detection.""" fixtures = _load_fixture("missing_resources") for case in fixtures: expected = case.get("expected_error", "") assert isinstance(expected, str), ( f"expected_error must be a string for {case['name']}" ) # All missing resource cases must have an error assert len(expected) > 0, f"case '{case['name']}' should have expected_error" # Verify PlanError can be raised as expected try: raise PlanError("test missing resource error") except PlanError as exc: assert "test missing resource error" in str(exc) print("missing-resources-ok") def test_timeout_simulation() -> None: """Validate timeout simulation fixture data.""" fixtures = _load_fixture("validation_timeouts") for case in fixtures: inp = case["input"] timeout_ms = inp["timeout_ms"] expected = case["expected_error"] if timeout_ms < 0: assert "invalid timeout" in expected elif timeout_ms == 0: assert "exceeded 0ms" in expected else: assert "timeout" in expected.lower() or "deadline" in expected.lower() print("timeout-simulation-ok") def test_schema_validation() -> None: """Validate schema transform fixture data.""" fixtures = _load_fixture("invalid_schema_transforms") for case in fixtures: inp = case["input"] transform_result = inp["transform_result"] expected = case["expected_error"] if transform_result is None: assert "null" in expected elif not isinstance(transform_result, dict): assert "expected dict" in expected elif "type" not in transform_result: assert "type" in expected else: # Has type - check for additional problems assert len(expected) > 0 print("schema-validation-ok") # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- if __name__ == "__main__": cmd = sys.argv[1] if len(sys.argv) > 1 else "load_fixtures" commands = { "load_fixtures": test_load_fixtures, "malformed_output": test_malformed_output, "missing_resources": test_missing_resources, "timeout_simulation": test_timeout_simulation, "schema_validation": test_schema_validation, } handler = commands.get(cmd) if handler is None: print(f"Unknown command: {cmd}", file=sys.stderr) sys.exit(1) try: handler() except Exception as exc: print(f"FAIL: {exc}", file=sys.stderr) sys.exit(1)