160 lines
5.5 KiB
Python
160 lines
5.5 KiB
Python
"""Helper script for Robot Framework validation edge case tests.
|
|
|
|
Self-contained Python helper that loads validation fixtures and runs
|
|
edge-case checks, printing sentinel strings on success and exiting
|
|
with code 1 on failure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
# Ensure src is importable when run from workspace root
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
|
|
from cleveragents.core.exceptions import PlanError
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture loading
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_FIXTURE_DIR = (
|
|
Path(__file__).resolve().parents[1] / "features" / "fixtures" / "validation"
|
|
)
|
|
|
|
|
|
def _load_fixture(name: str) -> list[dict[str, Any]]:
|
|
"""Load a named fixture file and return its fixture list."""
|
|
path = _FIXTURE_DIR / f"{name}.json"
|
|
with open(path) as fh:
|
|
data = json.load(fh)
|
|
return data["fixtures"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sub-commands
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_load_fixtures() -> None:
|
|
"""Verify all fixture files load without errors."""
|
|
fixture_names = [
|
|
"malformed_tool_output",
|
|
"missing_resources",
|
|
"validation_timeouts",
|
|
"invalid_schema_transforms",
|
|
"mixed_ordering",
|
|
]
|
|
for name in fixture_names:
|
|
fixtures = _load_fixture(name)
|
|
assert len(fixtures) > 0, f"fixture '{name}' must have at least one case"
|
|
for case in fixtures:
|
|
assert "name" in case, f"fixture case in '{name}' missing 'name'"
|
|
assert "description" in case, (
|
|
f"fixture case in '{name}' missing 'description'"
|
|
)
|
|
assert "input" in case, f"fixture case in '{name}' missing 'input'"
|
|
print("fixtures-loaded-ok")
|
|
|
|
|
|
def test_malformed_output() -> None:
|
|
"""Validate malformed tool output detection."""
|
|
fixtures = _load_fixture("malformed_tool_output")
|
|
for case in fixtures:
|
|
expected = case.get("expected_error")
|
|
if expected is None:
|
|
continue
|
|
inp = case["input"]
|
|
output = inp.get("output")
|
|
# Basic structural checks
|
|
if "tool_name" not in inp:
|
|
assert "tool_name" in expected.lower() or "missing" in expected.lower()
|
|
continue
|
|
if isinstance(output, str):
|
|
assert "object" in expected.lower()
|
|
continue
|
|
if isinstance(output, dict) and "status" not in output and not output:
|
|
assert "status" in expected.lower() or "contain" in expected.lower()
|
|
continue
|
|
print("malformed-output-ok")
|
|
|
|
|
|
def test_missing_resources() -> None:
|
|
"""Validate missing resource reference detection."""
|
|
fixtures = _load_fixture("missing_resources")
|
|
for case in fixtures:
|
|
expected = case.get("expected_error", "")
|
|
assert isinstance(expected, str), (
|
|
f"expected_error must be a string for {case['name']}"
|
|
)
|
|
# All missing resource cases must have an error
|
|
assert len(expected) > 0, f"case '{case['name']}' should have expected_error"
|
|
# Verify PlanError can be raised as expected
|
|
try:
|
|
raise PlanError("test missing resource error")
|
|
except PlanError as exc:
|
|
assert "test missing resource error" in str(exc)
|
|
print("missing-resources-ok")
|
|
|
|
|
|
def test_timeout_simulation() -> None:
|
|
"""Validate timeout simulation fixture data."""
|
|
fixtures = _load_fixture("validation_timeouts")
|
|
for case in fixtures:
|
|
inp = case["input"]
|
|
timeout_ms = inp["timeout_ms"]
|
|
expected = case["expected_error"]
|
|
if timeout_ms < 0:
|
|
assert "invalid timeout" in expected
|
|
elif timeout_ms == 0:
|
|
assert "exceeded 0ms" in expected
|
|
else:
|
|
assert "timeout" in expected.lower() or "deadline" in expected.lower()
|
|
print("timeout-simulation-ok")
|
|
|
|
|
|
def test_schema_validation() -> None:
|
|
"""Validate schema transform fixture data."""
|
|
fixtures = _load_fixture("invalid_schema_transforms")
|
|
for case in fixtures:
|
|
inp = case["input"]
|
|
transform_result = inp["transform_result"]
|
|
expected = case["expected_error"]
|
|
if transform_result is None:
|
|
assert "null" in expected
|
|
elif not isinstance(transform_result, dict):
|
|
assert "expected dict" in expected
|
|
elif "type" not in transform_result:
|
|
assert "type" in expected
|
|
else:
|
|
# Has type - check for additional problems
|
|
assert len(expected) > 0
|
|
print("schema-validation-ok")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if __name__ == "__main__":
|
|
cmd = sys.argv[1] if len(sys.argv) > 1 else "load_fixtures"
|
|
commands = {
|
|
"load_fixtures": test_load_fixtures,
|
|
"malformed_output": test_malformed_output,
|
|
"missing_resources": test_missing_resources,
|
|
"timeout_simulation": test_timeout_simulation,
|
|
"schema_validation": test_schema_validation,
|
|
}
|
|
handler = commands.get(cmd)
|
|
if handler is None:
|
|
print(f"Unknown command: {cmd}", file=sys.stderr)
|
|
sys.exit(1)
|
|
try:
|
|
handler()
|
|
except Exception as exc:
|
|
print(f"FAIL: {exc}", file=sys.stderr)
|
|
sys.exit(1)
|