"""ASV benchmarks for validation edge case processing. Measures the performance of: - Fixture loading and parsing - Malformed output detection - Missing resource validation - Timeout simulation logic - Schema transform validation - Validation ordering and deduplication """ from __future__ import annotations import json import sys from pathlib import Path from typing import Any _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) _FIXTURE_DIR = ( Path(__file__).resolve().parents[1] / "features" / "fixtures" / "validation" ) def _load_fixture(name: str) -> list[dict[str, Any]]: """Load a named fixture file and return its fixture list.""" path = _FIXTURE_DIR / f"{name}.json" with open(path) as fh: data = json.load(fh) return data["fixtures"] def _validate_tool_output(case: dict[str, Any]) -> str | None: """Lightweight tool output validation for benchmarking.""" inp = case["input"] output = inp.get("output") if "tool_name" not in inp: return "missing required field: tool_name" if isinstance(output, str): return "tool output must be an object" if not isinstance(output, dict): return "tool output must be an object" if not output: return "tool output must contain at least 'status' and 'result'" if "status" not in output: return "missing required field: status" if not isinstance(output.get("status"), str): return "invalid type for field 'status'" if "result" in output and output["result"] is None: return "result payload must not be null" return None def _validate_resource(case: dict[str, Any]) -> str | None: """Lightweight resource reference validation for benchmarking.""" inp = case["input"] if "action_ref" in inp and "nonexistent" in inp["action_ref"]: return "action not found" if "project_refs" in inp: for ref in inp["project_refs"]: if ".." in ref: return "unresolvable project path" if "phase" in inp and "actor_config" in inp: phase = inp["phase"] actor_key = f"{phase}_actor" if actor_key not in inp["actor_config"]: return f"missing actor for phase '{phase}'" return None def _sort_validations(case: dict[str, Any]) -> tuple[list[str] | None, str | None]: """Sort validations by level priority.""" inp = case["input"] validations = inp["validations"] attachment_map: dict[str, str] = {} for v in validations: att_id = v.get("attachment_id") if att_id is not None: if att_id in attachment_map: return None, f"duplicate attachment ID: {att_id}" attachment_map[att_id] = v["id"] required = [v for v in validations if v["level"] == "required"] informational = [v for v in validations if v["level"] == "informational"] return [v["id"] for v in required + informational], None class ValidationEdgeFixtureLoadSuite: """Benchmarks for fixture loading and parsing.""" timeout = 60 def time_load_malformed_tool_output(self) -> None: """Benchmark loading malformed_tool_output fixture.""" _load_fixture("malformed_tool_output") def time_load_missing_resources(self) -> None: """Benchmark loading missing_resources fixture.""" _load_fixture("missing_resources") def time_load_validation_timeouts(self) -> None: """Benchmark loading validation_timeouts fixture.""" _load_fixture("validation_timeouts") def time_load_invalid_schema_transforms(self) -> None: """Benchmark loading invalid_schema_transforms fixture.""" _load_fixture("invalid_schema_transforms") def time_load_mixed_ordering(self) -> None: """Benchmark loading mixed_ordering fixture.""" _load_fixture("mixed_ordering") def time_load_all_fixtures(self) -> None: """Benchmark loading all fixture files.""" names = [ "malformed_tool_output", "missing_resources", "validation_timeouts", "invalid_schema_transforms", "mixed_ordering", ] for name in names: _load_fixture(name) class ValidationEdgeDetectionSuite: """Benchmarks for validation edge case detection.""" timeout = 60 def setup(self) -> None: """Load all fixtures for benchmarking.""" self.malformed_fixtures = _load_fixture("malformed_tool_output") self.resource_fixtures = _load_fixture("missing_resources") self.ordering_fixtures = _load_fixture("mixed_ordering") def time_malformed_output_detection(self) -> None: """Benchmark malformed output detection across all cases.""" for case in self.malformed_fixtures: _validate_tool_output(case) def time_missing_resource_detection(self) -> None: """Benchmark missing resource detection across all cases.""" for case in self.resource_fixtures: _validate_resource(case) def time_validation_ordering(self) -> None: """Benchmark validation ordering and duplicate detection.""" for case in self.ordering_fixtures: _sort_validations(case) def time_combined_edge_detection(self) -> None: """Benchmark combined edge case detection across all fixture types.""" for case in self.malformed_fixtures: _validate_tool_output(case) for case in self.resource_fixtures: _validate_resource(case) for case in self.ordering_fixtures: _sort_validations(case)