cleveragents-core/benchmarks/validation_edge_bench.py

"""ASV benchmarks for validation edge case processing.

Measures the performance of:
- Fixture loading and parsing
- Malformed output detection
- Missing resource validation
- Timeout simulation logic
- Schema transform validation
- Validation ordering and deduplication
"""

from __future__ import annotations

import json
import sys
from pathlib import Path
from typing import Any

_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

_FIXTURE_DIR = (
    Path(__file__).resolve().parents[1] / "features" / "fixtures" / "validation"
)


def _load_fixture(name: str) -> list[dict[str, Any]]:
    """Load a named fixture file and return its fixture list."""
    path = _FIXTURE_DIR / f"{name}.json"
    with open(path) as fh:
        data = json.load(fh)
    return data["fixtures"]


def _validate_tool_output(case: dict[str, Any]) -> str | None:
    """Lightweight tool output validation for benchmarking."""
    inp = case["input"]
    output = inp.get("output")
    if "tool_name" not in inp:
        return "missing required field: tool_name"
    if isinstance(output, str):
        return "tool output must be an object"
    if not isinstance(output, dict):
        return "tool output must be an object"
    if not output:
        return "tool output must contain at least 'status' and 'result'"
    if "status" not in output:
        return "missing required field: status"
    if not isinstance(output.get("status"), str):
        return "invalid type for field 'status'"
    if "result" in output and output["result"] is None:
        return "result payload must not be null"
    return None


def _validate_resource(case: dict[str, Any]) -> str | None:
    """Lightweight resource reference validation for benchmarking."""
    inp = case["input"]
    if "action_ref" in inp and "nonexistent" in inp["action_ref"]:
        return "action not found"
    if "project_refs" in inp:
        for ref in inp["project_refs"]:
            if ".." in ref:
                return "unresolvable project path"
    if "phase" in inp and "actor_config" in inp:
        phase = inp["phase"]
        actor_key = f"{phase}_actor"
        if actor_key not in inp["actor_config"]:
            return f"missing actor for phase '{phase}'"
    return None


def _sort_validations(case: dict[str, Any]) -> tuple[list[str] | None, str | None]:
    """Sort validations by level priority."""
    inp = case["input"]
    validations = inp["validations"]
    attachment_map: dict[str, str] = {}
    for v in validations:
        att_id = v.get("attachment_id")
        if att_id is not None:
            if att_id in attachment_map:
                return None, f"duplicate attachment ID: {att_id}"
            attachment_map[att_id] = v["id"]
    required = [v for v in validations if v["level"] == "required"]
    informational = [v for v in validations if v["level"] == "informational"]
    return [v["id"] for v in required + informational], None


class ValidationEdgeFixtureLoadSuite:
    """Benchmarks for fixture loading and parsing."""

    timeout = 60

    def time_load_malformed_tool_output(self) -> None:
        """Benchmark loading malformed_tool_output fixture."""
        _load_fixture("malformed_tool_output")

    def time_load_missing_resources(self) -> None:
        """Benchmark loading missing_resources fixture."""
        _load_fixture("missing_resources")

    def time_load_validation_timeouts(self) -> None:
        """Benchmark loading validation_timeouts fixture."""
        _load_fixture("validation_timeouts")

    def time_load_invalid_schema_transforms(self) -> None:
        """Benchmark loading invalid_schema_transforms fixture."""
        _load_fixture("invalid_schema_transforms")

    def time_load_mixed_ordering(self) -> None:
        """Benchmark loading mixed_ordering fixture."""
        _load_fixture("mixed_ordering")

    def time_load_all_fixtures(self) -> None:
        """Benchmark loading all fixture files."""
        names = [
            "malformed_tool_output",
            "missing_resources",
            "validation_timeouts",
            "invalid_schema_transforms",
            "mixed_ordering",
        ]
        for name in names:
            _load_fixture(name)


class ValidationEdgeDetectionSuite:
    """Benchmarks for validation edge case detection."""

    timeout = 60

    def setup(self) -> None:
        """Load all fixtures for benchmarking."""
        self.malformed_fixtures = _load_fixture("malformed_tool_output")
        self.resource_fixtures = _load_fixture("missing_resources")
        self.ordering_fixtures = _load_fixture("mixed_ordering")

    def time_malformed_output_detection(self) -> None:
        """Benchmark malformed output detection across all cases."""
        for case in self.malformed_fixtures:
            _validate_tool_output(case)

    def time_missing_resource_detection(self) -> None:
        """Benchmark missing resource detection across all cases."""
        for case in self.resource_fixtures:
            _validate_resource(case)

    def time_validation_ordering(self) -> None:
        """Benchmark validation ordering and duplicate detection."""
        for case in self.ordering_fixtures:
            _sort_validations(case)

    def time_combined_edge_detection(self) -> None:
        """Benchmark combined edge case detection across all fixture types."""
        for case in self.malformed_fixtures:
            _validate_tool_output(case)
        for case in self.resource_fixtures:
            _validate_resource(case)
        for case in self.ordering_fixtures:
            _sort_validations(case)