cleveragents-core/benchmarks/safety_profile_bench.py

"""ASV benchmarks for Safety Profile domain model operations.

Measures the performance of:
- SafetyProfile construction (Pydantic validation)
- SafetyProfile serialization (model_dump / model_validate)
- resolve_safety_profile precedence resolution
- SafetyProfileProvenance enum operations
"""

from __future__ import annotations

import sys
from pathlib import Path

try:
    from cleveragents.domain.models.core.safety_profile import (
        DEFAULT_SAFETY_PROFILE,
        SafetyProfile,
        SafetyProfileProvenance,
        resolve_safety_profile,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.domain.models.core.safety_profile import (
        DEFAULT_SAFETY_PROFILE,
        SafetyProfile,
        SafetyProfileProvenance,
        resolve_safety_profile,
    )


# ---------------------------------------------------------------------------
# Module-level helpers
# ---------------------------------------------------------------------------


def _make_profile(**kwargs: object) -> SafetyProfile:
    """Create a SafetyProfile with sensible defaults for benchmarks."""
    return SafetyProfile(**kwargs)  # type: ignore[arg-type]


_FULL_CONFIG = {
    "allowed_skill_categories": ["code", "test", "deploy"],
    "require_sandbox": False,
    "require_checkpoints": False,
    "require_human_approval": True,
    "allow_unsafe_tools": True,
    "max_cost_per_plan": 100.0,
    "max_retries_per_step": 5,
    "max_total_cost": 500.0,
}


# ---------------------------------------------------------------------------
# Construction suite
# ---------------------------------------------------------------------------


class ConstructionSuite:
    """Benchmark SafetyProfile construction."""

    def time_default_construction(self) -> None:
        """Benchmark creating a default SafetyProfile."""
        SafetyProfile()

    def time_full_construction(self) -> None:
        """Benchmark creating a fully-populated SafetyProfile."""
        SafetyProfile(
            allowed_skill_categories=["code", "test", "deploy"],
            require_sandbox=False,
            require_checkpoints=False,
            require_human_approval=True,
            allow_unsafe_tools=True,
            max_cost_per_plan=100.0,
            max_retries_per_step=5,
            max_total_cost=500.0,
        )

    def time_from_config(self) -> None:
        """Benchmark SafetyProfile.from_config factory."""
        SafetyProfile.from_config(_FULL_CONFIG)


# ---------------------------------------------------------------------------
# Serialization suite
# ---------------------------------------------------------------------------


class SerializationSuite:
    """Benchmark SafetyProfile serialization round-trips."""

    def setup(self) -> None:
        """Create profiles for serialization benchmarks."""
        self.profile = _make_profile(
            allowed_skill_categories=["code", "test"],
            max_cost_per_plan=50.0,
            max_total_cost=200.0,
        )
        self.dump = self.profile.model_dump()

    def time_model_dump(self) -> None:
        """Benchmark model_dump serialization."""
        self.profile.model_dump()

    def time_model_dump_json(self) -> None:
        """Benchmark model_dump_json serialization."""
        self.profile.model_dump_json()

    def time_model_validate(self) -> None:
        """Benchmark model_validate deserialization."""
        SafetyProfile.model_validate(self.dump)


# ---------------------------------------------------------------------------
# Resolution suite
# ---------------------------------------------------------------------------


class ResolutionSuite:
    """Benchmark resolve_safety_profile precedence resolution."""

    def setup(self) -> None:
        """Create profiles for resolution benchmarks."""
        self.plan = SafetyProfile(allow_unsafe_tools=True, require_sandbox=False)
        self.action = SafetyProfile(allow_unsafe_tools=False)
        self.project = SafetyProfile(max_cost_per_plan=100.0)
        self.global_ = SafetyProfile()

    def time_resolve_all_levels(self) -> None:
        """Benchmark resolution with all four levels populated."""
        resolve_safety_profile(
            plan_profile=self.plan,
            action_profile=self.action,
            project_profile=self.project,
            global_profile=self.global_,
        )

    def time_resolve_global_only(self) -> None:
        """Benchmark resolution with only global level."""
        resolve_safety_profile(global_profile=self.global_)

    def time_resolve_none(self) -> None:
        """Benchmark resolution with all None (falls back to default)."""
        resolve_safety_profile()

    def time_resolve_action_level(self) -> None:
        """Benchmark resolution with action and lower levels."""
        resolve_safety_profile(
            action_profile=self.action,
            project_profile=self.project,
            global_profile=self.global_,
        )


# ---------------------------------------------------------------------------
# Enum suite
# ---------------------------------------------------------------------------


class DefaultProfileSuite:
    """Benchmark DEFAULT_SAFETY_PROFILE operations."""

    def time_default_access(self) -> None:
        """Benchmark accessing the DEFAULT_SAFETY_PROFILE constant."""
        _ = DEFAULT_SAFETY_PROFILE

    def time_default_model_dump(self) -> None:
        """Benchmark serializing the default profile."""
        DEFAULT_SAFETY_PROFILE.model_dump()


# ---------------------------------------------------------------------------
# Enum suite
# ---------------------------------------------------------------------------


class ProvenanceEnumSuite:
    """Benchmark SafetyProfileProvenance enum operations."""

    def time_enum_access(self) -> None:
        """Benchmark direct enum member access."""
        _ = SafetyProfileProvenance.PLAN

    def time_enum_from_value(self) -> None:
        """Benchmark enum construction from string value."""
        SafetyProfileProvenance("plan")

    def time_enum_iteration(self) -> None:
        """Benchmark enum iteration."""
        list(SafetyProfileProvenance)