"""ASV benchmarks for Safety Profile domain model operations. Measures the performance of: - SafetyProfile construction (Pydantic validation) - SafetyProfile serialization (model_dump / model_validate) - resolve_safety_profile precedence resolution - SafetyProfileProvenance enum operations """ from __future__ import annotations import sys from pathlib import Path try: from cleveragents.domain.models.core.safety_profile import ( DEFAULT_SAFETY_PROFILE, SafetyProfile, SafetyProfileProvenance, resolve_safety_profile, ) except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.domain.models.core.safety_profile import ( DEFAULT_SAFETY_PROFILE, SafetyProfile, SafetyProfileProvenance, resolve_safety_profile, ) # --------------------------------------------------------------------------- # Module-level helpers # --------------------------------------------------------------------------- def _make_profile(**kwargs: object) -> SafetyProfile: """Create a SafetyProfile with sensible defaults for benchmarks.""" return SafetyProfile(**kwargs) # type: ignore[arg-type] _FULL_CONFIG = { "allowed_skill_categories": ["code", "test", "deploy"], "require_sandbox": False, "require_checkpoints": False, "require_human_approval": True, "allow_unsafe_tools": True, "max_cost_per_plan": 100.0, "max_retries_per_step": 5, "max_total_cost": 500.0, } # --------------------------------------------------------------------------- # Construction suite # --------------------------------------------------------------------------- class ConstructionSuite: """Benchmark SafetyProfile construction.""" def time_default_construction(self) -> None: """Benchmark creating a default SafetyProfile.""" SafetyProfile() def time_full_construction(self) -> None: """Benchmark creating a fully-populated SafetyProfile.""" SafetyProfile( allowed_skill_categories=["code", "test", "deploy"], require_sandbox=False, require_checkpoints=False, require_human_approval=True, allow_unsafe_tools=True, max_cost_per_plan=100.0, max_retries_per_step=5, max_total_cost=500.0, ) def time_from_config(self) -> None: """Benchmark SafetyProfile.from_config factory.""" SafetyProfile.from_config(_FULL_CONFIG) # --------------------------------------------------------------------------- # Serialization suite # --------------------------------------------------------------------------- class SerializationSuite: """Benchmark SafetyProfile serialization round-trips.""" def setup(self) -> None: """Create profiles for serialization benchmarks.""" self.profile = _make_profile( allowed_skill_categories=["code", "test"], max_cost_per_plan=50.0, max_total_cost=200.0, ) self.dump = self.profile.model_dump() def time_model_dump(self) -> None: """Benchmark model_dump serialization.""" self.profile.model_dump() def time_model_dump_json(self) -> None: """Benchmark model_dump_json serialization.""" self.profile.model_dump_json() def time_model_validate(self) -> None: """Benchmark model_validate deserialization.""" SafetyProfile.model_validate(self.dump) # --------------------------------------------------------------------------- # Resolution suite # --------------------------------------------------------------------------- class ResolutionSuite: """Benchmark resolve_safety_profile precedence resolution.""" def setup(self) -> None: """Create profiles for resolution benchmarks.""" self.plan = SafetyProfile(allow_unsafe_tools=True, require_sandbox=False) self.action = SafetyProfile(allow_unsafe_tools=False) self.project = SafetyProfile(max_cost_per_plan=100.0) self.global_ = SafetyProfile() def time_resolve_all_levels(self) -> None: """Benchmark resolution with all four levels populated.""" resolve_safety_profile( plan_profile=self.plan, action_profile=self.action, project_profile=self.project, global_profile=self.global_, ) def time_resolve_global_only(self) -> None: """Benchmark resolution with only global level.""" resolve_safety_profile(global_profile=self.global_) def time_resolve_none(self) -> None: """Benchmark resolution with all None (falls back to default).""" resolve_safety_profile() def time_resolve_action_level(self) -> None: """Benchmark resolution with action and lower levels.""" resolve_safety_profile( action_profile=self.action, project_profile=self.project, global_profile=self.global_, ) # --------------------------------------------------------------------------- # Enum suite # --------------------------------------------------------------------------- class DefaultProfileSuite: """Benchmark DEFAULT_SAFETY_PROFILE operations.""" def time_default_access(self) -> None: """Benchmark accessing the DEFAULT_SAFETY_PROFILE constant.""" _ = DEFAULT_SAFETY_PROFILE def time_default_model_dump(self) -> None: """Benchmark serializing the default profile.""" DEFAULT_SAFETY_PROFILE.model_dump() # --------------------------------------------------------------------------- # Enum suite # --------------------------------------------------------------------------- class ProvenanceEnumSuite: """Benchmark SafetyProfileProvenance enum operations.""" def time_enum_access(self) -> None: """Benchmark direct enum member access.""" _ = SafetyProfileProvenance.PLAN def time_enum_from_value(self) -> None: """Benchmark enum construction from string value.""" SafetyProfileProvenance("plan") def time_enum_iteration(self) -> None: """Benchmark enum iteration.""" list(SafetyProfileProvenance)