"""ASV benchmarks for AutomationProfile validation, serialization, and guards. Measures the performance of: - AutomationProfile model construction (Pydantic validation) - AutomationProfile.model_dump() serialization - AutomationProfile.from_config() factory - Built-in profile lookup via get_builtin_profile() - BUILTIN_PROFILES iteration - AutomationGuard check_guard() enforcement overhead """ from __future__ import annotations import sys from pathlib import Path try: from cleveragents.domain.models.core.automation_guard import ( AutomationGuard, ) from cleveragents.domain.models.core.automation_profile import ( BUILTIN_PROFILES, AutomationProfile, get_builtin_profile, ) from cleveragents.domain.models.core.safety_profile import SafetyProfile except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.domain.models.core.automation_guard import ( AutomationGuard, ) from cleveragents.domain.models.core.automation_profile import ( BUILTIN_PROFILES, AutomationProfile, get_builtin_profile, ) from cleveragents.domain.models.core.safety_profile import SafetyProfile def _make_profile() -> AutomationProfile: """Create a fully-populated profile for benchmarking.""" return AutomationProfile( name="bench/test-profile", description="Benchmark profile", decompose_task=0.7, create_tool=0.5, select_tool=1.0, edit_code=0.6, execute_command=0.8, create_file=0.3, delete_content=0.9, access_network=0.4, install_dependency=0.7, modify_config=0.1, approve_plan=0.5, safety=SafetyProfile( require_sandbox=True, require_checkpoints=True, allow_unsafe_tools=False, ), ) class ProfileValidationSuite: """Benchmark AutomationProfile construction.""" def time_profile_construction(self) -> None: """Benchmark fully-populated profile creation.""" _make_profile() def time_profile_minimal_construction(self) -> None: """Benchmark minimal profile creation.""" AutomationProfile(name="bench/minimal") def time_profile_all_thresholds_max(self) -> None: """Benchmark profile with all thresholds at 1.0.""" AutomationProfile( name="bench/max", decompose_task=1.0, create_tool=1.0, select_tool=1.0, edit_code=1.0, execute_command=1.0, create_file=1.0, delete_content=1.0, access_network=1.0, install_dependency=1.0, modify_config=1.0, approve_plan=1.0, ) class ProfileSerializationSuite: """Benchmark AutomationProfile serialization.""" def setup(self) -> None: """Create objects for serialization benchmarks.""" self.profile = _make_profile() def time_profile_model_dump(self) -> None: """Benchmark model_dump() serialization.""" self.profile.model_dump() def time_profile_model_dump_json(self) -> None: """Benchmark model_dump_json() JSON serialization.""" self.profile.model_dump_json() class ProfileFromConfigSuite: """Benchmark AutomationProfile.from_config() factory.""" def setup(self) -> None: """Prepare config dicts for benchmarks.""" self.config = { "name": "bench/from-config", "description": "Config benchmark", "decompose_task": 0.7, "create_tool": 0.5, "select_tool": 1.0, } def time_profile_from_config(self) -> None: """Benchmark from_config().""" AutomationProfile.from_config(self.config) class BuiltinProfileSuite: """Benchmark built-in profile operations.""" def time_get_builtin_profile(self) -> None: """Benchmark get_builtin_profile() lookup.""" get_builtin_profile("cautious") def time_iterate_all_builtins(self) -> None: """Benchmark iterating all built-in profiles.""" for name in BUILTIN_PROFILES: _ = BUILTIN_PROFILES[name] class GuardCheckSuite: """Benchmark AutomationGuard check_guard() operations.""" def setup(self) -> None: """Create profiles with various guard configurations.""" self.no_guard_profile = AutomationProfile(name="bench/no-guard") self.guarded_profile = AutomationProfile( name="bench/guarded", guards=AutomationGuard( max_tool_calls_per_step=10, max_total_cost=100.0, tool_denylist=["dangerous_tool"], tool_allowlist=["safe_tool", "read_file", "list_dir"], require_approval_for_writes=True, require_approval_for_apply=True, ), ) self.minimal_guard = AutomationProfile( name="bench/minimal-guard", guards=AutomationGuard(max_tool_calls_per_step=50), ) def time_check_guard_no_guards(self) -> None: """Benchmark check_guard with no guards configured.""" self.no_guard_profile.check_guard("any_tool") def time_check_guard_allowed(self) -> None: """Benchmark check_guard when tool is allowed.""" self.guarded_profile.check_guard("safe_tool", calls_so_far=5) def time_check_guard_denied(self) -> None: """Benchmark check_guard when tool is on denylist.""" self.guarded_profile.check_guard("dangerous_tool") def time_check_guard_write(self) -> None: """Benchmark check_guard with write flag.""" self.guarded_profile.check_guard("safe_tool", is_write=True) def time_check_guard_cost(self) -> None: """Benchmark check_guard with cost check.""" self.guarded_profile.check_guard( "safe_tool", cost_so_far=50.0, ) def time_check_guard_minimal(self) -> None: """Benchmark check_guard with minimal guard config.""" self.minimal_guard.check_guard("tool", calls_so_far=10) def time_guard_construction(self) -> None: """Benchmark AutomationGuard model construction.""" AutomationGuard( max_tool_calls_per_step=10, max_total_cost=100.0, tool_denylist=["tool_a", "tool_b"], )