cleveragents-core/benchmarks/automation_profile_bench.py

"""ASV benchmarks for AutomationProfile validation, serialization, and guards.

Measures the performance of:
- AutomationProfile model construction (Pydantic validation)
- AutomationProfile.model_dump() serialization
- AutomationProfile.from_config() factory
- Built-in profile lookup via get_builtin_profile()
- BUILTIN_PROFILES iteration
- AutomationGuard check_guard() enforcement overhead
"""

from __future__ import annotations

import sys
from pathlib import Path

try:
    from cleveragents.domain.models.core.automation_guard import (
        AutomationGuard,
    )
    from cleveragents.domain.models.core.automation_profile import (
        BUILTIN_PROFILES,
        AutomationProfile,
        get_builtin_profile,
    )
    from cleveragents.domain.models.core.safety_profile import SafetyProfile
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.domain.models.core.automation_guard import (
        AutomationGuard,
    )
    from cleveragents.domain.models.core.automation_profile import (
        BUILTIN_PROFILES,
        AutomationProfile,
        get_builtin_profile,
    )
    from cleveragents.domain.models.core.safety_profile import SafetyProfile


def _make_profile() -> AutomationProfile:
    """Create a fully-populated profile for benchmarking."""
    return AutomationProfile(
        name="bench/test-profile",
        description="Benchmark profile",
        decompose_task=0.7,
        create_tool=0.5,
        select_tool=1.0,
        edit_code=0.6,
        execute_command=0.8,
        create_file=0.3,
        delete_content=0.9,
        access_network=0.4,
        install_dependency=0.7,
        modify_config=0.1,
        approve_plan=0.5,
        safety=SafetyProfile(
            require_sandbox=True,
            require_checkpoints=True,
            allow_unsafe_tools=False,
        ),
    )


class ProfileValidationSuite:
    """Benchmark AutomationProfile construction."""

    def time_profile_construction(self) -> None:
        """Benchmark fully-populated profile creation."""
        _make_profile()

    def time_profile_minimal_construction(self) -> None:
        """Benchmark minimal profile creation."""
        AutomationProfile(name="bench/minimal")

    def time_profile_all_thresholds_max(self) -> None:
        """Benchmark profile with all thresholds at 1.0."""
        AutomationProfile(
            name="bench/max",
            decompose_task=1.0,
            create_tool=1.0,
            select_tool=1.0,
            edit_code=1.0,
            execute_command=1.0,
            create_file=1.0,
            delete_content=1.0,
            access_network=1.0,
            install_dependency=1.0,
            modify_config=1.0,
            approve_plan=1.0,
        )


class ProfileSerializationSuite:
    """Benchmark AutomationProfile serialization."""

    def setup(self) -> None:
        """Create objects for serialization benchmarks."""
        self.profile = _make_profile()

    def time_profile_model_dump(self) -> None:
        """Benchmark model_dump() serialization."""
        self.profile.model_dump()

    def time_profile_model_dump_json(self) -> None:
        """Benchmark model_dump_json() JSON serialization."""
        self.profile.model_dump_json()


class ProfileFromConfigSuite:
    """Benchmark AutomationProfile.from_config() factory."""

    def setup(self) -> None:
        """Prepare config dicts for benchmarks."""
        self.config = {
            "name": "bench/from-config",
            "description": "Config benchmark",
            "decompose_task": 0.7,
            "create_tool": 0.5,
            "select_tool": 1.0,
        }

    def time_profile_from_config(self) -> None:
        """Benchmark from_config()."""
        AutomationProfile.from_config(self.config)


class BuiltinProfileSuite:
    """Benchmark built-in profile operations."""

    def time_get_builtin_profile(self) -> None:
        """Benchmark get_builtin_profile() lookup."""
        get_builtin_profile("cautious")

    def time_iterate_all_builtins(self) -> None:
        """Benchmark iterating all built-in profiles."""
        for name in BUILTIN_PROFILES:
            _ = BUILTIN_PROFILES[name]


class GuardCheckSuite:
    """Benchmark AutomationGuard check_guard() operations."""

    def setup(self) -> None:
        """Create profiles with various guard configurations."""
        self.no_guard_profile = AutomationProfile(name="bench/no-guard")
        self.guarded_profile = AutomationProfile(
            name="bench/guarded",
            guards=AutomationGuard(
                max_tool_calls_per_step=10,
                max_total_cost=100.0,
                tool_denylist=["dangerous_tool"],
                tool_allowlist=["safe_tool", "read_file", "list_dir"],
                require_approval_for_writes=True,
                require_approval_for_apply=True,
            ),
        )
        self.minimal_guard = AutomationProfile(
            name="bench/minimal-guard",
            guards=AutomationGuard(max_tool_calls_per_step=50),
        )

    def time_check_guard_no_guards(self) -> None:
        """Benchmark check_guard with no guards configured."""
        self.no_guard_profile.check_guard("any_tool")

    def time_check_guard_allowed(self) -> None:
        """Benchmark check_guard when tool is allowed."""
        self.guarded_profile.check_guard("safe_tool", calls_so_far=5)

    def time_check_guard_denied(self) -> None:
        """Benchmark check_guard when tool is on denylist."""
        self.guarded_profile.check_guard("dangerous_tool")

    def time_check_guard_write(self) -> None:
        """Benchmark check_guard with write flag."""
        self.guarded_profile.check_guard("safe_tool", is_write=True)

    def time_check_guard_cost(self) -> None:
        """Benchmark check_guard with cost check."""
        self.guarded_profile.check_guard(
            "safe_tool",
            cost_so_far=50.0,
        )

    def time_check_guard_minimal(self) -> None:
        """Benchmark check_guard with minimal guard config."""
        self.minimal_guard.check_guard("tool", calls_so_far=10)

    def time_guard_construction(self) -> None:
        """Benchmark AutomationGuard model construction."""
        AutomationGuard(
            max_tool_calls_per_step=10,
            max_total_cost=100.0,
            tool_denylist=["tool_a", "tool_b"],
        )