cleveragents-core/benchmarks/automation_profile_bench.py

"""ASV benchmarks for AutomationProfile validation, serialization, and guards.

Measures the performance of:
- AutomationProfile model construction (Pydantic validation)
- AutomationProfile.model_dump() serialization
- AutomationProfile.from_config() factory
- Built-in profile lookup via get_builtin_profile()
- BUILTIN_PROFILES iteration
- AutomationGuard check_guard() enforcement overhead
"""

from __future__ import annotations

import sys
from pathlib import Path

try:
    from cleveragents.domain.models.core.automation_guard import (
        AutomationGuard,
    )
    from cleveragents.domain.models.core.automation_profile import (
        BUILTIN_PROFILES,
        AutomationProfile,
        get_builtin_profile,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.domain.models.core.automation_guard import (
        AutomationGuard,
    )
    from cleveragents.domain.models.core.automation_profile import (
        BUILTIN_PROFILES,
        AutomationProfile,
        get_builtin_profile,
    )


def _make_profile() -> AutomationProfile:
    """Create a fully-populated profile for benchmarking."""
    return AutomationProfile(
        name="bench/test-profile",
        description="Benchmark profile",
        auto_strategize=0.7,
        auto_execute=0.5,
        auto_apply=1.0,
        auto_decisions_strategize=0.6,
        auto_decisions_execute=0.8,
        auto_validation_fix=0.3,
        auto_strategy_revision=0.9,
        auto_reversion_from_apply=0.4,
        auto_child_plans=0.7,
        auto_retry_transient=0.1,
        auto_checkpoint_restore=0.5,
        require_sandbox=True,
        require_checkpoints=True,
        allow_unsafe_tools=False,
    )


class ProfileValidationSuite:
    """Benchmark AutomationProfile construction."""

    def time_profile_construction(self) -> None:
        """Benchmark fully-populated profile creation."""
        _make_profile()

    def time_profile_minimal_construction(self) -> None:
        """Benchmark minimal profile creation."""
        AutomationProfile(name="bench/minimal")

    def time_profile_all_thresholds_max(self) -> None:
        """Benchmark profile with all thresholds at 1.0."""
        AutomationProfile(
            name="bench/max",
            auto_strategize=1.0,
            auto_execute=1.0,
            auto_apply=1.0,
            auto_decisions_strategize=1.0,
            auto_decisions_execute=1.0,
            auto_validation_fix=1.0,
            auto_strategy_revision=1.0,
            auto_reversion_from_apply=1.0,
            auto_child_plans=1.0,
            auto_retry_transient=1.0,
            auto_checkpoint_restore=1.0,
        )


class ProfileSerializationSuite:
    """Benchmark AutomationProfile serialization."""

    def setup(self) -> None:
        """Create objects for serialization benchmarks."""
        self.profile = _make_profile()

    def time_profile_model_dump(self) -> None:
        """Benchmark model_dump() serialization."""
        self.profile.model_dump()

    def time_profile_model_dump_json(self) -> None:
        """Benchmark model_dump_json() JSON serialization."""
        self.profile.model_dump_json()


class ProfileFromConfigSuite:
    """Benchmark AutomationProfile.from_config() factory."""

    def setup(self) -> None:
        """Prepare config dicts for benchmarks."""
        self.config = {
            "name": "bench/from-config",
            "description": "Config benchmark",
            "auto_strategize": 0.7,
            "auto_execute": 0.5,
            "auto_apply": 1.0,
        }

    def time_profile_from_config(self) -> None:
        """Benchmark from_config()."""
        AutomationProfile.from_config(self.config)


class BuiltinProfileSuite:
    """Benchmark built-in profile operations."""

    def time_get_builtin_profile(self) -> None:
        """Benchmark get_builtin_profile() lookup."""
        get_builtin_profile("cautious")

    def time_iterate_all_builtins(self) -> None:
        """Benchmark iterating all built-in profiles."""
        for name in BUILTIN_PROFILES:
            _ = BUILTIN_PROFILES[name]


class GuardCheckSuite:
    """Benchmark AutomationGuard check_guard() operations."""

    def setup(self) -> None:
        """Create profiles with various guard configurations."""
        self.no_guard_profile = AutomationProfile(name="bench/no-guard")
        self.guarded_profile = AutomationProfile(
            name="bench/guarded",
            guards=AutomationGuard(
                max_tool_calls_per_step=10,
                max_total_cost=100.0,
                tool_denylist=["dangerous_tool"],
                tool_allowlist=["safe_tool", "read_file", "list_dir"],
                require_approval_for_writes=True,
                require_approval_for_apply=True,
            ),
        )
        self.minimal_guard = AutomationProfile(
            name="bench/minimal-guard",
            guards=AutomationGuard(max_tool_calls_per_step=50),
        )

    def time_check_guard_no_guards(self) -> None:
        """Benchmark check_guard with no guards configured."""
        self.no_guard_profile.check_guard("any_tool")

    def time_check_guard_allowed(self) -> None:
        """Benchmark check_guard when tool is allowed."""
        self.guarded_profile.check_guard("safe_tool", calls_so_far=5)

    def time_check_guard_denied(self) -> None:
        """Benchmark check_guard when tool is on denylist."""
        self.guarded_profile.check_guard("dangerous_tool")

    def time_check_guard_write(self) -> None:
        """Benchmark check_guard with write flag."""
        self.guarded_profile.check_guard("safe_tool", is_write=True)

    def time_check_guard_cost(self) -> None:
        """Benchmark check_guard with cost check."""
        self.guarded_profile.check_guard(
            "safe_tool",
            cost_so_far=50.0,
        )

    def time_check_guard_minimal(self) -> None:
        """Benchmark check_guard with minimal guard config."""
        self.minimal_guard.check_guard("tool", calls_so_far=10)

    def time_guard_construction(self) -> None:
        """Benchmark AutomationGuard model construction."""
        AutomationGuard(
            max_tool_calls_per_step=10,
            max_total_cost=100.0,
            tool_denylist=["tool_a", "tool_b"],
        )