forked from HAL9000/cleveragents-core
127 lines
3.8 KiB
Python
127 lines
3.8 KiB
Python
"""ASV benchmarks for AutomationProfile validation and serialization.
|
|
|
|
Measures the performance of:
|
|
- AutomationProfile model construction (Pydantic validation)
|
|
- AutomationProfile.model_dump() serialization
|
|
- AutomationProfile.from_config() factory
|
|
- Built-in profile lookup via get_builtin_profile()
|
|
- BUILTIN_PROFILES iteration
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.domain.models.core.automation_profile import (
|
|
BUILTIN_PROFILES,
|
|
AutomationProfile,
|
|
get_builtin_profile,
|
|
)
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.domain.models.core.automation_profile import (
|
|
BUILTIN_PROFILES,
|
|
AutomationProfile,
|
|
get_builtin_profile,
|
|
)
|
|
|
|
|
|
def _make_profile() -> AutomationProfile:
|
|
"""Create a fully-populated profile for benchmarking."""
|
|
return AutomationProfile(
|
|
name="bench/test-profile",
|
|
description="Benchmark profile",
|
|
auto_strategize=0.7,
|
|
auto_execute=0.5,
|
|
auto_apply=1.0,
|
|
auto_decisions_strategize=0.6,
|
|
auto_decisions_execute=0.8,
|
|
auto_validation_fix=0.3,
|
|
auto_strategy_revision=0.9,
|
|
auto_reversion_from_apply=0.4,
|
|
auto_child_plans=0.7,
|
|
auto_retry_transient=0.1,
|
|
auto_checkpoint_restore=0.5,
|
|
require_sandbox=True,
|
|
require_checkpoints=True,
|
|
allow_unsafe_tools=False,
|
|
)
|
|
|
|
|
|
class ProfileValidationSuite:
|
|
"""Benchmark AutomationProfile construction."""
|
|
|
|
def time_profile_construction(self) -> None:
|
|
"""Benchmark fully-populated profile creation."""
|
|
_make_profile()
|
|
|
|
def time_profile_minimal_construction(self) -> None:
|
|
"""Benchmark minimal profile creation."""
|
|
AutomationProfile(name="bench/minimal")
|
|
|
|
def time_profile_all_thresholds_max(self) -> None:
|
|
"""Benchmark profile with all thresholds at 1.0."""
|
|
AutomationProfile(
|
|
name="bench/max",
|
|
auto_strategize=1.0,
|
|
auto_execute=1.0,
|
|
auto_apply=1.0,
|
|
auto_decisions_strategize=1.0,
|
|
auto_decisions_execute=1.0,
|
|
auto_validation_fix=1.0,
|
|
auto_strategy_revision=1.0,
|
|
auto_reversion_from_apply=1.0,
|
|
auto_child_plans=1.0,
|
|
auto_retry_transient=1.0,
|
|
auto_checkpoint_restore=1.0,
|
|
)
|
|
|
|
|
|
class ProfileSerializationSuite:
|
|
"""Benchmark AutomationProfile serialization."""
|
|
|
|
def setup(self) -> None:
|
|
"""Create objects for serialization benchmarks."""
|
|
self.profile = _make_profile()
|
|
|
|
def time_profile_model_dump(self) -> None:
|
|
"""Benchmark model_dump() serialization."""
|
|
self.profile.model_dump()
|
|
|
|
def time_profile_model_dump_json(self) -> None:
|
|
"""Benchmark model_dump_json() JSON serialization."""
|
|
self.profile.model_dump_json()
|
|
|
|
|
|
class ProfileFromConfigSuite:
|
|
"""Benchmark AutomationProfile.from_config() factory."""
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare config dicts for benchmarks."""
|
|
self.config = {
|
|
"name": "bench/from-config",
|
|
"description": "Config benchmark",
|
|
"auto_strategize": 0.7,
|
|
"auto_execute": 0.5,
|
|
"auto_apply": 1.0,
|
|
}
|
|
|
|
def time_profile_from_config(self) -> None:
|
|
"""Benchmark from_config()."""
|
|
AutomationProfile.from_config(self.config)
|
|
|
|
|
|
class BuiltinProfileSuite:
|
|
"""Benchmark built-in profile operations."""
|
|
|
|
def time_get_builtin_profile(self) -> None:
|
|
"""Benchmark get_builtin_profile() lookup."""
|
|
get_builtin_profile("cautious")
|
|
|
|
def time_iterate_all_builtins(self) -> None:
|
|
"""Benchmark iterating all built-in profiles."""
|
|
for name in BUILTIN_PROFILES:
|
|
_ = BUILTIN_PROFILES[name]
|