forked from cleveragents/cleveragents-core
freemo
254fd07496
Implemented 5 new ASV benchmark files under benchmarks/: - providers_cost_table_bench.py — ProviderCostTable construction (default + custom entries), iteration throughput across all providers/models, fallback path for unknown providers - providers_cost_tracker_bench.py — CostTracker construction with various budget configurations, accumulation throughput (10/50 calls, mixed providers), daily spend tracking, get_cost_entry delegation - providers_fallback_selector_bench.py — FallbackSelector construction with custom order and cost tracker, selection when no providers configured (exhausts full list), selection with configured provider at various positions - providers_registry_bench.py — ProviderRegistry.get_all_providers, get_provider_info (by enum and string), is_provider_configured, multi-provider initialization - providers_llm_adapters_bench.py — LangChainChatProvider, AnthropicChatProvider, GoogleChatProvider, OpenAIChatProvider, OpenRouterChatProvider instantiation with various configurations Key design decisions: - Carefully audited existing cost_controls_bench.py and provider_selection_bench.py to avoid duplicating any already-covered benchmarks - Used MagicMock for Settings objects to avoid requiring real API keys in benchmarks - LLM adapter benchmarks use mock factories to measure pure instantiation cost without network calls - All benchmark classes use setup() fixtures to isolate measurement from fixture construction - 68 benchmark methods total across 5 files, all verified to execute without errors ISSUES CLOSED: #2800
180 lines
6.5 KiB
Python
180 lines
6.5 KiB
Python
"""ASV benchmarks for ProviderRegistry operations beyond initialization.
|
|
|
|
Measures the performance of:
|
|
- get_all_providers (all known providers, configured or not)
|
|
- get_provider_info (by ProviderType and by string name)
|
|
- is_provider_configured (by ProviderType and by string name)
|
|
- ProviderRegistry with multiple providers configured
|
|
|
|
Note: Registry initialization, get_default_provider_type,
|
|
get_configured_providers, and get_default_model are already covered by
|
|
provider_selection_bench.py and are not duplicated here.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.providers.registry import ( # noqa: E402
|
|
ProviderRegistry,
|
|
ProviderType,
|
|
reset_provider_registry,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_settings(**overrides: object) -> MagicMock:
|
|
"""Build a minimal mock Settings object for registry construction."""
|
|
settings = MagicMock()
|
|
settings.openai_api_key = None
|
|
settings.anthropic_api_key = None
|
|
settings.google_api_key = None
|
|
settings.gemini_api_key = None
|
|
settings.azure_api_key = None
|
|
settings.openrouter_api_key = None
|
|
settings.cohere_api_key = None
|
|
settings.groq_api_key = None
|
|
settings.together_api_key = None
|
|
settings.default_provider = None
|
|
settings.default_model = None
|
|
settings.azure_openai_endpoint = None
|
|
settings.azure_openai_api_version = None
|
|
settings.azure_openai_deployment = None
|
|
settings.openrouter_organization = None
|
|
settings.mock_providers = False
|
|
for key, value in overrides.items():
|
|
setattr(settings, key, value)
|
|
return settings
|
|
|
|
|
|
class TimeRegistryGetAllProviders:
|
|
"""Benchmark ProviderRegistry.get_all_providers."""
|
|
|
|
def setup(self) -> None:
|
|
reset_provider_registry()
|
|
self.registry_none = ProviderRegistry(settings=_make_settings())
|
|
self.registry_one = ProviderRegistry(
|
|
settings=_make_settings(openai_api_key="sk-bench-key")
|
|
)
|
|
self.registry_multi = ProviderRegistry(
|
|
settings=_make_settings(
|
|
openai_api_key="sk-bench-key",
|
|
anthropic_api_key="ant-bench-key",
|
|
google_api_key="goog-bench-key",
|
|
)
|
|
)
|
|
|
|
def time_get_all_providers_none_configured(self) -> None:
|
|
"""Time get_all_providers when no providers are configured."""
|
|
self.registry_none.get_all_providers()
|
|
|
|
def time_get_all_providers_one_configured(self) -> None:
|
|
"""Time get_all_providers when one provider is configured."""
|
|
self.registry_one.get_all_providers()
|
|
|
|
def time_get_all_providers_multi_configured(self) -> None:
|
|
"""Time get_all_providers when multiple providers are configured."""
|
|
self.registry_multi.get_all_providers()
|
|
|
|
|
|
class TimeRegistryGetProviderInfo:
|
|
"""Benchmark ProviderRegistry.get_provider_info lookups."""
|
|
|
|
def setup(self) -> None:
|
|
reset_provider_registry()
|
|
self.registry = ProviderRegistry(
|
|
settings=_make_settings(openai_api_key="sk-bench-key")
|
|
)
|
|
|
|
def time_get_provider_info_by_enum(self) -> None:
|
|
"""Time get_provider_info using a ProviderType enum value."""
|
|
self.registry.get_provider_info(ProviderType.OPENAI)
|
|
|
|
def time_get_provider_info_by_string(self) -> None:
|
|
"""Time get_provider_info using a string name (requires coercion)."""
|
|
self.registry.get_provider_info("openai")
|
|
|
|
def time_get_provider_info_unknown_string(self) -> None:
|
|
"""Time get_provider_info for an unknown provider name (returns None)."""
|
|
self.registry.get_provider_info("nonexistent_provider")
|
|
|
|
def time_get_provider_info_all_types(self) -> None:
|
|
"""Time get_provider_info for every known ProviderType."""
|
|
for provider_type in ProviderType:
|
|
self.registry.get_provider_info(provider_type)
|
|
|
|
|
|
class TimeRegistryIsProviderConfigured:
|
|
"""Benchmark ProviderRegistry.is_provider_configured checks."""
|
|
|
|
def setup(self) -> None:
|
|
reset_provider_registry()
|
|
self.registry = ProviderRegistry(
|
|
settings=_make_settings(
|
|
openai_api_key="sk-bench-key",
|
|
anthropic_api_key="ant-bench-key",
|
|
)
|
|
)
|
|
|
|
def time_is_configured_true_by_enum(self) -> None:
|
|
"""Time is_provider_configured for a configured provider (enum)."""
|
|
self.registry.is_provider_configured(ProviderType.OPENAI)
|
|
|
|
def time_is_configured_false_by_enum(self) -> None:
|
|
"""Time is_provider_configured for an unconfigured provider (enum)."""
|
|
self.registry.is_provider_configured(ProviderType.GROQ)
|
|
|
|
def time_is_configured_true_by_string(self) -> None:
|
|
"""Time is_provider_configured for a configured provider (string)."""
|
|
self.registry.is_provider_configured("anthropic")
|
|
|
|
def time_is_configured_false_by_string(self) -> None:
|
|
"""Time is_provider_configured for an unconfigured provider (string)."""
|
|
self.registry.is_provider_configured("cohere")
|
|
|
|
def time_is_configured_all_types(self) -> None:
|
|
"""Time is_provider_configured for every known ProviderType."""
|
|
for provider_type in ProviderType:
|
|
self.registry.is_provider_configured(provider_type)
|
|
|
|
|
|
class TimeRegistryMultiProviderInit:
|
|
"""Benchmark ProviderRegistry initialization with many providers configured."""
|
|
|
|
def setup(self) -> None:
|
|
self.settings_all = _make_settings(
|
|
openai_api_key="sk-bench-key",
|
|
anthropic_api_key="ant-bench-key",
|
|
google_api_key="goog-bench-key",
|
|
groq_api_key="groq-bench-key",
|
|
together_api_key="together-bench-key",
|
|
cohere_api_key="cohere-bench-key",
|
|
openrouter_api_key="or-bench-key",
|
|
)
|
|
|
|
def time_init_all_providers_configured(self) -> None:
|
|
"""Time registry initialization when all major providers are configured."""
|
|
reset_provider_registry()
|
|
ProviderRegistry(settings=self.settings_all)
|
|
|
|
def time_get_configured_providers_all(self) -> None:
|
|
"""Time get_configured_providers when all major providers are configured."""
|
|
reset_provider_registry()
|
|
registry = ProviderRegistry(settings=self.settings_all)
|
|
registry.get_configured_providers()
|