forked from cleveragents/cleveragents-core
88 lines
2.7 KiB
Python
88 lines
2.7 KiB
Python
"""ASV benchmarks for provider selection and registry initialization.
|
|
|
|
Measures the performance of:
|
|
- Provider registry initialization (discovering configured providers)
|
|
- Provider resolution by name
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.providers.registry import ( # noqa: E402
|
|
ProviderRegistry,
|
|
ProviderType,
|
|
reset_provider_registry,
|
|
)
|
|
|
|
|
|
def _make_settings(openai: str | None = None) -> MagicMock:
|
|
settings = MagicMock()
|
|
settings.openai_api_key = openai
|
|
settings.anthropic_api_key = None
|
|
settings.google_api_key = None
|
|
settings.gemini_api_key = None
|
|
settings.azure_api_key = None
|
|
settings.openrouter_api_key = None
|
|
settings.cohere_api_key = None
|
|
settings.groq_api_key = None
|
|
settings.together_api_key = None
|
|
settings.default_provider = None
|
|
settings.default_model = None
|
|
settings.azure_openai_endpoint = None
|
|
settings.azure_openai_api_version = None
|
|
settings.azure_openai_deployment = None
|
|
settings.openrouter_organization = None
|
|
settings.mock_providers = False
|
|
return settings
|
|
|
|
|
|
class TimeProviderRegistryInit:
|
|
"""Benchmark provider registry initialization."""
|
|
|
|
def setup(self) -> None:
|
|
self.settings_none = _make_settings()
|
|
self.settings_openai = _make_settings(openai="sk-bench-key")
|
|
|
|
def time_provider_registry_init(self) -> None:
|
|
"""Time registry creation with no providers configured."""
|
|
reset_provider_registry()
|
|
ProviderRegistry(settings=self.settings_none)
|
|
|
|
def time_provider_registry_init_with_provider(self) -> None:
|
|
"""Time registry creation with one provider configured."""
|
|
reset_provider_registry()
|
|
ProviderRegistry(settings=self.settings_openai)
|
|
|
|
|
|
class TimeProviderResolution:
|
|
"""Benchmark provider resolution."""
|
|
|
|
def setup(self) -> None:
|
|
self.settings = _make_settings(openai="sk-bench-key")
|
|
reset_provider_registry()
|
|
self.registry = ProviderRegistry(settings=self.settings)
|
|
|
|
def time_provider_resolution(self) -> None:
|
|
"""Time default provider type resolution."""
|
|
self.registry.get_default_provider_type()
|
|
|
|
def time_get_configured_providers(self) -> None:
|
|
"""Time listing configured providers."""
|
|
self.registry.get_configured_providers()
|
|
|
|
def time_get_default_model(self) -> None:
|
|
"""Time default model resolution."""
|
|
self.registry.get_default_model(ProviderType.OPENAI)
|