122 lines
4.1 KiB
Python
122 lines
4.1 KiB
Python
"""ASV benchmarks for provider selection and resolution performance.
|
|
|
|
Measures the performance of:
|
|
- Provider auto-detection from configured API keys
|
|
- Provider resolution by name
|
|
- Default provider/model selection
|
|
- Provider registry initialisation
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.config.settings import Settings # noqa: E402
|
|
from cleveragents.providers.registry import ( # noqa: E402
|
|
ProviderRegistry,
|
|
reset_provider_registry,
|
|
)
|
|
|
|
|
|
class ProviderRegistryInitSuite:
|
|
"""Benchmark registry initialisation cost."""
|
|
|
|
def setup(self) -> None:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
# Ensure at least one provider key for realistic behaviour
|
|
os.environ["OPENAI_API_KEY"] = "bench-key-1234"
|
|
|
|
def teardown(self) -> None:
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
def time_registry_init(self) -> None:
|
|
"""Benchmark creating a fresh ProviderRegistry."""
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
ProviderRegistry()
|
|
|
|
def time_registry_get_configured(self) -> None:
|
|
"""Benchmark listing configured providers."""
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
registry = ProviderRegistry()
|
|
registry.get_configured_providers()
|
|
|
|
|
|
class ProviderSelectionSuite:
|
|
"""Benchmark provider selection logic."""
|
|
|
|
def setup(self) -> None:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
os.environ["OPENAI_API_KEY"] = "bench-key-1234"
|
|
self._registry = ProviderRegistry()
|
|
|
|
def teardown(self) -> None:
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
def time_get_default_provider_type(self) -> None:
|
|
"""Benchmark default provider type resolution."""
|
|
self._registry.get_default_provider_type()
|
|
|
|
def time_get_default_model(self) -> None:
|
|
"""Benchmark default model resolution."""
|
|
self._registry.get_default_model()
|
|
|
|
def time_resolve_provider_by_name(self) -> None:
|
|
"""Benchmark resolve_provider_by_name for configured provider."""
|
|
self._registry.resolve_provider_by_name("openai")
|
|
|
|
def time_get_all_providers(self) -> None:
|
|
"""Benchmark listing all known providers."""
|
|
self._registry.get_all_providers()
|
|
|
|
def time_is_provider_configured(self) -> None:
|
|
"""Benchmark checking if a provider is configured."""
|
|
self._registry.is_provider_configured("openai")
|
|
|
|
|
|
class ProviderAutoDetectSuite:
|
|
"""Benchmark provider auto-detection with multiple keys."""
|
|
|
|
def setup(self) -> None:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
os.environ["OPENAI_API_KEY"] = "bench-openai"
|
|
os.environ["ANTHROPIC_API_KEY"] = "bench-anthropic"
|
|
os.environ["GOOGLE_API_KEY"] = "bench-google"
|
|
|
|
def teardown(self) -> None:
|
|
for key in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY"):
|
|
os.environ.pop(key, None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
def time_auto_detect_multiple_providers(self) -> None:
|
|
"""Benchmark auto-detection with multiple configured providers."""
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
registry = ProviderRegistry()
|
|
registry.get_default_provider_type()
|
|
|
|
def time_get_configured_count(self) -> None:
|
|
"""Benchmark counting configured providers."""
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
registry = ProviderRegistry()
|
|
len(registry.get_configured_providers())
|