"""ASV benchmarks for ProviderRegistry operations beyond initialization. Measures the performance of: - get_all_providers (all known providers, configured or not) - get_provider_info (by ProviderType and by string name) - is_provider_configured (by ProviderType and by string name) - ProviderRegistry with multiple providers configured Note: Registry initialization, get_default_provider_type, get_configured_providers, and get_default_model are already covered by provider_selection_bench.py and are not duplicated here. """ from __future__ import annotations import importlib import sys from pathlib import Path from unittest.mock import MagicMock _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) import cleveragents # noqa: E402 importlib.reload(cleveragents) from cleveragents.providers.registry import ( # noqa: E402 ProviderRegistry, ProviderType, reset_provider_registry, ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_settings(**overrides: object) -> MagicMock: """Build a minimal mock Settings object for registry construction.""" settings = MagicMock() settings.openai_api_key = None settings.anthropic_api_key = None settings.google_api_key = None settings.gemini_api_key = None settings.azure_api_key = None settings.openrouter_api_key = None settings.cohere_api_key = None settings.groq_api_key = None settings.together_api_key = None settings.default_provider = None settings.default_model = None settings.azure_openai_endpoint = None settings.azure_openai_api_version = None settings.azure_openai_deployment = None settings.openrouter_organization = None settings.mock_providers = False for key, value in overrides.items(): setattr(settings, key, value) return settings class TimeRegistryGetAllProviders: """Benchmark ProviderRegistry.get_all_providers.""" def setup(self) -> None: reset_provider_registry() self.registry_none = ProviderRegistry(settings=_make_settings()) self.registry_one = ProviderRegistry( settings=_make_settings(openai_api_key="sk-bench-key") ) self.registry_multi = ProviderRegistry( settings=_make_settings( openai_api_key="sk-bench-key", anthropic_api_key="ant-bench-key", google_api_key="goog-bench-key", ) ) def time_get_all_providers_none_configured(self) -> None: """Time get_all_providers when no providers are configured.""" self.registry_none.get_all_providers() def time_get_all_providers_one_configured(self) -> None: """Time get_all_providers when one provider is configured.""" self.registry_one.get_all_providers() def time_get_all_providers_multi_configured(self) -> None: """Time get_all_providers when multiple providers are configured.""" self.registry_multi.get_all_providers() class TimeRegistryGetProviderInfo: """Benchmark ProviderRegistry.get_provider_info lookups.""" def setup(self) -> None: reset_provider_registry() self.registry = ProviderRegistry( settings=_make_settings(openai_api_key="sk-bench-key") ) def time_get_provider_info_by_enum(self) -> None: """Time get_provider_info using a ProviderType enum value.""" self.registry.get_provider_info(ProviderType.OPENAI) def time_get_provider_info_by_string(self) -> None: """Time get_provider_info using a string name (requires coercion).""" self.registry.get_provider_info("openai") def time_get_provider_info_unknown_string(self) -> None: """Time get_provider_info for an unknown provider name (returns None).""" self.registry.get_provider_info("nonexistent_provider") def time_get_provider_info_all_types(self) -> None: """Time get_provider_info for every known ProviderType.""" for provider_type in ProviderType: self.registry.get_provider_info(provider_type) class TimeRegistryIsProviderConfigured: """Benchmark ProviderRegistry.is_provider_configured checks.""" def setup(self) -> None: reset_provider_registry() self.registry = ProviderRegistry( settings=_make_settings( openai_api_key="sk-bench-key", anthropic_api_key="ant-bench-key", ) ) def time_is_configured_true_by_enum(self) -> None: """Time is_provider_configured for a configured provider (enum).""" self.registry.is_provider_configured(ProviderType.OPENAI) def time_is_configured_false_by_enum(self) -> None: """Time is_provider_configured for an unconfigured provider (enum).""" self.registry.is_provider_configured(ProviderType.GROQ) def time_is_configured_true_by_string(self) -> None: """Time is_provider_configured for a configured provider (string).""" self.registry.is_provider_configured("anthropic") def time_is_configured_false_by_string(self) -> None: """Time is_provider_configured for an unconfigured provider (string).""" self.registry.is_provider_configured("cohere") def time_is_configured_all_types(self) -> None: """Time is_provider_configured for every known ProviderType.""" for provider_type in ProviderType: self.registry.is_provider_configured(provider_type) class TimeRegistryMultiProviderInit: """Benchmark ProviderRegistry initialization with many providers configured.""" def setup(self) -> None: self.settings_all = _make_settings( openai_api_key="sk-bench-key", anthropic_api_key="ant-bench-key", google_api_key="goog-bench-key", groq_api_key="groq-bench-key", together_api_key="together-bench-key", cohere_api_key="cohere-bench-key", openrouter_api_key="or-bench-key", ) def time_init_all_providers_configured(self) -> None: """Time registry initialization when all major providers are configured.""" reset_provider_registry() ProviderRegistry(settings=self.settings_all) def time_get_configured_providers_all(self) -> None: """Time get_configured_providers when all major providers are configured.""" reset_provider_registry() registry = ProviderRegistry(settings=self.settings_all) registry.get_configured_providers()