cleveragents-core/benchmarks/providers_registry_bench.py

"""ASV benchmarks for ProviderRegistry operations beyond initialization.

Measures the performance of:
- get_all_providers (all known providers, configured or not)
- get_provider_info (by ProviderType and by string name)
- is_provider_configured (by ProviderType and by string name)
- ProviderRegistry with multiple providers configured

Note: Registry initialization, get_default_provider_type,
get_configured_providers, and get_default_model are already covered by
provider_selection_bench.py and are not duplicated here.
"""

from __future__ import annotations

import importlib
import sys
from pathlib import Path
from unittest.mock import MagicMock

_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

import cleveragents  # noqa: E402

importlib.reload(cleveragents)

from cleveragents.providers.registry import (  # noqa: E402
    ProviderRegistry,
    ProviderType,
    reset_provider_registry,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_settings(**overrides: object) -> MagicMock:
    """Build a minimal mock Settings object for registry construction."""
    settings = MagicMock()
    settings.openai_api_key = None
    settings.anthropic_api_key = None
    settings.google_api_key = None
    settings.gemini_api_key = None
    settings.azure_api_key = None
    settings.openrouter_api_key = None
    settings.cohere_api_key = None
    settings.groq_api_key = None
    settings.together_api_key = None
    settings.default_provider = None
    settings.default_model = None
    settings.azure_openai_endpoint = None
    settings.azure_openai_api_version = None
    settings.azure_openai_deployment = None
    settings.openrouter_organization = None
    settings.mock_providers = False
    for key, value in overrides.items():
        setattr(settings, key, value)
    return settings


class TimeRegistryGetAllProviders:
    """Benchmark ProviderRegistry.get_all_providers."""

    def setup(self) -> None:
        reset_provider_registry()
        self.registry_none = ProviderRegistry(settings=_make_settings())
        self.registry_one = ProviderRegistry(
            settings=_make_settings(openai_api_key="sk-bench-key")
        )
        self.registry_multi = ProviderRegistry(
            settings=_make_settings(
                openai_api_key="sk-bench-key",
                anthropic_api_key="ant-bench-key",
                google_api_key="goog-bench-key",
            )
        )

    def time_get_all_providers_none_configured(self) -> None:
        """Time get_all_providers when no providers are configured."""
        self.registry_none.get_all_providers()

    def time_get_all_providers_one_configured(self) -> None:
        """Time get_all_providers when one provider is configured."""
        self.registry_one.get_all_providers()

    def time_get_all_providers_multi_configured(self) -> None:
        """Time get_all_providers when multiple providers are configured."""
        self.registry_multi.get_all_providers()


class TimeRegistryGetProviderInfo:
    """Benchmark ProviderRegistry.get_provider_info lookups."""

    def setup(self) -> None:
        reset_provider_registry()
        self.registry = ProviderRegistry(
            settings=_make_settings(openai_api_key="sk-bench-key")
        )

    def time_get_provider_info_by_enum(self) -> None:
        """Time get_provider_info using a ProviderType enum value."""
        self.registry.get_provider_info(ProviderType.OPENAI)

    def time_get_provider_info_by_string(self) -> None:
        """Time get_provider_info using a string name (requires coercion)."""
        self.registry.get_provider_info("openai")

    def time_get_provider_info_unknown_string(self) -> None:
        """Time get_provider_info for an unknown provider name (returns None)."""
        self.registry.get_provider_info("nonexistent_provider")

    def time_get_provider_info_all_types(self) -> None:
        """Time get_provider_info for every known ProviderType."""
        for provider_type in ProviderType:
            self.registry.get_provider_info(provider_type)


class TimeRegistryIsProviderConfigured:
    """Benchmark ProviderRegistry.is_provider_configured checks."""

    def setup(self) -> None:
        reset_provider_registry()
        self.registry = ProviderRegistry(
            settings=_make_settings(
                openai_api_key="sk-bench-key",
                anthropic_api_key="ant-bench-key",
            )
        )

    def time_is_configured_true_by_enum(self) -> None:
        """Time is_provider_configured for a configured provider (enum)."""
        self.registry.is_provider_configured(ProviderType.OPENAI)

    def time_is_configured_false_by_enum(self) -> None:
        """Time is_provider_configured for an unconfigured provider (enum)."""
        self.registry.is_provider_configured(ProviderType.GROQ)

    def time_is_configured_true_by_string(self) -> None:
        """Time is_provider_configured for a configured provider (string)."""
        self.registry.is_provider_configured("anthropic")

    def time_is_configured_false_by_string(self) -> None:
        """Time is_provider_configured for an unconfigured provider (string)."""
        self.registry.is_provider_configured("cohere")

    def time_is_configured_all_types(self) -> None:
        """Time is_provider_configured for every known ProviderType."""
        for provider_type in ProviderType:
            self.registry.is_provider_configured(provider_type)


class TimeRegistryMultiProviderInit:
    """Benchmark ProviderRegistry initialization with many providers configured."""

    def setup(self) -> None:
        self.settings_all = _make_settings(
            openai_api_key="sk-bench-key",
            anthropic_api_key="ant-bench-key",
            google_api_key="goog-bench-key",
            groq_api_key="groq-bench-key",
            together_api_key="together-bench-key",
            cohere_api_key="cohere-bench-key",
            openrouter_api_key="or-bench-key",
        )

    def time_init_all_providers_configured(self) -> None:
        """Time registry initialization when all major providers are configured."""
        reset_provider_registry()
        ProviderRegistry(settings=self.settings_all)

    def time_get_configured_providers_all(self) -> None:
        """Time get_configured_providers when all major providers are configured."""
        reset_provider_registry()
        registry = ProviderRegistry(settings=self.settings_all)
        registry.get_configured_providers()