cleveragents-core/benchmarks/providers_llm_adapters_bench.py

"""ASV benchmarks for LLM provider adapter instantiation.

Measures the performance of:
- LangChainChatProvider instantiation (base adapter)
- AnthropicChatProvider instantiation
- GoogleChatProvider instantiation
- OpenAIChatProvider instantiation
- OpenRouterChatProvider instantiation

These benchmarks focus on the construction cost of each adapter class,
which is the performance-sensitive path exercised before every LLM call.
No actual network calls are made — all LLM factories are mocked.
"""

from __future__ import annotations

import importlib
import sys
from pathlib import Path
from unittest.mock import MagicMock

_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

import cleveragents  # noqa: E402

importlib.reload(cleveragents)

from cleveragents.providers.llm.anthropic_provider import (  # noqa: E402
    AnthropicChatProvider,
)
from cleveragents.providers.llm.google_provider import GoogleChatProvider  # noqa: E402
from cleveragents.providers.llm.langchain_chat_provider import (  # noqa: E402
    LangChainChatProvider,
)
from cleveragents.providers.llm.openai_provider import OpenAIChatProvider  # noqa: E402
from cleveragents.providers.llm.openrouter_provider import (  # noqa: E402
    OpenRouterChatProvider,
)

# ---------------------------------------------------------------------------
# Shared mock LLM factory
# ---------------------------------------------------------------------------


def _mock_llm_factory(model_id: str) -> MagicMock:
    """Return a MagicMock that satisfies the BaseLanguageModel interface."""
    return MagicMock()


class TimeLangChainChatProviderInstantiation:
    """Benchmark LangChainChatProvider (base adapter) instantiation."""

    def time_instantiate_default(self) -> None:
        """Time instantiation with default settings."""
        LangChainChatProvider(
            name="bench_provider",
            model_id="bench-model",
            llm_factory=_mock_llm_factory,
        )

    def time_instantiate_with_max_retries(self) -> None:
        """Time instantiation with a custom max_retries value."""
        LangChainChatProvider(
            name="bench_provider",
            model_id="bench-model",
            llm_factory=_mock_llm_factory,
            max_retries=5,
        )

    def time_instantiate_no_streaming(self) -> None:
        """Time instantiation with streaming disabled."""
        LangChainChatProvider(
            name="bench_provider",
            model_id="bench-model",
            llm_factory=_mock_llm_factory,
            supports_streaming=False,
        )

    def time_instantiate_with_progress_map(self) -> None:
        """Time instantiation with a custom progress map."""
        LangChainChatProvider(
            name="bench_provider",
            model_id="bench-model",
            llm_factory=_mock_llm_factory,
            progress_map={
                "load_context": 10,
                "analyze_requirements": 30,
                "generate_plan": 60,
                "validate": 85,
            },
        )


class TimeAnthropicChatProviderInstantiation:
    """Benchmark AnthropicChatProvider instantiation."""

    def time_instantiate_default_model(self) -> None:
        """Time instantiation with the default Claude model."""
        AnthropicChatProvider(api_key="ant-bench-key")

    def time_instantiate_custom_model(self) -> None:
        """Time instantiation with a custom model identifier."""
        AnthropicChatProvider(
            api_key="ant-bench-key",
            model="claude-3-5-haiku-20241022",
        )

    def time_instantiate_custom_retries(self) -> None:
        """Time instantiation with a non-default max_retries value."""
        AnthropicChatProvider(
            api_key="ant-bench-key",
            max_retries=1,
        )


class TimeGoogleChatProviderInstantiation:
    """Benchmark GoogleChatProvider instantiation."""

    def time_instantiate_default_model(self) -> None:
        """Time instantiation with the default Gemini model."""
        GoogleChatProvider(api_key="goog-bench-key")

    def time_instantiate_custom_model(self) -> None:
        """Time instantiation with a custom Gemini model identifier."""
        GoogleChatProvider(
            api_key="goog-bench-key",
            model="gemini-1.5-pro",
        )

    def time_instantiate_custom_retries(self) -> None:
        """Time instantiation with a non-default max_retries value."""
        GoogleChatProvider(
            api_key="goog-bench-key",
            max_retries=2,
        )


class TimeOpenAIChatProviderInstantiation:
    """Benchmark OpenAIChatProvider instantiation."""

    def time_instantiate_default_model(self) -> None:
        """Time instantiation with the default GPT-4o model."""
        OpenAIChatProvider(api_key="sk-bench-key")

    def time_instantiate_custom_model(self) -> None:
        """Time instantiation with a custom model identifier."""
        OpenAIChatProvider(
            api_key="sk-bench-key",
            model="gpt-4o-mini",
        )

    def time_instantiate_with_organization(self) -> None:
        """Time instantiation with an organization identifier."""
        OpenAIChatProvider(
            api_key="sk-bench-key",
            organization="org-bench-12345",
        )

    def time_instantiate_custom_retries(self) -> None:
        """Time instantiation with a non-default max_retries value."""
        OpenAIChatProvider(
            api_key="sk-bench-key",
            max_retries=1,
        )


class TimeOpenRouterChatProviderInstantiation:
    """Benchmark OpenRouterChatProvider instantiation."""

    def time_instantiate_default_model(self) -> None:
        """Time instantiation with the default OpenRouter model."""
        OpenRouterChatProvider(api_key="or-bench-key")

    def time_instantiate_custom_model(self) -> None:
        """Time instantiation with a custom model identifier."""
        OpenRouterChatProvider(
            api_key="or-bench-key",
            model="openai/gpt-4o",
        )

    def time_instantiate_with_organization(self) -> None:
        """Time instantiation with an organization/referer header."""
        OpenRouterChatProvider(
            api_key="or-bench-key",
            organization="https://bench.example.com",
        )

    def time_instantiate_with_custom_headers(self) -> None:
        """Time instantiation with custom default headers."""
        OpenRouterChatProvider(
            api_key="or-bench-key",
            default_headers={
                "HTTP-Referer": "https://bench.example.com",
                "X-Title": "BenchmarkSuite",
            },
        )

    def time_instantiate_custom_retries(self) -> None:
        """Time instantiation with a non-default max_retries value."""
        OpenRouterChatProvider(
            api_key="or-bench-key",
            max_retries=5,
        )