"""ASV benchmarks for LLM provider adapter instantiation. Measures the performance of: - LangChainChatProvider instantiation (base adapter) - AnthropicChatProvider instantiation - GoogleChatProvider instantiation - OpenAIChatProvider instantiation - OpenRouterChatProvider instantiation These benchmarks focus on the construction cost of each adapter class, which is the performance-sensitive path exercised before every LLM call. No actual network calls are made — all LLM factories are mocked. """ from __future__ import annotations import importlib import sys from pathlib import Path from unittest.mock import MagicMock _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) import cleveragents # noqa: E402 importlib.reload(cleveragents) from cleveragents.providers.llm.anthropic_provider import ( # noqa: E402 AnthropicChatProvider, ) from cleveragents.providers.llm.google_provider import GoogleChatProvider # noqa: E402 from cleveragents.providers.llm.langchain_chat_provider import ( # noqa: E402 LangChainChatProvider, ) from cleveragents.providers.llm.openai_provider import OpenAIChatProvider # noqa: E402 from cleveragents.providers.llm.openrouter_provider import ( # noqa: E402 OpenRouterChatProvider, ) # --------------------------------------------------------------------------- # Shared mock LLM factory # --------------------------------------------------------------------------- def _mock_llm_factory(model_id: str) -> MagicMock: """Return a MagicMock that satisfies the BaseLanguageModel interface.""" return MagicMock() class TimeLangChainChatProviderInstantiation: """Benchmark LangChainChatProvider (base adapter) instantiation.""" def time_instantiate_default(self) -> None: """Time instantiation with default settings.""" LangChainChatProvider( name="bench_provider", model_id="bench-model", llm_factory=_mock_llm_factory, ) def time_instantiate_with_max_retries(self) -> None: """Time instantiation with a custom max_retries value.""" LangChainChatProvider( name="bench_provider", model_id="bench-model", llm_factory=_mock_llm_factory, max_retries=5, ) def time_instantiate_no_streaming(self) -> None: """Time instantiation with streaming disabled.""" LangChainChatProvider( name="bench_provider", model_id="bench-model", llm_factory=_mock_llm_factory, supports_streaming=False, ) def time_instantiate_with_progress_map(self) -> None: """Time instantiation with a custom progress map.""" LangChainChatProvider( name="bench_provider", model_id="bench-model", llm_factory=_mock_llm_factory, progress_map={ "load_context": 10, "analyze_requirements": 30, "generate_plan": 60, "validate": 85, }, ) class TimeAnthropicChatProviderInstantiation: """Benchmark AnthropicChatProvider instantiation.""" def time_instantiate_default_model(self) -> None: """Time instantiation with the default Claude model.""" AnthropicChatProvider(api_key="ant-bench-key") def time_instantiate_custom_model(self) -> None: """Time instantiation with a custom model identifier.""" AnthropicChatProvider( api_key="ant-bench-key", model="claude-3-5-haiku-20241022", ) def time_instantiate_custom_retries(self) -> None: """Time instantiation with a non-default max_retries value.""" AnthropicChatProvider( api_key="ant-bench-key", max_retries=1, ) class TimeGoogleChatProviderInstantiation: """Benchmark GoogleChatProvider instantiation.""" def time_instantiate_default_model(self) -> None: """Time instantiation with the default Gemini model.""" GoogleChatProvider(api_key="goog-bench-key") def time_instantiate_custom_model(self) -> None: """Time instantiation with a custom Gemini model identifier.""" GoogleChatProvider( api_key="goog-bench-key", model="gemini-1.5-pro", ) def time_instantiate_custom_retries(self) -> None: """Time instantiation with a non-default max_retries value.""" GoogleChatProvider( api_key="goog-bench-key", max_retries=2, ) class TimeOpenAIChatProviderInstantiation: """Benchmark OpenAIChatProvider instantiation.""" def time_instantiate_default_model(self) -> None: """Time instantiation with the default GPT-4o model.""" OpenAIChatProvider(api_key="sk-bench-key") def time_instantiate_custom_model(self) -> None: """Time instantiation with a custom model identifier.""" OpenAIChatProvider( api_key="sk-bench-key", model="gpt-4o-mini", ) def time_instantiate_with_organization(self) -> None: """Time instantiation with an organization identifier.""" OpenAIChatProvider( api_key="sk-bench-key", organization="org-bench-12345", ) def time_instantiate_custom_retries(self) -> None: """Time instantiation with a non-default max_retries value.""" OpenAIChatProvider( api_key="sk-bench-key", max_retries=1, ) class TimeOpenRouterChatProviderInstantiation: """Benchmark OpenRouterChatProvider instantiation.""" def time_instantiate_default_model(self) -> None: """Time instantiation with the default OpenRouter model.""" OpenRouterChatProvider(api_key="or-bench-key") def time_instantiate_custom_model(self) -> None: """Time instantiation with a custom model identifier.""" OpenRouterChatProvider( api_key="or-bench-key", model="openai/gpt-4o", ) def time_instantiate_with_organization(self) -> None: """Time instantiation with an organization/referer header.""" OpenRouterChatProvider( api_key="or-bench-key", organization="https://bench.example.com", ) def time_instantiate_with_custom_headers(self) -> None: """Time instantiation with custom default headers.""" OpenRouterChatProvider( api_key="or-bench-key", default_headers={ "HTTP-Referer": "https://bench.example.com", "X-Title": "BenchmarkSuite", }, ) def time_instantiate_custom_retries(self) -> None: """Time instantiation with a non-default max_retries value.""" OpenRouterChatProvider( api_key="or-bench-key", max_retries=5, )