forked from cleveragents/cleveragents-core
freemo
a074b4846f
Remove FakeListLLM as a silent fallback in agent graph constructors (plan_generation.py, context_analysis.py, auto_debug.py). All three now raise ValueError when llm=None, making missing-provider errors explicit. Add Settings.mock_providers flag and validate_provider_availability() method. Update container.get_ai_provider() to check Settings.mock_providers first, with env-var fallback for backward compatibility. Add resolve_provider_by_name() helper to the provider registry and export it from cleveragents.providers. Add structlog trace logging to ProviderRegistry.get_default_provider_type() to record selection reasoning. Update all existing behave step files, robot tests, and benchmarks that relied on the implicit FakeListLLM default to pass an explicit LLM instance instead. Add new BDD tests (features/provider_fixes.feature with 17 scenarios), Robot Framework integration tests (robot/provider_detection_smoke.robot), and ASV benchmarks (benchmarks/provider_selection_bench.py). ISSUES CLOSED: #323
88 lines
2.7 KiB
Python
88 lines
2.7 KiB
Python
"""ASV benchmarks for provider selection and registry initialization.
|
|
|
|
Measures the performance of:
|
|
- Provider registry initialization (discovering configured providers)
|
|
- Provider resolution by name
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.providers.registry import ( # noqa: E402
|
|
ProviderRegistry,
|
|
ProviderType,
|
|
reset_provider_registry,
|
|
)
|
|
|
|
|
|
def _make_settings(openai: str | None = None) -> MagicMock:
|
|
settings = MagicMock()
|
|
settings.openai_api_key = openai
|
|
settings.anthropic_api_key = None
|
|
settings.google_api_key = None
|
|
settings.gemini_api_key = None
|
|
settings.azure_api_key = None
|
|
settings.openrouter_api_key = None
|
|
settings.cohere_api_key = None
|
|
settings.groq_api_key = None
|
|
settings.together_api_key = None
|
|
settings.default_provider = None
|
|
settings.default_model = None
|
|
settings.azure_openai_endpoint = None
|
|
settings.azure_openai_api_version = None
|
|
settings.azure_openai_deployment = None
|
|
settings.openrouter_organization = None
|
|
settings.mock_providers = False
|
|
return settings
|
|
|
|
|
|
class TimeProviderRegistryInit:
|
|
"""Benchmark provider registry initialization."""
|
|
|
|
def setup(self) -> None:
|
|
self.settings_none = _make_settings()
|
|
self.settings_openai = _make_settings(openai="sk-bench-key")
|
|
|
|
def time_provider_registry_init(self) -> None:
|
|
"""Time registry creation with no providers configured."""
|
|
reset_provider_registry()
|
|
ProviderRegistry(settings=self.settings_none)
|
|
|
|
def time_provider_registry_init_with_provider(self) -> None:
|
|
"""Time registry creation with one provider configured."""
|
|
reset_provider_registry()
|
|
ProviderRegistry(settings=self.settings_openai)
|
|
|
|
|
|
class TimeProviderResolution:
|
|
"""Benchmark provider resolution."""
|
|
|
|
def setup(self) -> None:
|
|
self.settings = _make_settings(openai="sk-bench-key")
|
|
reset_provider_registry()
|
|
self.registry = ProviderRegistry(settings=self.settings)
|
|
|
|
def time_provider_resolution(self) -> None:
|
|
"""Time default provider type resolution."""
|
|
self.registry.get_default_provider_type()
|
|
|
|
def time_get_configured_providers(self) -> None:
|
|
"""Time listing configured providers."""
|
|
self.registry.get_configured_providers()
|
|
|
|
def time_get_default_model(self) -> None:
|
|
"""Time default model resolution."""
|
|
self.registry.get_default_model(ProviderType.OPENAI)
|