HAL9000
5c5cff8ee2
Add comprehensive Robot Framework integration test suite for the provider module with 43 test cases covering: ProviderRegistry discovery, configured provider filtering, provider info lookups by enum/string/uppercase, default provider/model resolution with env var overrides, DEFAULT_CAPABILITIES and DEFAULT_MODELS verification for all 10 provider types, FALLBACK_ORDER order, global singleton/reset behavior, AI provider creation (including MOCK sandboxing), LLM factory instantiation, named provider resolution with error handling, ProviderCapabilities immutability, and Azure provider config fields. Also update CHANGELOG.md and CONTRIBUTORS.md. ISSUES CLOSED: #10952
827 lines
31 KiB
Python
827 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
"""Helper script for provider integration Robot Framework tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_src = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _src not in sys.path:
|
|
sys.path.insert(0, _src)
|
|
|
|
|
|
def test_discover_all_providers() -> None:
|
|
"""Verify ProviderRegistry discovers all 10 ProviderType entries."""
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
registry = ProviderRegistry()
|
|
providers = registry.get_all_providers()
|
|
assert len(providers) == 10, f"Expected 10 providers, got {len(providers)}"
|
|
|
|
|
|
def test_get_all_providers_count() -> None:
|
|
"""Verify get_all_providers() returns at least as many configs."""
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
registry = ProviderRegistry()
|
|
all_p = registry.get_all_providers()
|
|
configured_p = registry.get_configured_providers()
|
|
assert len(all_p) >= len(configured_p), "All should be >= configured count"
|
|
|
|
|
|
def test_get_configured_providers_filtering() -> None:
|
|
"""get_configured_providers correctly filters among API-key scenarios."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
saved = os.environ.pop("OPENAI_API_KEY", None)
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
configured = registry.get_configured_providers()
|
|
provider_types = [p.provider_type.value for p in configured]
|
|
assert "openai" not in provider_types, (
|
|
f"openai should be unconfigured, got: {provider_types}"
|
|
)
|
|
|
|
os.environ["OPENAI_API_KEY"] = "fake-key-for-robot"
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry2 = ProviderRegistry()
|
|
configured2 = registry2.get_configured_providers()
|
|
provider_types2 = [p.provider_type.value for p in configured2]
|
|
assert "openai" in provider_types2, f"openai should be configured: {provider_types2}"
|
|
finally:
|
|
if saved is not None:
|
|
os.environ["OPENAI_API_KEY"] = saved
|
|
else:
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_get_provider_info_by_enum() -> None:
|
|
"""get_provider_info works when passed a ProviderType enum."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
registry = ProviderRegistry()
|
|
info = registry.get_provider_info(ProviderType.OPENAI)
|
|
assert info is not None
|
|
assert info.provider_type == ProviderType.OPENAI
|
|
|
|
|
|
def test_get_provider_info_by_lowercase_string() -> None:
|
|
"""get_provider_info works with lowercase string lookup."""
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
registry = ProviderRegistry()
|
|
info = registry.get_provider_info("openai")
|
|
assert info is not None
|
|
assert info.provider_type.value == "openai"
|
|
|
|
|
|
def test_get_provider_info_by_uppercase_string() -> None:
|
|
"""get_provider_info handles uppercase strings case-insensitively."""
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
registry = ProviderRegistry()
|
|
info = registry.get_provider_info("ANTHROPIC")
|
|
assert info is not None
|
|
assert info.provider_type.value == "anthropic"
|
|
|
|
|
|
def test_get_provider_info_unknown_returns_none() -> None:
|
|
"""get_provider_info returns None for unknown provider names."""
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
registry = ProviderRegistry()
|
|
assert registry.get_provider_info("nonexistent") is None
|
|
assert registry.get_provider_info("FAKEPROVIDER") is None
|
|
|
|
|
|
def test_is_provider_configured_true() -> None:
|
|
"""is_provider_configured returns True for a configured provider."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
os.environ["AZURE_API_KEY"] = "test-key-for-robot"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
assert registry.is_provider_configured("azure") is True
|
|
assert registry.is_provider_configured("openai") is False
|
|
finally:
|
|
os.environ.pop("AZURE_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_is_provider_configured_false() -> None:
|
|
"""is_provider_configured returns False for an unconfigured provider."""
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
registry = ProviderRegistry()
|
|
assert registry.is_provider_configured("openai") is False
|
|
|
|
|
|
def test_get_default_provider_type_env_override() -> None:
|
|
"""get_default_provider_type respects CLEVERAGENTS_DEFAULT_PROVIDER env var."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, ProviderType, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["ANTHROPIC_API_KEY"] = "robot-anthropic-key"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None)
|
|
default = registry.get_default_provider_type()
|
|
assert default == ProviderType.ANTHROPIC, f"Expected anthropic, got {default}"
|
|
|
|
os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = "anthropic"
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry2 = ProviderRegistry()
|
|
default2 = registry2.get_default_provider_type()
|
|
assert default2 == ProviderType.ANTHROPIC, f"Expected anthropic override, got {default2}"
|
|
finally:
|
|
os.environ.pop("ANTHROPIC_API_KEY", None)
|
|
os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_get_default_provider_type_fallback_order() -> None:
|
|
"""get_default_provider_type uses fallback order when env var is absent."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, ProviderType, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["GROQ_API_KEY"] = "robot-groq-key"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
default = registry.get_default_provider_type()
|
|
assert default == ProviderType.GROQ, f"Expected groq fallback, got {default}"
|
|
finally:
|
|
os.environ.pop("GROQ_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_get_default_provider_type_invalid_env_ignored() -> None:
|
|
"""get_default_provider_type ignores invalid env var and falls back."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
os.environ["TOGETHER_API_KEY"] = "robot-together-key"
|
|
try:
|
|
os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = "nonexistent-provider"
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
default = registry.get_default_provider_type()
|
|
assert default.value == "together", f"Expected fallback to together, got {default}"
|
|
finally:
|
|
os.environ.pop("TOGETHER_API_KEY", None)
|
|
os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_get_default_provider_type_unconfigured_env_skipped() -> None:
|
|
"""get_default_provider_type skips env var when provider is unconfigured."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
os.environ["TOGETHER_API_KEY"] = "robot-together-key"
|
|
try:
|
|
os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = "openai"
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
default = registry.get_default_provider_type()
|
|
assert default.value == "together", (
|
|
f"Should skip unconfigured openai, got {default}"
|
|
)
|
|
finally:
|
|
os.environ.pop("TOGETHER_API_KEY", None)
|
|
os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_get_default_provider_type_none_nothing_configured() -> None:
|
|
"""get_default_provider_type returns None when no provider is configured."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
api_key_vars = [
|
|
"OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY",
|
|
"AZURE_OPENAI_API_KEY", "AZURE_API_KEY", "OPENROUTER_API_KEY",
|
|
"GROQ_API_KEY", "TOGETHER_API_KEY", "COHERE_API_KEY", "PERPLEXITY_API_KEY",
|
|
]
|
|
saved: dict[str, str] = {}
|
|
for var in api_key_vars:
|
|
val = os.environ.get(var)
|
|
if val is not None:
|
|
saved[var] = val
|
|
for var in api_key_vars:
|
|
os.environ.pop(var, None)
|
|
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
assert registry.get_default_provider_type() is None
|
|
finally:
|
|
for var in api_key_vars:
|
|
if var in saved:
|
|
os.environ[var] = saved[var]
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_get_default_model_env_override() -> None:
|
|
"""get_default_model returns CLEVERAGENTS_DEFAULT_MODEL env var when set."""
|
|
import os
|
|
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
os.environ["CLEVERAGENTS_DEFAULT_MODEL"] = "overridden-model-v2"
|
|
try:
|
|
registry = ProviderRegistry()
|
|
model = registry.get_default_model()
|
|
assert model == "overridden-model-v2", f"Expected overridden, got {model}"
|
|
finally:
|
|
os.environ.pop("CLEVERAGENTS_DEFAULT_MODEL", None)
|
|
|
|
|
|
def test_get_default_model_specific_provider() -> None:
|
|
"""get_default_model returns provider-specific default when no env var."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
registry = ProviderRegistry()
|
|
model = registry.get_default_model(ProviderType.ANTHROPIC)
|
|
assert model == "claude-sonnet-4-20250514", f"Expected anthropic default, got {model}"
|
|
|
|
|
|
def test_get_default_model_invalid_returns_none() -> None:
|
|
"""get_default_model returns None for an invalid provider type."""
|
|
from cleveragents.providers.registry import ProviderRegistry
|
|
|
|
registry = ProviderRegistry()
|
|
assert registry.get_default_model("totally-invalid-provider") is None
|
|
|
|
|
|
def test_default_capabilities_openai() -> None:
|
|
"""DEFAULT_CAPABILITIES are correct for OPENAI."""
|
|
from cleveragents.providers.registry import (
|
|
ProviderCapabilities, ProviderRegistry, ProviderType,
|
|
)
|
|
|
|
caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.OPENAI]
|
|
assert caps.supports_streaming is True
|
|
assert caps.supports_tool_calls is True
|
|
assert caps.supports_vision is True
|
|
assert caps.max_context_length == 128000
|
|
assert caps.supports_json_mode is True
|
|
|
|
|
|
def test_default_capabilities_anthropic() -> None:
|
|
"""DEFAULT_CAPABILITIES ANTHROPIC has no json mode."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.ANTHROPIC]
|
|
assert caps.supports_json_mode is False
|
|
assert caps.max_context_length == 200000
|
|
|
|
|
|
def test_default_capabilities_google() -> None:
|
|
"""DEFAULT_CAPABILITIES GOOGLE has 1M context."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.GOOGLE]
|
|
assert caps.max_context_length == 1000000
|
|
|
|
|
|
def test_default_capabilities_cohere() -> None:
|
|
"""DEFAULT_CAPABILITIES COHERE lacks vision."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.COHERE]
|
|
assert caps.supports_vision is False
|
|
|
|
|
|
def test_default_capabilities_mock() -> None:
|
|
"""DEFAULT_CAPABILITIES MOCK is very limited."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.MOCK]
|
|
assert caps.supports_streaming is False
|
|
assert caps.supports_tool_calls is False
|
|
assert caps.max_context_length == 4096
|
|
|
|
|
|
def test_default_models_all_providers() -> None:
|
|
"""DEFAULT_MODELS have entries for every provider type."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
for pt in ProviderType:
|
|
model = ProviderRegistry.DEFAULT_MODELS.get(pt)
|
|
assert model is not None and model != "unknown", (
|
|
f"Missing DEFAULT_MODEL for {pt.value}"
|
|
)
|
|
|
|
|
|
def test_fallback_order_sequence() -> None:
|
|
"""FALLBACK_ORDER has the correct provider priority sequence."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
order = ProviderRegistry.FALLBACK_ORDER
|
|
assert len(order) == 8
|
|
assert order[0] == ProviderType.OPENAI
|
|
assert order[1] == ProviderType.ANTHROPIC
|
|
assert order[2] == ProviderType.GOOGLE
|
|
|
|
|
|
def test_global_singleton_get_provider_registry() -> None:
|
|
"""get_provider_registry returns a singleton instance."""
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
get_provider_registry, reset_provider_registry,
|
|
)
|
|
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry1 = get_provider_registry()
|
|
registry2 = get_provider_registry()
|
|
assert registry1 is registry2, "Expected singleton behavior"
|
|
|
|
|
|
def test_global_reset_clears_singleton() -> None:
|
|
"""reset_provider_registry clears the global registry instance."""
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
get_provider_registry, reset_provider_registry,
|
|
)
|
|
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
r1 = get_provider_registry()
|
|
reset_provider_registry()
|
|
r2 = get_provider_registry()
|
|
assert r1 is not r2, "reset should create a new instance"
|
|
|
|
|
|
def test_create_ai_provider_configured_openai() -> None:
|
|
"""create_ai_provider succeeds with a configured provider."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
ai = registry.create_ai_provider()
|
|
assert ai is not None
|
|
assert ai.name == "openai"
|
|
finally:
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_create_ai_provider_explicit_model_override() -> None:
|
|
"""create_ai_provider accepts an explicit model_id override."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
ai = registry.create_ai_provider(model_id="gpt-4o-mini")
|
|
assert ai is not None
|
|
assert ai.model_id == "gpt-4o-mini"
|
|
finally:
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_create_ai_provider_mock_rejected_without_env() -> None:
|
|
"""create_ai_provider rejects MOCK provider without env var."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
registry = ProviderRegistry()
|
|
try:
|
|
registry.create_ai_provider(provider_type=ProviderType.MOCK)
|
|
assert False, "Should have raised ValueError"
|
|
except ValueError as e:
|
|
assert "Mock provider is not allowed" in str(e)
|
|
|
|
|
|
def test_create_ai_provider_mock_succeeds_with_env() -> None:
|
|
"""create_ai_provider accepts MOCK when env var is set."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, ProviderType, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["CLEVERAGENTS_ALLOW_MOCK_PROVIDER"] = "true"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
ai = registry.create_ai_provider(provider_type=ProviderType.MOCK)
|
|
assert ai is not None
|
|
finally:
|
|
os.environ.pop("CLEVERAGENTS_ALLOW_MOCK_PROVIDER", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_create_llm_no_provider_errors() -> None:
|
|
"""create_llm raises ValueError when no provider is configured."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
api_key_vars = [
|
|
"OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY",
|
|
"AZURE_OPENAI_API_KEY", "AZURE_API_KEY", "OPENROUTER_API_KEY",
|
|
"GROQ_API_KEY", "TOGETHER_API_KEY", "COHERE_API_KEY",
|
|
]
|
|
saved: dict[str, str] = {}
|
|
for var in api_key_vars:
|
|
val = os.environ.get(var)
|
|
if val is not None:
|
|
saved[var] = val
|
|
for var in api_key_vars:
|
|
os.environ.pop(var, None)
|
|
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
try:
|
|
registry.create_llm()
|
|
assert False, "Should have raised ValueError"
|
|
except ValueError as e:
|
|
assert "No AI provider configured" in str(e)
|
|
finally:
|
|
for var in api_key_vars:
|
|
if var in saved:
|
|
os.environ[var] = saved[var]
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_create_llm_configured_succeeds() -> None:
|
|
"""create_llm succeeds with a configured provider."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
os.environ["GROQ_API_KEY"] = "robot-groq-key-fake"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
llm = registry.create_llm()
|
|
assert llm is not None
|
|
finally:
|
|
os.environ.pop("GROQ_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_create_llm_mock_rejected() -> None:
|
|
"""create_llm rejects MOCK provider without env var."""
|
|
from cleveragents.providers.registry import ProviderRegistry, ProviderType
|
|
|
|
registry = ProviderRegistry()
|
|
try:
|
|
registry.create_llm(provider_type=ProviderType.MOCK)
|
|
assert False, "Should have raised ValueError"
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
def test_create_llm_mock_succeeds_with_env() -> None:
|
|
"""create_llm accepts MOCK when env var is set."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, ProviderType, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["CLEVERAGENTS_ALLOW_MOCK_PROVIDER"] = "true"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
llm = registry.create_llm(provider_type=ProviderType.MOCK)
|
|
assert llm is not None
|
|
finally:
|
|
os.environ.pop("CLEVERAGENTS_ALLOW_MOCK_PROVIDER", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_create_llm_explicit_model_id() -> None:
|
|
"""create_llm uses the explicit model_id when provided."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry
|
|
|
|
os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
llm = registry.create_llm(model_id="gpt-4o-mini")
|
|
assert llm is not None
|
|
finally:
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_resolve_provider_by_name_success() -> None:
|
|
"""resolve_provider_by_name works with a configured provider."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, resolve_provider_by_name, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
provider = resolve_provider_by_name("openai")
|
|
assert provider is not None
|
|
finally:
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_resolve_provider_by_name_case_insensitive() -> None:
|
|
"""resolve_provider_by_name handles case-insensitive names."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, resolve_provider_by_name, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["GROQ_API_KEY"] = "robot-groq-key-fake"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
provider = resolve_provider_by_name("Groq")
|
|
assert provider is not None
|
|
finally:
|
|
os.environ.pop("GROQ_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_resolve_provider_by_name_unknown_raises() -> None:
|
|
"""resolve_provider_by_name raises ValueError for unknown provider names."""
|
|
from cleveragents.providers.registry import resolve_provider_by_name
|
|
|
|
try:
|
|
resolve_provider_by_name("totally-unregistered-fake-provider")
|
|
assert False, "Should have raised ValueError"
|
|
except ValueError as e:
|
|
string = str(e).lower()
|
|
assert "unknown provider" in string or "not configured" in string
|
|
|
|
|
|
def test_resolve_provider_by_name_unconfigured_raises() -> None:
|
|
"""resolve_provider_by_name raises ValueError when provider has no API key."""
|
|
from cleveragents.providers.registry import resolve_provider_by_name
|
|
|
|
try:
|
|
resolve_provider_by_name("openai") # Not configured here
|
|
assert False, "Should have raised ValueError"
|
|
except ValueError as e:
|
|
string = str(e).lower()
|
|
assert "not configured" in string
|
|
|
|
|
|
def test_create_provider_instance_groq_type() -> None:
|
|
"""_create_provider_instance builds the correct LLM type for Groq."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, reset_provider_registry, ProviderType,
|
|
)
|
|
|
|
api_key_set = "GROQ_API_KEY"
|
|
os.environ[api_key_set] = "robot-groq-key-12345"
|
|
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
from langchain_groq import ChatGroq
|
|
llm = registry._create_provider_instance(
|
|
ProviderType.GROQ, "llama-3.1-70b-versatile",
|
|
)
|
|
assert isinstance(llm, ChatGroq), f"Expected ChatGroq, got {type(llm)}"
|
|
finally:
|
|
os.environ.pop(api_key_set, None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_provider_capabilities_immutability() -> None:
|
|
"""ProviderCapabilities is a frozen dataclass and cannot be mutated."""
|
|
from cleveragents.providers.registry import ProviderCapabilities
|
|
|
|
caps = ProviderCapabilities()
|
|
try:
|
|
caps.supports_streaming = False # type: ignore[assignment]
|
|
assert False, "Should not be able to mutate frozen dataclass"
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def test_azure_provider_config_fields() -> None:
|
|
"""Azure provider _create_provider_instance deploys with correct fields."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, reset_provider_registry,
|
|
)
|
|
|
|
azure_key = "AZURE_API_KEY"
|
|
azure_endpoint = "https://robot-test-svc.openai.azure.com"
|
|
os.environ[azure_key] = "robot-azure-key-12345"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
from langchain_openai import AzureChatOpenAI
|
|
llm = registry._create_provider_instance(
|
|
"azure", "test-deployment",
|
|
azure_endpoint=azure_endpoint,
|
|
)
|
|
assert isinstance(llm, AzureChatOpenAI), (
|
|
f"Expected AzureChatOpenAI, got {type(llm)}"
|
|
)
|
|
finally:
|
|
os.environ.pop(azure_key, None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
def test_azure_missing_endpoint_raises() -> None:
|
|
"""Azure _create_provider_instance raises when endpoint is missing."""
|
|
import os
|
|
|
|
from cleveragents.config.settings import Settings, get_settings
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry, reset_provider_registry,
|
|
)
|
|
|
|
os.environ["AZURE_API_KEY"] = "robot-azure-key-12345"
|
|
try:
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
registry = ProviderRegistry()
|
|
try:
|
|
registry._create_provider_instance("azure", "my-deploy")
|
|
assert False, "Should have raised ValueError"
|
|
except ValueError as e:
|
|
assert "Azure OpenAI endpoint not configured" in str(e)
|
|
finally:
|
|
os.environ.pop("AZURE_API_KEY", None)
|
|
Settings._instance = None # type: ignore[attr-defined]
|
|
reset_provider_registry()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print("Usage: helper_provider_integration.py <test-name>", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
test_name = sys.argv[1]
|
|
tests = {
|
|
"discover-all-providers": test_discover_all_providers,
|
|
"get-all-providers-count": test_get_all_providers_count,
|
|
"get-configured-providers-filtering": test_get_configured_providers_filtering,
|
|
"get-provider-info-by-enum": test_get_provider_info_by_enum,
|
|
"get-provider-info-by-lowercase-string": test_get_provider_info_by_lowercase_string,
|
|
"get-provider-info-by-uppercase-string": test_get_provider_info_by_uppercase_string,
|
|
"get-provider-info-unknown-returns-none": test_get_provider_info_unknown_returns_none,
|
|
"is-provider-configured-true": test_is_provider_configured_true,
|
|
"is-provider-configured-false": test_is_provider_configured_false,
|
|
"get-default-provider-type-env-override": test_get_default_provider_type_env_override,
|
|
"get-default-provider-type-fallback-order": test_get_default_provider_type_fallback_order,
|
|
"get-default-provider-type-invalid-env-ignored": test_get_default_provider_type_invalid_env_ignored,
|
|
"get-default-provider-type-unconfigured-env-skipped": test_get_default_provider_type_unconfigured_env_skipped,
|
|
"get-default-provider-type-none": test_get_default_provider_type_none_nothing_configured,
|
|
"get-default-model-env-override": test_get_default_model_env_override,
|
|
"get-default-model-specific-provider": test_get_default_model_specific_provider,
|
|
"get-default-model-invalid-returns-none": test_get_default_model_invalid_returns_none,
|
|
"default-capabilities-openai": test_default_capabilities_openai,
|
|
"default-capabilities-anthropic": test_default_capabilities_anthropic,
|
|
"default-capabilities-google": test_default_capabilities_google,
|
|
"default-capabilities-cohere": test_default_capabilities_cohere,
|
|
"default-capabilities-mock": test_default_capabilities_mock,
|
|
"default-models-all-providers": test_default_models_all_providers,
|
|
"fallback-order-sequence": test_fallback_order_sequence,
|
|
"global-singleton-get-provider-registry": test_global_singleton_get_provider_registry,
|
|
"global-reset-clears-singleton": test_global_reset_clears_singleton,
|
|
"create-ai-provider-configured-openai": test_create_ai_provider_configured_openai,
|
|
"create-ai-provider-explicit-model-override": test_create_ai_provider_explicit_model_override,
|
|
"create-ai-provider-mock-rejected-without-env": test_create_ai_provider_mock_rejected_without_env,
|
|
"create-ai-provider-mock-succeeds-with-env": test_create_ai_provider_mock_succeeds_with_env,
|
|
"create-llm-no-provider-errors": test_create_llm_no_provider_errors,
|
|
"create-llm-configured-succeeds": test_create_llm_configured_succeeds,
|
|
"create-llm-mock-rejected": test_create_llm_mock_rejected,
|
|
"create-llm-mock-succeeds-with-env": test_create_llm_mock_succeeds_with_env,
|
|
"create-llm-explicit-model-id": test_create_llm_explicit_model_id,
|
|
"resolve-provider-by-name-success": test_resolve_provider_by_name_success,
|
|
"resolve-provider-by-name-case-insensitive": test_resolve_provider_by_name_case_insensitive,
|
|
"resolve-provider-by-name-unknown-raises": test_resolve_provider_by_name_unknown_raises,
|
|
"resolve-provider-by-name-unconfigured-raises": test_resolve_provider_by_name_unconfigured_raises,
|
|
"create-provider-instance-groq-type": test_create_provider_instance_groq_type,
|
|
"provider-capabilities-immutability": test_provider_capabilities_immutability,
|
|
"azure-provider-config-fields": test_azure_provider_config_fields,
|
|
"azure-missing-endpoint-raises": test_azure_missing_endpoint_raises,
|
|
}
|
|
|
|
test_fn = tests.get(test_name)
|
|
if test_fn is None:
|
|
print(f"Unknown test: {test_name}", file=sys.stderr)
|
|
sys.exit(1)
|
|
test_fn()
|