Files
cleveragents-core/features/steps/provider_registry_steps.py

913 lines
33 KiB
Python

"""Step definitions for provider registry feature tests."""
import os
import sys
import types
from dataclasses import FrozenInstanceError
from typing import Any
from unittest.mock import MagicMock
from behave import given, then, when # type: ignore[import-untyped]
from cleveragents.providers.llm.google_provider import GoogleChatProvider
from cleveragents.providers.llm.langchain_chat_provider import LangChainChatProvider
from cleveragents.providers.llm.openrouter_provider import OpenRouterChatProvider
from cleveragents.providers.registry import (
ProviderCapabilities,
ProviderInfo,
ProviderRegistry,
ProviderType,
_coerce_optional_str,
get_provider_registry,
reset_provider_registry,
)
def _make_settings(
openai: str | None = None,
anthropic: str | None = None,
google: str | None = None,
gemini: str | None = None,
azure: str | None = None,
openrouter: str | None = None,
cohere: str | None = None,
groq: str | None = None,
together: str | None = None,
default_provider: str | None = None,
default_model: str | None = None,
azure_endpoint: str | None = None,
azure_api_version: str | None = None,
azure_deployment: str | None = None,
openrouter_org: str | None = None,
):
"""Return a fake Settings-like object populated with API keys."""
settings = MagicMock()
settings.openai_api_key = openai
settings.anthropic_api_key = anthropic
settings.google_api_key = google
settings.gemini_api_key = gemini
settings.azure_api_key = azure
settings.openrouter_api_key = openrouter
settings.cohere_api_key = cohere
settings.groq_api_key = groq
settings.together_api_key = together
settings.default_provider = default_provider
settings.default_model = default_model
if azure and not azure_endpoint:
azure_endpoint = "https://example.openai.azure.com"
settings.azure_openai_endpoint = azure_endpoint
settings.azure_openai_api_version = azure_api_version
settings.azure_openai_deployment = azure_deployment
settings.openrouter_organization = openrouter_org
settings.provider_expected_env_vars = lambda: []
settings.configured_provider_names = lambda: []
settings.provider_configuration_diagnostics = lambda: {
"configured_providers": settings.configured_provider_names(),
"default_provider": default_provider or "",
"default_model": default_model or "",
"expected_env_vars": settings.provider_expected_env_vars(),
}
return settings
def _ensure_provider_type(context: Any) -> type[ProviderType]:
"""Ensure the ProviderType enum is available on the context."""
if not hasattr(context, "ProviderType"):
context.ProviderType = ProviderType
return context.ProviderType
def _get_provider_info_from_context(context: Any) -> ProviderInfo:
"""Return the ProviderInfo stored on the context or from the last result."""
info = getattr(context, "provider_info", None)
if info is None:
info = getattr(context, "result", None)
assert isinstance(info, ProviderInfo)
return info
def _stub_langchain_client_module(
context: Any,
provider_key: str,
module_name: str,
class_name: str,
) -> type:
"""Stub a LangChain client module and track constructor calls."""
original_module = sys.modules.get(module_name)
calls: list[dict[str, Any]] = []
class FakeClient: # pragma: no cover - helper class for coverage
def __init__(self, *args: Any, **kwargs: Any) -> None:
calls.append({"args": args, "kwargs": kwargs})
stub_module = types.ModuleType(module_name)
setattr(stub_module, class_name, FakeClient)
sys.modules[module_name] = stub_module
if not hasattr(context, "_cleanup_handlers"):
context._cleanup_handlers = []
def _restore_module() -> None:
if original_module is None:
sys.modules.pop(module_name, None)
else:
sys.modules[module_name] = original_module
context._cleanup_handlers.append(_restore_module)
if not hasattr(context, "stubbed_clients"):
context.stubbed_clients = {}
context.stubbed_clients[provider_key] = {
"calls": calls,
"class": FakeClient,
"module": module_name,
"class_name": class_name,
}
return FakeClient
LANGCHAIN_CLIENT_SPECS: dict[str, tuple[str, str]] = {
"openai": ("langchain_openai", "ChatOpenAI"),
"anthropic": ("langchain_anthropic", "ChatAnthropic"),
"google": ("langchain_google_genai", "ChatGoogleGenerativeAI"),
"gemini": ("langchain_google_genai", "ChatGoogleGenerativeAI"),
"azure": ("langchain_openai", "AzureChatOpenAI"),
"openrouter": ("langchain_openai", "ChatOpenAI"),
"groq": ("langchain_groq", "ChatGroq"),
"together": ("langchain_together", "ChatTogether"),
"cohere": ("langchain_cohere", "ChatCohere"),
}
# -- Given ------------------------------------------------------------------
@given("I import the ProviderRegistry class")
def step_impl_import_registry(context: Any) -> None:
context.ProviderRegistry = ProviderRegistry
@given("I import the ProviderType enum")
def step_impl_import_provider_type(context: Any) -> None:
context.ProviderType = ProviderType
@given("I import the ProviderCapabilities class")
def step_impl_import_capabilities(context: Any) -> None:
context.ProviderCapabilities = ProviderCapabilities
@given("I import the ProviderInfo class")
def step_impl_import_provider_info(context: Any) -> None:
context.ProviderInfo = ProviderInfo
@given("I import the optional string coercion helper")
def step_impl_import_optional_str_helper(context: Any) -> None:
context.coerce_optional_str = _coerce_optional_str
@given("I import the get_provider_registry function")
def step_impl_import_get_registry(context: Any) -> None:
context.get_provider_registry = get_provider_registry
context.reset_provider_registry = reset_provider_registry
@given("I import the reset_provider_registry function")
def step_impl_import_reset_registry(context: Any) -> None:
context.reset_provider_registry = reset_provider_registry
@given("I have a ProviderRegistry instance")
def step_impl_have_registry_instance(context: Any) -> None:
context.registry = ProviderRegistry()
@given("I have the ProviderRegistry class")
def step_impl_have_registry_class(context: Any) -> None:
context.ProviderRegistry = ProviderRegistry
@given("I have a ProviderRegistry with no API keys")
def step_impl_registry_no_keys(context: Any) -> None:
context.registry = ProviderRegistry(settings=_make_settings())
@given("I have a ProviderRegistry with OpenAI API key set")
def step_impl_registry_openai_key(context: Any) -> None:
context.registry = ProviderRegistry(settings=_make_settings(openai="sk-test"))
@given("I have a ProviderRegistry with Anthropic API key set")
def step_impl_registry_anthropic_key(context: Any) -> None:
context.registry = ProviderRegistry(settings=_make_settings(anthropic="sk-ant"))
@given('I have a ProviderRegistry with "{provider}" API key set')
def step_impl_registry_dynamic_key(context: Any, provider: str) -> None:
provider_key = provider.lower()
valid_keys = {
"openai",
"anthropic",
"google",
"gemini",
"azure",
"openrouter",
"cohere",
"groq",
"together",
}
if provider_key not in valid_keys: # pragma: no cover - defensive guard
raise AssertionError(f"Unsupported provider key: {provider}")
context.registry = ProviderRegistry(
settings=_make_settings(**{provider_key: f"sk-{provider_key}-test"})
)
@given(
'I have a ProviderRegistry with "{provider}" API key set and settings default provider "{default_provider}"'
)
def step_impl_registry_with_settings_default_provider(
context: Any, provider: str, default_provider: str
) -> None:
provider_key = provider.lower()
valid_keys = {
"openai",
"anthropic",
"google",
"gemini",
"azure",
"openrouter",
"cohere",
"groq",
"together",
}
if provider_key not in valid_keys: # pragma: no cover - defensive guard
raise AssertionError(f"Unsupported provider key: {provider}")
settings = _make_settings(
**{provider_key: f"sk-{provider_key}-test"}, default_provider=default_provider
)
context.registry = ProviderRegistry(settings=settings)
@given('I have a ProviderRegistry with default model "{model}" configured')
def step_impl_registry_with_default_model(context: Any, model: str) -> None:
context.registry = ProviderRegistry(settings=_make_settings(default_model=model))
@given('the OpenRouter organization setting is "{value}"')
def step_impl_set_openrouter_org(context: Any, value: str) -> None:
assert hasattr(context, "registry"), (
"Registry must exist before setting organization"
)
context.registry._settings.openrouter_organization = value
@given("the OpenRouter default headers include numeric entries")
def step_impl_set_openrouter_default_headers(context: Any) -> None:
context.openrouter_default_headers = {123: 456, "X-Debug": 789}
@given("the Azure endpoint setting is cleared")
def step_impl_clear_azure_endpoint(context: Any) -> None:
assert hasattr(context, "registry"), "Registry must exist before clearing endpoint"
context.registry._settings.azure_openai_endpoint = None
@given('CLEVERAGENTS_DEFAULT_PROVIDER is set to "{value}"')
def step_impl_set_default_provider_env(context: Any, value: str) -> None:
context.original_provider_env = os.environ.get("CLEVERAGENTS_DEFAULT_PROVIDER")
os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = value
@given("CLEVERAGENTS_DEFAULT_PROVIDER is not set")
def step_impl_unset_default_provider_env(context: Any) -> None:
context.original_provider_env = os.environ.get("CLEVERAGENTS_DEFAULT_PROVIDER")
os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None)
@given('CLEVERAGENTS_DEFAULT_MODEL is set to "{value}"')
def step_impl_set_default_model_env(context: Any, value: str) -> None:
context.original_model_env = os.environ.get("CLEVERAGENTS_DEFAULT_MODEL")
os.environ["CLEVERAGENTS_DEFAULT_MODEL"] = value
@given("CLEVERAGENTS_DEFAULT_MODEL is not set")
def step_impl_unset_default_model_env(context: Any) -> None:
context.original_model_env = os.environ.get("CLEVERAGENTS_DEFAULT_MODEL")
os.environ.pop("CLEVERAGENTS_DEFAULT_MODEL", None)
@given("langchain OpenAI client is stubbed")
def step_impl_stub_langchain_openai(context: Any) -> None:
module_name, class_name = LANGCHAIN_CLIENT_SPECS["openai"]
fake_class = _stub_langchain_client_module(
context,
"openai",
module_name,
class_name,
)
context.chat_openai_calls = context.stubbed_clients["openai"]["calls"]
context.fake_chat_openai_class = fake_class
@given('the "{provider_name}" LangChain client is stubbed')
def step_impl_stub_named_langchain_client(context: Any, provider_name: str) -> None:
provider_key = provider_name.lower()
if provider_key not in LANGCHAIN_CLIENT_SPECS: # pragma: no cover - defensive guard
raise AssertionError(f"Unsupported LangChain client: {provider_name}")
module_name, class_name = LANGCHAIN_CLIENT_SPECS[provider_key]
_stub_langchain_client_module(context, provider_key, module_name, class_name)
@given("the provider registry LLM factory is stubbed")
def step_impl_stub_provider_factory(context: Any) -> None:
context.stubbed_llm = object()
context.registry._create_provider_llm = MagicMock(return_value=context.stubbed_llm)
# -- When -------------------------------------------------------------------
@when("I create a ProviderRegistry with default settings")
def step_impl_create_registry_default(context: Any) -> None:
context.registry = context.ProviderRegistry()
@when("I coerce optional string with integer {value:d}")
def step_impl_coerce_optional_string_with_integer(context: Any, value: int) -> None:
coerce_helper = getattr(context, "coerce_optional_str", _coerce_optional_str)
context.coerced_value = coerce_helper(value)
@when("I create a ProviderCapabilities with all fields")
def step_impl_capabilities_all_fields(context: Any) -> None:
context.capabilities = context.ProviderCapabilities(
supports_streaming=True,
supports_tool_calls=True,
supports_vision=True,
max_context_length=128000,
supports_json_mode=True,
)
@when("I create a ProviderCapabilities with default values")
def step_impl_capabilities_defaults(context: Any) -> None:
context.capabilities = context.ProviderCapabilities()
@when("I create a ProviderCapabilities instance")
def step_impl_capabilities_instance(context: Any) -> None:
context.capabilities = context.ProviderCapabilities()
@when("I create a ProviderInfo for openai")
def step_impl_provider_info_openai(context: Any) -> None:
"""Create a ProviderInfo for OpenAI."""
provider_enum = _ensure_provider_type(context)
context.provider_info = context.ProviderInfo(
provider_type=provider_enum.OPENAI,
name="Openai",
api_key_env_var="OPENAI_API_KEY",
default_model="gpt-4o",
)
@when("I call get_all_providers")
def step_impl_call_get_all(context: Any) -> None:
context.result = context.registry.get_all_providers()
@when("I call get_configured_providers")
def step_impl_call_get_configured(context: Any) -> None:
context.result = context.registry.get_configured_providers()
@when("I get provider info for ProviderType.OPENAI")
def step_impl_get_provider_info_enum(context: Any) -> None:
"""Get provider info for ProviderType.OPENAI."""
provider_enum = _ensure_provider_type(context)
context.result = context.registry.get_provider_info(provider_enum.OPENAI)
@when('I get provider info for string "{value}"')
def step_impl_get_provider_info_string(context: Any, value: str) -> None:
context.result = context.registry.get_provider_info(value)
@when("I check if openai is configured")
def step_impl_check_openai_configured(context: Any) -> None:
context.result = context.registry.is_provider_configured("openai")
@when("I call get_default_provider_type")
def step_impl_call_get_default_provider(context: Any) -> None:
context.result = context.registry.get_default_provider_type()
@when("I call get_default_model")
def step_impl_call_get_default_model(context: Any) -> None:
context.result = context.registry.get_default_model()
@when("I call get_default_model for ProviderType.OPENAI")
def step_impl_call_get_default_model_openai(context: Any) -> None:
"""Call get_default_model for OpenAI."""
provider_enum = _ensure_provider_type(context)
context.result = context.registry.get_default_model(provider_enum.OPENAI)
@when('I call get_default_model for string "{value}"')
def step_impl_call_get_default_model_string(context: Any, value: str) -> None:
context.result = context.registry.get_default_model(value)
@when("I get capabilities for ProviderType.OPENAI")
def step_impl_get_capabilities_openai(context: Any) -> None:
info = context.registry.get_provider_info(_ensure_provider_type(context).OPENAI)
context.capabilities = info.capabilities if info else None
@when("I get capabilities for ProviderType.ANTHROPIC")
def step_impl_get_capabilities_anthropic(context: Any) -> None:
info = context.registry.get_provider_info(_ensure_provider_type(context).ANTHROPIC)
context.capabilities = info.capabilities if info else None
@when("I get capabilities for ProviderType.GOOGLE")
def step_impl_get_capabilities_google(context: Any) -> None:
info = context.registry.get_provider_info(_ensure_provider_type(context).GOOGLE)
context.capabilities = info.capabilities if info else None
@when("I call get_provider_registry twice")
def step_impl_call_get_registry_twice(context: Any) -> None:
reset_provider_registry()
context.first_registry = get_provider_registry()
context.second_registry = get_provider_registry()
@when("I get the global registry and then reset it")
def step_impl_get_and_reset_registry(context: Any) -> None:
context.original_registry = get_provider_registry()
reset_provider_registry()
@when("I try to create an LLM without specifying a provider")
def step_impl_create_llm_no_provider(context: Any) -> None:
try:
context.registry.create_llm()
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when('I try to create an LLM for provider "{value}"')
def step_impl_create_llm_specific_provider(context: Any, value: str) -> None:
try:
context.registry.create_llm(provider_type=value)
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when('I call create_llm for provider "{value}" and model "{model}"')
def step_impl_call_create_llm_with_model(context: Any, value: str, model: str) -> None:
context.created_llm = context.registry.create_llm(
provider_type=value, model_id=model
)
@when("I call create_llm without specifying a provider")
def step_impl_call_create_llm_without_provider(context: Any) -> None:
context.created_llm = context.registry.create_llm()
@when("I call create_llm for ProviderType.{enum_name}")
def step_impl_call_create_llm_enum(context: Any, enum_name: str) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, enum_name.upper())
context.created_llm = context.registry.create_llm(provider_type=provider_type)
@when('I call create_llm for provider "{value}" without specifying a model')
def step_impl_call_create_llm_no_model(context: Any, value: str) -> None:
context.created_llm = context.registry.create_llm(provider_type=value)
@when('I call _create_provider_llm for provider "{value}" with model "{model}"')
def step_impl_call_private_llm(context: Any, value: str, model: str) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
context.created_llm = context.registry._create_provider_llm(provider_type, model)
@when(
'I call _create_provider_llm for provider "{value}" with model "{model}" and custom default headers'
)
def step_impl_call_private_llm_with_headers(
context: Any, value: str, model: str
) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
default_headers = getattr(context, "openrouter_default_headers", None)
assert default_headers is not None, "OpenRouter default headers were not configured"
context.created_llm = context.registry._create_provider_llm(
provider_type, model, default_headers=default_headers
)
@when('I try to call _create_provider_llm for provider "{value}" with model "{model}"')
def step_impl_try_private_llm(context: Any, value: str, model: str) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
try:
context.registry._create_provider_llm(provider_type, model)
context.error = None
except Exception as exc: # pragma: no cover - defensive path
context.error = exc
@when("I try to create an AI provider without specifying a provider")
def step_impl_create_ai_provider_no_provider(context: Any) -> None:
try:
context.registry.create_ai_provider()
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when("I create an AI provider without specifying a provider")
def step_impl_create_ai_provider_default(context: Any) -> None:
context.ai_provider = context.registry.create_ai_provider()
@when('I try to create an AI provider for provider "{value}"')
def step_impl_create_ai_provider_invalid(context: Any, value: str) -> None:
try:
context.registry.create_ai_provider(provider_type=value)
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when('I create an AI provider for provider "{value}"')
def step_impl_create_ai_provider_success(context: Any, value: str) -> None:
context.ai_provider = context.registry.create_ai_provider(provider_type=value)
@when('I create an AI provider specifying provider "{value}" and model "{model}"')
def step_impl_create_ai_provider_with_model(
context: Any, value: str, model: str
) -> None:
context.ai_provider = context.registry.create_ai_provider(
provider_type=value, model_id=model
)
@when('I call the AI provider llm factory with model "{model}"')
def step_impl_call_ai_provider_llm_factory(context: Any, model: str) -> None:
assert hasattr(context, "ai_provider"), "AI provider has not been created"
context.created_llm = context.ai_provider._llm_factory(model)
@when("I check the FALLBACK_ORDER")
def step_impl_check_fallback_order(context: Any) -> None:
context.fallback_order = context.ProviderRegistry.FALLBACK_ORDER
@when("I check the DEFAULT_MODELS")
def step_impl_check_default_models(context: Any) -> None:
context.default_models = context.ProviderRegistry.DEFAULT_MODELS
# -- Then -------------------------------------------------------------------
@then("the registry should be initialized")
def step_impl_registry_initialized(context: Any) -> None:
assert context.registry is not None
@then("the registry should have provider information")
def step_impl_registry_has_providers(context: Any) -> None:
assert len(context.registry.get_all_providers()) > 0
@then('the coerced optional string result should be "{expected}"')
def step_impl_coerced_optional_string_result(context: Any, expected: str) -> None:
assert getattr(context, "coerced_value", None) == expected, (
f"Expected coerced value {expected!r}, got {getattr(context, 'coerced_value', None)!r}"
)
@then('ProviderType should have {enum_name} value "{value}"')
def step_impl_provider_type_values(context: Any, enum_name: str, value: str) -> None:
enum_member = getattr(_ensure_provider_type(context), enum_name.upper())
assert enum_member.value == value
@then("supports_streaming should be {expected}")
def step_impl_supports_streaming(context: Any, expected: str) -> None:
assert context.capabilities.supports_streaming is (expected == "True")
@then("supports_tool_calls should be {expected}")
def step_impl_supports_tool_calls(context: Any, expected: str) -> None:
assert context.capabilities.supports_tool_calls is (expected == "True")
@then("supports_vision should be {expected}")
def step_impl_supports_vision(context: Any, expected: str) -> None:
assert context.capabilities.supports_vision is (expected == "True")
@then("max_context_length should be {length:d}")
def step_impl_max_context_length(context: Any, length: int) -> None:
assert context.capabilities.max_context_length == length
@then("supports_json_mode should be {expected}")
def step_impl_supports_json_mode(context: Any, expected: str) -> None:
assert context.capabilities.supports_json_mode is (expected == "True")
@then("the provider_type should be ProviderType.OPENAI")
def step_impl_provider_type_is_openai(context: Any) -> None:
provider_enum = _ensure_provider_type(context)
info = _get_provider_info_from_context(context)
assert info.provider_type == provider_enum.OPENAI
@then("the provider_type should be ProviderType.ANTHROPIC")
def step_impl_provider_type_is_anthropic(context: Any) -> None:
provider_enum = _ensure_provider_type(context)
info = _get_provider_info_from_context(context)
assert info.provider_type == provider_enum.ANTHROPIC
@then('the provider info name should be "{value}"')
def step_impl_provider_info_name(context: Any, value: str) -> None:
assert context.provider_info.name == value
@then("the is_configured should be False by default")
def step_impl_provider_info_is_configured_default(context: Any) -> None:
assert context.provider_info.is_configured is False
@then("the provider registry result should be a list of ProviderInfo")
def step_impl_result_list_provider_info(context: Any) -> None:
assert isinstance(context.result, list)
assert all(isinstance(item, ProviderInfo) for item in context.result)
@then("the provider registry result should include core provider types")
def step_impl_result_contains_core_providers(context: Any) -> None:
provider_enum = _ensure_provider_type(context)
provider_types = {p.provider_type for p in context.result}
core_types = {provider_enum.OPENAI, provider_enum.ANTHROPIC, provider_enum.GOOGLE}
assert core_types.issubset(provider_types)
@then("the provider registry result should be an empty list")
def step_impl_result_empty_list(context: Any) -> None:
assert context.result == []
@then("the provider registry result should contain OpenAI provider")
def step_impl_result_contains_openai(context: Any) -> None:
provider_types = [p.provider_type for p in context.result]
assert _ensure_provider_type(context).OPENAI in provider_types
@then("the provider registry OpenAI entry should be configured")
def step_impl_openai_is_configured_true(context: Any) -> None:
openai_provider = next(
(
p
for p in context.result
if p.provider_type == _ensure_provider_type(context).OPENAI
),
None,
)
assert openai_provider is not None
assert openai_provider.is_configured is True
@then("the provider registry result should be a ProviderInfo instance")
def step_impl_result_is_provider_info(context: Any) -> None:
assert isinstance(context.result, ProviderInfo)
@then("the provider registry result should be None")
def step_impl_result_is_none(context: Any) -> None:
assert context.result is None, f"Expected None, got {context.result!r}"
@then("the provider registry result should be False")
def step_impl_result_false(context: Any) -> None:
assert context.result is False, f"Expected False, got {context.result!r}"
@then("the provider registry result should be True")
def step_impl_result_true(context: Any) -> None:
assert context.result is True, f"Expected True, got {context.result!r}"
@then("the provider registry result should be ProviderType.OPENAI")
def step_impl_result_provider_type_openai(context: Any) -> None:
assert context.result == _ensure_provider_type(context).OPENAI
@then("the provider registry result should be ProviderType.ANTHROPIC")
def step_impl_result_provider_type_anthropic(context: Any) -> None:
assert context.result == _ensure_provider_type(context).ANTHROPIC
@then('the provider registry result should be "{value}"')
def step_impl_result_string(context: Any, value: str) -> None:
assert context.result == value, f"Expected {value!r}, got {context.result!r}"
@then("both provider registry calls should return the same instance")
def step_impl_same_instance(context: Any) -> None:
assert context.first_registry is context.second_registry
@then("resetting the provider registry should return a new instance")
def step_impl_new_registry_instance(context: Any) -> None:
new_registry = get_provider_registry()
assert new_registry is not context.original_registry
@then("a provider registry ValueError should be raised")
def step_impl_value_error(context: Any) -> None:
assert context.error is not None
assert isinstance(context.error, ValueError)
@then("the provider registry error should mention configuring API keys")
def step_impl_error_mentions_config(context: Any) -> None:
assert "api" in str(context.error).lower() or "config" in str(context.error).lower()
@then("the provider registry error should mention unknown provider type")
def step_impl_error_mentions_unknown_provider(context: Any) -> None:
assert "unknown" in str(context.error).lower()
@then("the provider registry error should mention missing Azure endpoint")
def step_impl_error_mentions_missing_endpoint(context: Any) -> None:
assert context.error is not None
message = str(context.error).lower()
assert "azure" in message and "endpoint" in message
@then(
'the provider registry error should mention missing "{env_var}" environment variable'
)
def step_impl_error_mentions_missing_env(context: Any, env_var: str) -> None:
assert context.error is not None
message = str(context.error)
assert env_var in message, (
f"Expected error message to mention {env_var!r}, got {message!r}"
)
@then("the provider registry AI provider should be a LangChainChatProvider")
def step_impl_ai_provider_is_langchain(context: Any) -> None:
assert isinstance(context.ai_provider, LangChainChatProvider)
@then("the provider registry AI provider should be a GoogleChatProvider")
def step_impl_ai_provider_is_google(context: Any) -> None:
assert isinstance(context.ai_provider, GoogleChatProvider)
@then("the provider registry AI provider should be an OpenRouterChatProvider")
def step_impl_ai_provider_is_openrouter(context: Any) -> None:
assert isinstance(context.ai_provider, OpenRouterChatProvider)
@then('the provider registry AI provider name should be "{value}"')
def step_impl_ai_provider_name(context: Any, value: str) -> None:
assert context.ai_provider.name == value
@then('the provider registry AI provider model_id should be "{value}"')
def step_impl_ai_provider_model_id(context: Any, value: str) -> None:
assert context.ai_provider.model_id == value
@then(
'the stubbed LLM factory should be called for ProviderType.{enum_name} with model "{model}"'
)
def step_impl_stubbed_llm_factory_called(
context: Any, enum_name: str, model: str
) -> None:
provider_enum = _ensure_provider_type(context)
expected_type = getattr(provider_enum, enum_name.upper())
context.registry._create_provider_llm.assert_called_once()
args, kwargs = context.registry._create_provider_llm.call_args
assert args[:2] == (expected_type, model)
if kwargs:
# max_retries is propagated by create_ai_provider to keep retry semantics aligned
max_retries = kwargs.get("max_retries")
assert max_retries is None or isinstance(max_retries, int), (
f"Expected max_retries int or None, got {max_retries!r}"
)
assert context.created_llm is context.stubbed_llm
@then('the stubbed ChatOpenAI should be created with model "{model}"')
def step_impl_stubbed_chat_openai_created(context: Any, model: str) -> None:
assert getattr(context, "chat_openai_calls", []), "Expected ChatOpenAI to be called"
last_call = context.chat_openai_calls[-1]
recorded_model = last_call.get("model")
if recorded_model is None:
recorded_model = last_call.get("kwargs", {}).get("model")
assert recorded_model == model
assert isinstance(context.created_llm, context.fake_chat_openai_class)
@then(
'the stubbed {provider_name} client should be created with argument "{arg_name}" value "{expected}"'
)
def step_impl_stubbed_client_argument(
context: Any, provider_name: str, arg_name: str, expected: str
) -> None:
provider_key = provider_name.lower()
stubbed_clients = getattr(context, "stubbed_clients", {})
assert provider_key in stubbed_clients, (
f"No stubbed client recorded for {provider_name}"
)
client_data = stubbed_clients[provider_key]
calls = client_data["calls"]
assert calls, f"Expected {provider_name} client to be instantiated"
last_call = calls[-1]
assert last_call["kwargs"].get(arg_name) == expected, (
f"Expected {arg_name}={expected!r}, got {last_call['kwargs'].get(arg_name)!r}"
)
assert isinstance(context.created_llm, client_data["class"])
@then(
'the stubbed OpenRouter client headers should include "{header_name}" value "{expected}"'
)
def step_impl_stubbed_openrouter_headers(
context: Any, header_name: str, expected: str
) -> None:
stubbed_clients = getattr(context, "stubbed_clients", {})
client_data = stubbed_clients.get("openrouter")
assert client_data, "No stubbed client recorded for OpenRouter"
calls = client_data["calls"]
assert calls, "Expected OpenRouter client to be instantiated"
headers = calls[-1]["kwargs"].get("default_headers")
assert isinstance(headers, dict), "Expected default_headers to be a dict"
assert headers.get(header_name) == expected, (
f"Expected {header_name}={expected!r}, got {headers.get(header_name)!r}"
)
@then("the instance should be frozen")
def step_impl_instance_frozen(context: Any) -> None:
try:
context.capabilities.supports_streaming = False
raise AssertionError("Expected FrozenInstanceError")
except FrozenInstanceError:
context.frozen_error = True
@then("attempting to modify it should raise an error")
def step_impl_modify_raises_error(context: Any) -> None:
assert getattr(context, "frozen_error", False) is True
@then("OpenAI should be first in the order")
def step_impl_openai_first(context: Any) -> None:
assert context.fallback_order[0] == _ensure_provider_type(context).OPENAI
@then("Anthropic should be second in the order")
def step_impl_anthropic_second(context: Any) -> None:
assert context.fallback_order[1] == _ensure_provider_type(context).ANTHROPIC
@then("Google should be third in the order")
def step_impl_google_third(context: Any) -> None:
assert context.fallback_order[2] == _ensure_provider_type(context).GOOGLE
@then('{provider} default should be "{model}"')
def step_impl_default_models(context: Any, provider: str, model: str) -> None:
provider_enum = _ensure_provider_type(context)
enum_member = getattr(provider_enum, provider.upper())
assert context.default_models[enum_member] == model
# -- Hooks ------------------------------------------------------------------
# Hook logic lives in features/environment.py to ensure scenarios reset state.