Files
cleveragents-core/features/steps/provider_registry_steps.py
T
hurui200320 7164b04019
CI / lint (push) Successful in 1m1s
CI / build (push) Successful in 53s
CI / quality (push) Successful in 1m23s
CI / typecheck (push) Successful in 1m24s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 1m42s
CI / push-validation (push) Successful in 21s
CI / helm (push) Successful in 25s
CI / e2e_tests (push) Successful in 4m6s
CI / integration_tests (push) Successful in 4m12s
CI / unit_tests (push) Successful in 6m39s
CI / docker (push) Successful in 1m44s
CI / coverage (push) Successful in 10m51s
CI / status-check (push) Successful in 2s
CI / benchmark-publish (push) Successful in 1h17m10s
CI / benchmark-publish (pull_request) Has been skipped
CI / helm (pull_request) Successful in 33s
CI / build (pull_request) Successful in 50s
CI / lint (pull_request) Successful in 1m7s
CI / quality (pull_request) Successful in 1m9s
CI / typecheck (pull_request) Successful in 1m20s
CI / security (pull_request) Successful in 1m25s
CI / push-validation (pull_request) Successful in 19s
CI / integration_tests (pull_request) Successful in 3m20s
CI / e2e_tests (pull_request) Failing after 5m3s
CI / unit_tests (pull_request) Successful in 6m40s
CI / docker (pull_request) Successful in 1m30s
CI / coverage (pull_request) Successful in 11m9s
CI / status-check (pull_request) Failing after 3s
CI / benchmark-regression (pull_request) Failing after 1m11s
refactor(providers): unify provider factory behind single source of truth
Extract _get_api_key() and _create_provider_instance() as a single internal
factory that handles all provider types, including MOCK. Both create_llm() and
create_ai_provider() now delegate to _create_provider_instance(), eliminating
the duplicated if/elif provider-switching chains and the divergence between the
two code paths. Adding a new provider now requires touching exactly one place.

Key changes:
- New _get_api_key(provider_type) consolidates API key lookup and validation
  that was previously duplicated across create_llm(), _create_provider_llm(),
  and every branch of create_ai_provider().
- New _create_provider_instance() is the unified raw-LLM factory, replacing
  _create_provider_llm() and the specialised adapter constructors in
  create_ai_provider(). MOCK is now handled here via FakeListLLM.
- create_ai_provider() wraps all providers uniformly in LangChainChatProvider
  instead of using specialised adapters (OpenAIChatProvider, AnthropicChatProvider,
  GoogleChatProvider, OpenRouterChatProvider) for the four primary providers.
- create_llm() drops its own API key validation block; validation now happens
  inside _create_provider_instance() via _get_api_key().
- All BDD scenarios updated: _create_provider_llm references become
  _create_provider_instance, and provider-type assertions on create_ai_provider
  results are updated to LangChainChatProvider.

Review fixes (Cycle 2):
- Add _coerce_env_bool() for consistent boolean env-var coercion in
  CLEVERAGENTS_ALLOW_MOCK_PROVIDER checks.
- Introduce _ApiKeyMissing sentinel to distinguish 'not provided' from None
  in provider configuration.
- Set MOCK supports_streaming=False in DEFAULT_CAPABILITIES to align with
  FakeListLLM semantics.
- Make MOCK is_configured conditional on CLEVERAGENTS_ALLOW_MOCK_PROVIDER.
- Use FakeListLLM(responses=['mock response'] * 10) to prevent IndexError
  in multi-step workflows.
- Strip max_retries from kwargs before passing to LangChain constructors.
- Extract _resolve_provider_type() helper shared by create_llm() and
  create_ai_provider() to reduce duplication and improve readability.
- Ensure __api_key_sentinel flows through factory_kwargs even when api_key
  is None, avoiding double _get_api_key() calls.
- Add CHANGELOG env-var documentation and providers.md env-var table.
- Resolve template DB lock issues by persisting in-memory SQLite via
  VACUUM INTO instead of direct file creation on tmpfs.

ISSUES CLOSED: #10949
2026-05-05 06:16:46 +00:00

1099 lines
40 KiB
Python

"""Step definitions for provider registry feature tests."""
import os
import sys
import types
from dataclasses import FrozenInstanceError
from typing import Any
from unittest.mock import MagicMock
from behave import given, then, when # type: ignore[import-untyped]
from cleveragents.providers.llm.langchain_chat_provider import LangChainChatProvider
from cleveragents.providers.registry import (
ProviderCapabilities,
ProviderInfo,
ProviderRegistry,
ProviderType,
_coerce_optional_str,
get_provider_registry,
reset_provider_registry,
)
def _make_settings(
openai: str | None = None,
anthropic: str | None = None,
google: str | None = None,
gemini: str | None = None,
azure: str | None = None,
openrouter: str | None = None,
cohere: str | None = None,
groq: str | None = None,
together: str | None = None,
default_provider: str | None = None,
default_model: str | None = None,
azure_endpoint: str | None = None,
azure_api_version: str | None = None,
azure_deployment: str | None = None,
openrouter_org: str | None = None,
):
"""Return a fake Settings-like object populated with API keys."""
settings = MagicMock()
settings.openai_api_key = openai
settings.anthropic_api_key = anthropic
settings.google_api_key = google
settings.gemini_api_key = gemini
settings.azure_api_key = azure
settings.openrouter_api_key = openrouter
settings.cohere_api_key = cohere
settings.groq_api_key = groq
settings.together_api_key = together
settings.default_provider = default_provider
settings.default_model = default_model
if azure and not azure_endpoint:
azure_endpoint = "https://example.openai.azure.com"
settings.azure_openai_endpoint = azure_endpoint
settings.azure_openai_api_version = azure_api_version
settings.azure_openai_deployment = azure_deployment
settings.openrouter_organization = openrouter_org
settings.provider_expected_env_vars = lambda: []
settings.configured_provider_names = lambda: []
settings.provider_configuration_diagnostics = lambda: {
"configured_providers": settings.configured_provider_names(),
"default_provider": default_provider or "",
"default_model": default_model or "",
"expected_env_vars": settings.provider_expected_env_vars(),
}
return settings
def _ensure_provider_type(context: Any) -> type[ProviderType]:
"""Ensure the ProviderType enum is available on the context."""
if not hasattr(context, "ProviderType"):
context.ProviderType = ProviderType
return context.ProviderType
def _get_provider_info_from_context(context: Any) -> ProviderInfo:
"""Return the ProviderInfo stored on the context or from the last result."""
info = getattr(context, "provider_info", None)
if info is None:
info = getattr(context, "result", None)
assert isinstance(info, ProviderInfo)
return info
def _stub_langchain_client_module(
context: Any,
provider_key: str,
module_name: str,
class_name: str,
) -> type:
"""Stub a LangChain client module and track constructor calls."""
original_module = sys.modules.get(module_name)
calls: list[dict[str, Any]] = []
class FakeClient: # pragma: no cover - helper class for coverage
def __init__(self, *args: Any, **kwargs: Any) -> None:
calls.append({"args": args, "kwargs": kwargs})
stub_module = types.ModuleType(module_name)
setattr(stub_module, class_name, FakeClient)
sys.modules[module_name] = stub_module
if not hasattr(context, "_cleanup_handlers"):
context._cleanup_handlers = []
def _restore_module() -> None:
if original_module is None:
sys.modules.pop(module_name, None)
else:
sys.modules[module_name] = original_module
context._cleanup_handlers.append(_restore_module)
if not hasattr(context, "stubbed_clients"):
context.stubbed_clients = {}
context.stubbed_clients[provider_key] = {
"calls": calls,
"class": FakeClient,
"module": module_name,
"class_name": class_name,
}
return FakeClient
LANGCHAIN_CLIENT_SPECS: dict[str, tuple[str, str]] = {
"openai": ("langchain_openai", "ChatOpenAI"),
"anthropic": ("langchain_anthropic", "ChatAnthropic"),
"google": ("langchain_google_genai", "ChatGoogleGenerativeAI"),
"gemini": ("langchain_google_genai", "ChatGoogleGenerativeAI"),
"azure": ("langchain_openai", "AzureChatOpenAI"),
"openrouter": ("langchain_openai", "ChatOpenAI"),
"groq": ("langchain_groq", "ChatGroq"),
"together": ("langchain_together", "ChatTogether"),
"cohere": ("langchain_cohere", "ChatCohere"),
"mock": ("langchain_community.llms", "FakeListLLM"),
}
# -- Given ------------------------------------------------------------------
@given("I import the ProviderRegistry class")
def step_impl_import_registry(context: Any) -> None:
context.ProviderRegistry = ProviderRegistry
@given("I import the ProviderType enum")
def step_impl_import_provider_type(context: Any) -> None:
context.ProviderType = ProviderType
@given("I import the ProviderCapabilities class")
def step_impl_import_capabilities(context: Any) -> None:
context.ProviderCapabilities = ProviderCapabilities
@given("I import the ProviderInfo class")
def step_impl_import_provider_info(context: Any) -> None:
context.ProviderInfo = ProviderInfo
@given("I import the optional string coercion helper")
def step_impl_import_optional_str_helper(context: Any) -> None:
context.coerce_optional_str = _coerce_optional_str
@given("I import the get_provider_registry function")
def step_impl_import_get_registry(context: Any) -> None:
context.get_provider_registry = get_provider_registry
context.reset_provider_registry = reset_provider_registry
@given("I import the reset_provider_registry function")
def step_impl_import_reset_registry(context: Any) -> None:
context.reset_provider_registry = reset_provider_registry
@given("I have a ProviderRegistry instance")
def step_impl_have_registry_instance(context: Any) -> None:
context.registry = ProviderRegistry()
@given("I have the ProviderRegistry class")
def step_impl_have_registry_class(context: Any) -> None:
context.ProviderRegistry = ProviderRegistry
@given("I have a ProviderRegistry with no API keys")
def step_impl_registry_no_keys(context: Any) -> None:
context.registry = ProviderRegistry(settings=_make_settings())
@given("I have a ProviderRegistry with OpenAI API key set")
def step_impl_registry_openai_key(context: Any) -> None:
context.registry = ProviderRegistry(settings=_make_settings(openai="sk-test"))
@given("I have a ProviderRegistry with Anthropic API key set")
def step_impl_registry_anthropic_key(context: Any) -> None:
context.registry = ProviderRegistry(settings=_make_settings(anthropic="sk-ant"))
@given('I have a ProviderRegistry with "{provider}" API key set')
def step_impl_registry_dynamic_key(context: Any, provider: str) -> None:
provider_key = provider.lower()
valid_keys = {
"openai",
"anthropic",
"google",
"gemini",
"azure",
"openrouter",
"cohere",
"groq",
"together",
}
if provider_key not in valid_keys: # pragma: no cover - defensive guard
raise AssertionError(f"Unsupported provider key: {provider}")
context.registry = ProviderRegistry(
settings=_make_settings(**{provider_key: f"sk-{provider_key}-test"})
)
@given(
'I have a ProviderRegistry with "{provider}" API key set and settings default provider "{default_provider}"'
)
def step_impl_registry_with_settings_default_provider(
context: Any, provider: str, default_provider: str
) -> None:
provider_key = provider.lower()
valid_keys = {
"openai",
"anthropic",
"google",
"gemini",
"azure",
"openrouter",
"cohere",
"groq",
"together",
}
if provider_key not in valid_keys: # pragma: no cover - defensive guard
raise AssertionError(f"Unsupported provider key: {provider}")
settings = _make_settings(
**{provider_key: f"sk-{provider_key}-test"}, default_provider=default_provider
)
context.registry = ProviderRegistry(settings=settings)
@given('I have a ProviderRegistry with default model "{model}" configured')
def step_impl_registry_with_default_model(context: Any, model: str) -> None:
context.registry = ProviderRegistry(settings=_make_settings(default_model=model))
@given('the OpenRouter organization setting is "{value}"')
def step_impl_set_openrouter_org(context: Any, value: str) -> None:
assert hasattr(context, "registry"), (
"Registry must exist before setting organization"
)
context.registry._settings.openrouter_organization = value
@given("the OpenRouter default headers include numeric entries")
def step_impl_set_openrouter_default_headers(context: Any) -> None:
context.openrouter_default_headers = {123: 456, "X-Debug": 789}
@given("the Azure endpoint setting is cleared")
def step_impl_clear_azure_endpoint(context: Any) -> None:
assert hasattr(context, "registry"), "Registry must exist before clearing endpoint"
context.registry._settings.azure_openai_endpoint = None
@given('CLEVERAGENTS_DEFAULT_PROVIDER is set to "{value}"')
def step_impl_set_default_provider_env(context: Any, value: str) -> None:
context.original_provider_env = os.environ.get("CLEVERAGENTS_DEFAULT_PROVIDER")
os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = value
@given("CLEVERAGENTS_DEFAULT_PROVIDER is not set")
def step_impl_unset_default_provider_env(context: Any) -> None:
context.original_provider_env = os.environ.get("CLEVERAGENTS_DEFAULT_PROVIDER")
os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None)
@given('CLEVERAGENTS_DEFAULT_MODEL is set to "{value}"')
def step_impl_set_default_model_env(context: Any, value: str) -> None:
context.original_model_env = os.environ.get("CLEVERAGENTS_DEFAULT_MODEL")
os.environ["CLEVERAGENTS_DEFAULT_MODEL"] = value
@given("CLEVERAGENTS_DEFAULT_MODEL is not set")
def step_impl_unset_default_model_env(context: Any) -> None:
context.original_model_env = os.environ.get("CLEVERAGENTS_DEFAULT_MODEL")
os.environ.pop("CLEVERAGENTS_DEFAULT_MODEL", None)
@given("CLEVERAGENTS_ALLOW_MOCK_PROVIDER is set")
def step_impl_allow_mock_provider_set(context: Any) -> None:
context.original_allow_mock_env = os.environ.get("CLEVERAGENTS_ALLOW_MOCK_PROVIDER")
os.environ["CLEVERAGENTS_ALLOW_MOCK_PROVIDER"] = "true"
@given("CLEVERAGENTS_ALLOW_MOCK_PROVIDER is not set")
def step_impl_allow_mock_provider_unset(context: Any) -> None:
context.original_allow_mock_env = os.environ.get("CLEVERAGENTS_ALLOW_MOCK_PROVIDER")
os.environ.pop("CLEVERAGENTS_ALLOW_MOCK_PROVIDER", None)
@given("langchain OpenAI client is stubbed")
def step_impl_stub_langchain_openai(context: Any) -> None:
module_name, class_name = LANGCHAIN_CLIENT_SPECS["openai"]
fake_class = _stub_langchain_client_module(
context,
"openai",
module_name,
class_name,
)
context.chat_openai_calls = context.stubbed_clients["openai"]["calls"]
context.fake_chat_openai_class = fake_class
@given('the "{provider_name}" LangChain client is stubbed')
def step_impl_stub_named_langchain_client(context: Any, provider_name: str) -> None:
provider_key = provider_name.lower()
if provider_key not in LANGCHAIN_CLIENT_SPECS: # pragma: no cover - defensive guard
raise AssertionError(f"Unsupported LangChain client: {provider_name}")
module_name, class_name = LANGCHAIN_CLIENT_SPECS[provider_key]
_stub_langchain_client_module(context, provider_key, module_name, class_name)
@given("the provider registry LLM factory is stubbed")
def step_impl_stub_provider_factory(context: Any) -> None:
context.stubbed_llm = object()
context.registry._create_provider_instance = MagicMock(
return_value=context.stubbed_llm
)
# -- When -------------------------------------------------------------------
@when("I create a ProviderRegistry with default settings")
def step_impl_create_registry_default(context: Any) -> None:
context.registry = context.ProviderRegistry()
@when("I create a ProviderCapabilities with all fields")
def step_impl_capabilities_all_fields(context: Any) -> None:
context.capabilities = context.ProviderCapabilities(
supports_streaming=True,
supports_tool_calls=True,
supports_vision=True,
max_context_length=128000,
supports_json_mode=True,
)
@when("I create a ProviderCapabilities with default values")
def step_impl_capabilities_defaults(context: Any) -> None:
context.capabilities = context.ProviderCapabilities()
@when("I create a ProviderCapabilities instance")
def step_impl_capabilities_instance(context: Any) -> None:
context.capabilities = context.ProviderCapabilities()
@when("I create a ProviderInfo for openai")
def step_impl_provider_info_openai(context: Any) -> None:
"""Create a ProviderInfo for OpenAI."""
provider_enum = _ensure_provider_type(context)
context.provider_info = context.ProviderInfo(
provider_type=provider_enum.OPENAI,
name="Openai",
api_key_env_var="OPENAI_API_KEY",
default_model="gpt-4o",
)
@when("I call get_all_providers")
def step_impl_call_get_all(context: Any) -> None:
context.result = context.registry.get_all_providers()
@when("I call get_configured_providers")
def step_impl_call_get_configured(context: Any) -> None:
context.result = context.registry.get_configured_providers()
@when("I get provider info for ProviderType.OPENAI")
def step_impl_get_provider_info_enum(context: Any) -> None:
"""Get provider info for ProviderType.OPENAI."""
provider_enum = _ensure_provider_type(context)
context.result = context.registry.get_provider_info(provider_enum.OPENAI)
@when('I get provider info for string "{value}"')
def step_impl_get_provider_info_string(context: Any, value: str) -> None:
context.result = context.registry.get_provider_info(value)
@when("I check if openai is configured")
def step_impl_check_openai_configured(context: Any) -> None:
context.result = context.registry.is_provider_configured("openai")
@when("I call get_default_provider_type")
def step_impl_call_get_default_provider(context: Any) -> None:
context.result = context.registry.get_default_provider_type()
@when("I call get_default_model")
def step_impl_call_get_default_model(context: Any) -> None:
context.result = context.registry.get_default_model()
@when("I call get_default_model for ProviderType.OPENAI")
def step_impl_call_get_default_model_openai(context: Any) -> None:
"""Call get_default_model for OpenAI."""
provider_enum = _ensure_provider_type(context)
context.result = context.registry.get_default_model(provider_enum.OPENAI)
@when('I call get_default_model for string "{value}"')
def step_impl_call_get_default_model_string(context: Any, value: str) -> None:
context.result = context.registry.get_default_model(value)
@when("I get capabilities for ProviderType.OPENAI")
def step_impl_get_capabilities_openai(context: Any) -> None:
info = context.registry.get_provider_info(_ensure_provider_type(context).OPENAI)
context.capabilities = info.capabilities if info else None
@when("I get capabilities for ProviderType.ANTHROPIC")
def step_impl_get_capabilities_anthropic(context: Any) -> None:
info = context.registry.get_provider_info(_ensure_provider_type(context).ANTHROPIC)
context.capabilities = info.capabilities if info else None
@when("I get capabilities for ProviderType.GOOGLE")
def step_impl_get_capabilities_google(context: Any) -> None:
info = context.registry.get_provider_info(_ensure_provider_type(context).GOOGLE)
context.capabilities = info.capabilities if info else None
@when("I call get_provider_registry twice")
def step_impl_call_get_registry_twice(context: Any) -> None:
reset_provider_registry()
context.first_registry = get_provider_registry()
context.second_registry = get_provider_registry()
@when("I get the global registry and then reset it")
def step_impl_get_and_reset_registry(context: Any) -> None:
context.original_registry = get_provider_registry()
reset_provider_registry()
@when("I try to create an LLM without specifying a provider")
def step_impl_create_llm_no_provider(context: Any) -> None:
try:
context.registry.create_llm()
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when('I try to create an LLM for provider "{value}"')
def step_impl_create_llm_specific_provider(context: Any, value: str) -> None:
try:
context.registry.create_llm(provider_type=value)
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when('I call create_llm for provider "{value}" and model "{model}"')
def step_impl_call_create_llm_with_model(context: Any, value: str, model: str) -> None:
context.created_llm = context.registry.create_llm(
provider_type=value, model_id=model
)
@when("I call create_llm without specifying a provider")
def step_impl_call_create_llm_without_provider(context: Any) -> None:
context.created_llm = context.registry.create_llm()
@when("I call create_llm for ProviderType.{enum_name}")
def step_impl_call_create_llm_enum(context: Any, enum_name: str) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, enum_name.upper())
context.created_llm = context.registry.create_llm(provider_type=provider_type)
@when('I call create_llm for provider "{value}" without specifying a model')
def step_impl_call_create_llm_no_model(context: Any, value: str) -> None:
context.created_llm = context.registry.create_llm(provider_type=value)
@when('I call _create_provider_instance for provider "{value}" with model "{model}"')
def step_impl_call_private_instance(context: Any, value: str, model: str) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
context.created_llm = context.registry._create_provider_instance(
provider_type, model
)
@when(
'I call _create_provider_instance for provider "{value}" with model "{model}" and custom default headers'
)
def step_impl_call_private_instance_with_headers(
context: Any, value: str, model: str
) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
default_headers = getattr(context, "openrouter_default_headers", None)
assert default_headers is not None, "OpenRouter default headers were not configured"
context.created_llm = context.registry._create_provider_instance(
provider_type, model, default_headers=default_headers
)
@when(
'I call _create_provider_instance for provider "{value}" without specifying a model'
)
def step_impl_call_private_instance_no_model(context: Any, value: str) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
context.created_llm = context.registry._create_provider_instance(
provider_type, None
)
@when(
'I call _create_provider_instance for provider "{value}" with model "{model}" and empty default headers'
)
def step_impl_call_private_instance_empty_headers(
context: Any, value: str, model: str
) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
context.created_llm = context.registry._create_provider_instance(
provider_type, model, default_headers={}
)
@when(
'I call _create_provider_instance for provider "{value}" with model "{model}" and temperature {temperature}'
)
def step_impl_call_private_instance_with_temperature(
context: Any, value: str, model: str, temperature: str
) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
context.created_llm = context.registry._create_provider_instance(
provider_type, model, temperature=float(temperature)
)
@when(
'I try to call _create_provider_instance for provider "{value}" with model "{model}"'
)
def step_impl_try_private_instance(context: Any, value: str, model: str) -> None:
provider_enum = _ensure_provider_type(context)
provider_type = getattr(provider_enum, value.upper())
try:
context.registry._create_provider_instance(provider_type, model)
context.error = None
except Exception as exc: # pragma: no cover - defensive path
context.error = exc
@when("I try to create an AI provider without specifying a provider")
def step_impl_create_ai_provider_no_provider(context: Any) -> None:
try:
context.registry.create_ai_provider()
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when("I create an AI provider without specifying a provider")
def step_impl_create_ai_provider_default(context: Any) -> None:
context.ai_provider = context.registry.create_ai_provider()
@when('I try to create an AI provider for provider "{value}"')
def step_impl_create_ai_provider_invalid(context: Any, value: str) -> None:
try:
context.registry.create_ai_provider(provider_type=value)
context.error = None
except ValueError as exc: # pragma: no cover - negative path
context.error = exc
@when('I create an AI provider for provider "{value}"')
def step_impl_create_ai_provider_success(context: Any, value: str) -> None:
context.ai_provider = context.registry.create_ai_provider(provider_type=value)
@when('I create an AI provider specifying provider "{value}" and model "{model}"')
def step_impl_create_ai_provider_with_model(
context: Any, value: str, model: str
) -> None:
context.ai_provider = context.registry.create_ai_provider(
provider_type=value, model_id=model
)
@when('I call the AI provider llm factory with model "{model}"')
def step_impl_call_ai_provider_llm_factory(context: Any, model: str) -> None:
assert hasattr(context, "ai_provider"), "AI provider has not been created"
context.created_llm = context.ai_provider._llm_factory(model)
@when("I check the FALLBACK_ORDER")
def step_impl_check_fallback_order(context: Any) -> None:
context.fallback_order = context.ProviderRegistry.FALLBACK_ORDER
@when("I check the DEFAULT_MODELS")
def step_impl_check_default_models(context: Any) -> None:
context.default_models = context.ProviderRegistry.DEFAULT_MODELS
# -- Then -------------------------------------------------------------------
@then("the registry should be initialized")
def step_impl_registry_initialized(context: Any) -> None:
assert context.registry is not None
@then("the registry should have provider information")
def step_impl_registry_has_providers(context: Any) -> None:
assert len(context.registry.get_all_providers()) > 0
@then('ProviderType should have {enum_name} value "{value}"')
def step_impl_provider_type_values(context: Any, enum_name: str, value: str) -> None:
enum_member = getattr(_ensure_provider_type(context), enum_name.upper())
assert enum_member.value == value
@then("supports_streaming should be {expected}")
def step_impl_supports_streaming(context: Any, expected: str) -> None:
assert context.capabilities.supports_streaming is (expected == "True")
@then("supports_tool_calls should be {expected}")
def step_impl_supports_tool_calls(context: Any, expected: str) -> None:
assert context.capabilities.supports_tool_calls is (expected == "True")
@then("supports_vision should be {expected}")
def step_impl_supports_vision(context: Any, expected: str) -> None:
assert context.capabilities.supports_vision is (expected == "True")
@then("max_context_length should be {length:d}")
def step_impl_max_context_length(context: Any, length: int) -> None:
assert context.capabilities.max_context_length == length
@then("supports_json_mode should be {expected}")
def step_impl_supports_json_mode(context: Any, expected: str) -> None:
assert context.capabilities.supports_json_mode is (expected == "True")
@then("the provider_type should be ProviderType.OPENAI")
def step_impl_provider_type_is_openai(context: Any) -> None:
provider_enum = _ensure_provider_type(context)
info = _get_provider_info_from_context(context)
assert info.provider_type == provider_enum.OPENAI
@then("the provider_type should be ProviderType.ANTHROPIC")
def step_impl_provider_type_is_anthropic(context: Any) -> None:
provider_enum = _ensure_provider_type(context)
info = _get_provider_info_from_context(context)
assert info.provider_type == provider_enum.ANTHROPIC
@then('the provider info name should be "{value}"')
def step_impl_provider_info_name(context: Any, value: str) -> None:
assert context.provider_info.name == value
@then("the is_configured should be False by default")
def step_impl_provider_info_is_configured_default(context: Any) -> None:
assert context.provider_info.is_configured is False
@then("the provider registry result should be a list of ProviderInfo")
def step_impl_result_list_provider_info(context: Any) -> None:
assert isinstance(context.result, list)
assert all(isinstance(item, ProviderInfo) for item in context.result)
@then("the provider registry result should include core provider types")
def step_impl_result_contains_core_providers(context: Any) -> None:
provider_enum = _ensure_provider_type(context)
provider_types = {p.provider_type for p in context.result}
core_types = {provider_enum.OPENAI, provider_enum.ANTHROPIC, provider_enum.GOOGLE}
assert core_types.issubset(provider_types)
@then("the provider registry result should be an empty list")
def step_impl_result_empty_list(context: Any) -> None:
assert context.result == []
@then("the provider registry result should contain OpenAI provider")
def step_impl_result_contains_openai(context: Any) -> None:
provider_types = [p.provider_type for p in context.result]
assert _ensure_provider_type(context).OPENAI in provider_types
@then("the provider registry OpenAI entry should be configured")
def step_impl_openai_is_configured_true(context: Any) -> None:
openai_provider = next(
(
p
for p in context.result
if p.provider_type == _ensure_provider_type(context).OPENAI
),
None,
)
assert openai_provider is not None
assert openai_provider.is_configured is True
@then("the provider registry result should be a ProviderInfo instance")
def step_impl_result_is_provider_info(context: Any) -> None:
assert isinstance(context.result, ProviderInfo)
@then("the provider registry result should be None")
def step_impl_result_is_none(context: Any) -> None:
assert context.result is None, f"Expected None, got {context.result!r}"
@then("the provider registry result should be False")
def step_impl_result_false(context: Any) -> None:
assert context.result is False, f"Expected False, got {context.result!r}"
@then("the provider registry result should be True")
def step_impl_result_true(context: Any) -> None:
assert context.result is True, f"Expected True, got {context.result!r}"
@then("the provider registry result should be ProviderType.OPENAI")
def step_impl_result_provider_type_openai(context: Any) -> None:
assert context.result == _ensure_provider_type(context).OPENAI
@then("the provider registry result should be ProviderType.ANTHROPIC")
def step_impl_result_provider_type_anthropic(context: Any) -> None:
assert context.result == _ensure_provider_type(context).ANTHROPIC
@then('the provider registry result should be "{value}"')
def step_impl_result_string(context: Any, value: str) -> None:
assert context.result == value, f"Expected {value!r}, got {context.result!r}"
@then("both provider registry calls should return the same instance")
def step_impl_same_instance(context: Any) -> None:
assert context.first_registry is context.second_registry
@then("resetting the provider registry should return a new instance")
def step_impl_new_registry_instance(context: Any) -> None:
new_registry = get_provider_registry()
assert new_registry is not context.original_registry
@then("a provider registry ValueError should be raised")
def step_impl_value_error(context: Any) -> None:
assert context.error is not None
assert isinstance(context.error, ValueError)
@then("the provider registry error should mention configuring API keys")
def step_impl_error_mentions_config(context: Any) -> None:
assert "api" in str(context.error).lower() or "config" in str(context.error).lower()
@then("the provider registry error should mention unknown provider type")
def step_impl_error_mentions_unknown_provider(context: Any) -> None:
assert "unknown" in str(context.error).lower()
@then("the provider registry error should mention missing Azure endpoint")
def step_impl_error_mentions_missing_endpoint(context: Any) -> None:
assert context.error is not None
message = str(context.error).lower()
assert "azure" in message and "endpoint" in message
@then(
'the provider registry error should mention missing "{env_var}" environment variable'
)
def step_impl_error_mentions_missing_env(context: Any, env_var: str) -> None:
assert context.error is not None
message = str(context.error)
assert env_var in message, (
f"Expected error message to mention {env_var!r}, got {message!r}"
)
@then("the provider registry AI provider should be a LangChainChatProvider")
def step_impl_ai_provider_is_langchain(context: Any) -> None:
assert isinstance(context.ai_provider, LangChainChatProvider)
@then("the stubbed mock client should be instantiated")
def step_impl_stubbed_mock_client_instantiated(context: Any) -> None:
stubbed_clients = getattr(context, "stubbed_clients", {})
client_data = stubbed_clients.get("mock")
assert client_data, "No stubbed client recorded for mock"
calls = client_data["calls"]
assert calls, "Expected FakeListLLM to be instantiated"
assert isinstance(context.created_llm, client_data["class"])
@then('the provider registry AI provider name should be "{value}"')
def step_impl_ai_provider_name(context: Any, value: str) -> None:
assert context.ai_provider.name == value
@then('the provider registry AI provider model_id should be "{value}"')
def step_impl_ai_provider_model_id(context: Any, value: str) -> None:
assert context.ai_provider.model_id == value
@then(
'the stubbed LLM factory should be called for ProviderType.{enum_name} with model "{model}"'
)
def step_impl_stubbed_llm_factory_called(
context: Any, enum_name: str, model: str
) -> None:
provider_enum = _ensure_provider_type(context)
expected_type = getattr(provider_enum, enum_name.upper())
context.registry._create_provider_instance.assert_called_once()
args, kwargs = context.registry._create_provider_instance.call_args
assert args[:2] == (expected_type, model)
if kwargs:
# max_retries is propagated by create_ai_provider to keep retry semantics aligned
max_retries = kwargs.get("max_retries")
assert max_retries is None or isinstance(max_retries, int), (
f"Expected max_retries int or None, got {max_retries!r}"
)
# api_key should be forwarded (even None) so _get_api_key is not called twice
assert "__api_key_sentinel" in kwargs or "api_key" in kwargs, (
f"Expected api_key or __api_key_sentinel in factory kwargs, got {sorted(kwargs.keys())!r}"
)
assert context.created_llm is context.stubbed_llm
@then('the stubbed ChatOpenAI should be created with model "{model}"')
def step_impl_stubbed_chat_openai_created(context: Any, model: str) -> None:
assert getattr(context, "chat_openai_calls", []), "Expected ChatOpenAI to be called"
last_call = context.chat_openai_calls[-1]
recorded_model = last_call.get("model")
if recorded_model is None:
recorded_model = last_call.get("kwargs", {}).get("model")
assert recorded_model == model
assert isinstance(context.created_llm, context.fake_chat_openai_class)
@then(
'the stubbed {provider_name} client should be created with argument "{arg_name}" value "{expected}"'
)
def step_impl_stubbed_client_argument(
context: Any, provider_name: str, arg_name: str, expected: str
) -> None:
provider_key = provider_name.lower()
stubbed_clients = getattr(context, "stubbed_clients", {})
assert provider_key in stubbed_clients, (
f"No stubbed client recorded for {provider_name}"
)
client_data = stubbed_clients[provider_key]
calls = client_data["calls"]
assert calls, f"Expected {provider_name} client to be instantiated"
last_call = calls[-1]
actual = last_call["kwargs"].get(arg_name)
if actual != expected:
try:
expected_num = float(expected)
assert actual == expected_num, (
f"Expected {arg_name}={expected!r} or {expected_num!r}, got {actual!r}"
)
except ValueError:
raise AssertionError(
f"Expected {arg_name}={expected!r}, got {actual!r}"
) from None
assert isinstance(context.created_llm, client_data["class"])
@then(
'the stubbed OpenRouter client headers should include "{header_name}" value "{expected}"'
)
def step_impl_stubbed_openrouter_headers(
context: Any, header_name: str, expected: str
) -> None:
stubbed_clients = getattr(context, "stubbed_clients", {})
client_data = stubbed_clients.get("openrouter")
assert client_data, "No stubbed client recorded for OpenRouter"
calls = client_data["calls"]
assert calls, "Expected OpenRouter client to be instantiated"
headers = calls[-1]["kwargs"].get("default_headers")
assert isinstance(headers, dict), "Expected default_headers to be a dict"
assert headers.get(header_name) == expected, (
f"Expected {header_name}={expected!r}, got {headers.get(header_name)!r}"
)
@then("the stubbed OpenRouter client headers should not include key {key}")
def step_impl_stubbed_openrouter_headers_not_include_key(
context: Any, key: str
) -> None:
stubbed_clients = getattr(context, "stubbed_clients", {})
client_data = stubbed_clients.get("openrouter")
assert client_data, "No stubbed client recorded for OpenRouter"
calls = client_data["calls"]
assert calls, "Expected OpenRouter client to be instantiated"
headers = calls[-1]["kwargs"].get("default_headers")
assert isinstance(headers, dict), "Expected default_headers to be a dict"
# Check both integer and string representations of the key
try:
key_int = int(key)
except ValueError:
key_int = key
assert key_int not in headers, (
f"Expected key {key_int!r} to be absent from headers, but it was present"
)
@then("the stubbed {provider_name} client should not have default_headers")
def step_impl_stubbed_client_no_default_headers(
context: Any, provider_name: str
) -> None:
provider_key = provider_name.lower()
stubbed_clients = getattr(context, "stubbed_clients", {})
assert provider_key in stubbed_clients, (
f"No stubbed client recorded for {provider_name}"
)
client_data = stubbed_clients[provider_key]
calls = client_data["calls"]
assert calls, f"Expected {provider_name} client to be instantiated"
last_call = calls[-1]
assert "default_headers" not in last_call["kwargs"], (
f"Expected default_headers to be absent, got {last_call['kwargs'].get('default_headers')!r}"
)
@then("the instance should be frozen")
def step_impl_instance_frozen(context: Any) -> None:
try:
context.capabilities.supports_streaming = False
raise AssertionError("Expected FrozenInstanceError")
except FrozenInstanceError:
context.frozen_error = True
@then("attempting to modify it should raise an error")
def step_impl_modify_raises_error(context: Any) -> None:
assert getattr(context, "frozen_error", False) is True
@then("OpenAI should be first in the order")
def step_impl_openai_first(context: Any) -> None:
assert context.fallback_order[0] == _ensure_provider_type(context).OPENAI
@then("Anthropic should be second in the order")
def step_impl_anthropic_second(context: Any) -> None:
assert context.fallback_order[1] == _ensure_provider_type(context).ANTHROPIC
@then("Google should be third in the order")
def step_impl_google_third(context: Any) -> None:
assert context.fallback_order[2] == _ensure_provider_type(context).GOOGLE
@then('{provider} default should be "{model}"')
def step_impl_default_models(context: Any, provider: str, model: str) -> None:
provider_enum = _ensure_provider_type(context)
enum_member = getattr(provider_enum, provider.upper())
assert context.default_models[enum_member] == model
@then('the provider registry error should mention "{message}"')
def step_impl_error_mentions_message(context: Any, message: str) -> None:
assert context.error is not None
assert message.lower() in str(context.error).lower(), (
f"Expected error to mention {message!r}, got {str(context.error)!r}"
)
@then("the created LLM should be a FakeListLLM instance")
def step_impl_created_llm_is_fake_list_llm(context: Any) -> None:
from langchain_community.llms import FakeListLLM
assert isinstance(context.created_llm, FakeListLLM), (
f"Expected FakeListLLM, got {type(context.created_llm)}"
)
@then("the provider registry result should contain only the MOCK provider")
def step_impl_result_contains_only_mock(context: Any) -> None:
provider_types = [p.provider_type for p in context.result]
provider_enum = _ensure_provider_type(context)
assert provider_types == [provider_enum.MOCK], (
f"Expected only MOCK, got {provider_types}"
)
@then("the provider registry result should contain the MOCK provider")
def step_impl_result_contains_mock(context: Any) -> None:
provider_types = [p.provider_type for p in context.result]
provider_enum = _ensure_provider_type(context)
assert provider_enum.MOCK in provider_types, (
f"Expected MOCK in list, got {provider_types}"
)
@then("the stubbed {provider_name} client responses should be {expected_responses}")
def step_impl_stubbed_client_responses(
context: Any, provider_name: str, expected_responses: str
) -> None:
provider_key = provider_name.lower()
stubbed_clients = getattr(context, "stubbed_clients", {})
assert provider_key in stubbed_clients, (
f"No stubbed client recorded for {provider_name}"
)
client_data = stubbed_clients[provider_key]
calls = client_data["calls"]
assert calls, f"Expected {provider_name} client to be instantiated"
last_call = calls[-1]
actual = last_call["kwargs"].get("responses")
if actual is None:
actual = last_call.get("args")
expected = eval(expected_responses) # e.g. ["mock response"] * 10
assert actual == expected, f"Expected responses={expected!r}, got {actual!r}"
assert isinstance(context.created_llm, client_data["class"])
@then("the stubbed LLM factory should be called with api_key present")
def step_impl_stubbed_llm_factory_api_key(context: Any) -> None:
context.registry._create_provider_instance.assert_called()
_args, kwargs = context.registry._create_provider_instance.call_args
assert "__api_key_sentinel" in kwargs or "api_key" in kwargs, (
f"Expected api_key or __api_key_sentinel in factory kwargs, got {sorted(kwargs.keys())!r}"
)
@then('the FakeListLLM produces response "{response}" on invoke')
def step_impl_fake_list_llm_invokes(context: Any, response: str) -> None:
from langchain_community.llms import FakeListLLM
llm = context.created_llm
assert isinstance(llm, FakeListLLM), f"Expected FakeListLLM, got {type(llm)}"
assert llm.invoke("test prompt") == response, (
f"Expected FakeListLLM to produce {response!r}"
)
@when('I call resolve_provider_by_name with "{name}"')
def step_impl_resolve_provider_by_name(context: Any, name: str) -> None:
from cleveragents.providers.registry import (
resolve_provider_by_name,
reset_provider_registry,
)
reset_provider_registry()
try:
context.resolved_provider = resolve_provider_by_name(name)
context.error = None
except ValueError as exc:
context.error = exc
@then("the resolved provider should be a LangChainChatProvider")
def step_impl_resolved_provider_is_langchain(context: Any) -> None:
assert isinstance(context.resolved_provider, LangChainChatProvider)
# Hook logic lives in features/environment.py to ensure scenarios reset state.