HAL9000
58c607fd79
The openrouter_provider_registry_steps.py file called use_step_matcher("re")
at module level without restoring the default "parse" matcher at the end.
This caused all step files loaded alphabetically after it (provider_registry_*,
resource_*, session_*, etc.) to use the regex matcher instead of the parse
matcher, breaking their {value}-style step patterns and causing unit_tests CI
failures.
Add use_step_matcher("parse") at the end of the file to restore the default
matcher after the openrouter step definitions are registered.
380 lines
13 KiB
Python
380 lines
13 KiB
Python
"""Step definitions for openrouter_provider_registry.feature.
|
|
|
|
Tests that ProviderRegistry.create_llm and _create_provider_llm correctly
|
|
dispatch to the OpenRouter ChatOpenAI backend without raising ValueError.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import types
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
from behave import given, then, use_step_matcher, when # type: ignore[import-untyped]
|
|
|
|
from cleveragents.providers.registry import (
|
|
ProviderRegistry,
|
|
ProviderType,
|
|
)
|
|
|
|
_OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_or_settings(
|
|
openrouter: str | None = None,
|
|
organization: str | None = None,
|
|
) -> MagicMock:
|
|
"""Return a minimal fake Settings object for OpenRouter tests."""
|
|
settings = MagicMock()
|
|
settings.openai_api_key = None
|
|
settings.anthropic_api_key = None
|
|
settings.google_api_key = None
|
|
settings.gemini_api_key = None
|
|
settings.azure_api_key = None
|
|
settings.openrouter_api_key = openrouter
|
|
settings.cohere_api_key = None
|
|
settings.groq_api_key = None
|
|
settings.together_api_key = None
|
|
settings.default_provider = None
|
|
settings.default_model = None
|
|
settings.azure_openai_endpoint = None
|
|
settings.azure_openai_api_version = None
|
|
settings.azure_openai_deployment = None
|
|
settings.openrouter_organization = organization
|
|
return settings
|
|
|
|
|
|
def _stub_chat_openai(context: Any) -> None:
|
|
"""Replace langchain_openai.ChatOpenAI with a recording stub."""
|
|
original = sys.modules.get("langchain_openai")
|
|
calls: list[dict[str, Any]] = []
|
|
|
|
class FakeChatOpenAI:
|
|
def __init__(self, **kwargs: Any) -> None:
|
|
calls.append(kwargs)
|
|
|
|
class FakeAzureChatOpenAI:
|
|
def __init__(self, **kwargs: Any) -> None:
|
|
pass
|
|
|
|
stub = types.ModuleType("langchain_openai")
|
|
stub.ChatOpenAI = FakeChatOpenAI # type: ignore[attr-defined]
|
|
stub.AzureChatOpenAI = FakeAzureChatOpenAI # type: ignore[attr-defined]
|
|
sys.modules["langchain_openai"] = stub
|
|
|
|
context._or_chat_openai_calls = calls
|
|
context._or_chat_openai_cls = FakeChatOpenAI
|
|
|
|
if not hasattr(context, "_or_cleanup"):
|
|
context._or_cleanup = []
|
|
|
|
def _restore() -> None:
|
|
if original is None:
|
|
sys.modules.pop("langchain_openai", None)
|
|
else:
|
|
sys.modules["langchain_openai"] = original
|
|
|
|
context._or_cleanup.append(_restore)
|
|
|
|
|
|
def _parse_headers_string(headers_string: str) -> dict[str, str]:
|
|
"""Parse 'Key=Value,Key2=Value2' into a dict."""
|
|
result: dict[str, str] = {}
|
|
for entry in headers_string.split(","):
|
|
entry = entry.strip()
|
|
if "=" in entry:
|
|
key, value = entry.split("=", 1)
|
|
result[key.strip()] = value.strip()
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given
|
|
# ---------------------------------------------------------------------------
|
|
|
|
use_step_matcher("re")
|
|
|
|
|
|
@given(
|
|
r'I have an openrouter registry with API key "(?P<or_api_key>[^"]+)" and organization "(?P<or_org>[^"]+)"'
|
|
)
|
|
def step_or_registry_with_key_and_org(
|
|
context: Any, or_api_key: str, or_org: str
|
|
) -> None:
|
|
context.or_registry = ProviderRegistry(
|
|
settings=_make_or_settings(openrouter=or_api_key, organization=or_org)
|
|
)
|
|
|
|
|
|
@given(r'I have an openrouter registry with API key "(?P<or_api_key>[^"]+)"')
|
|
def step_or_registry_with_key(context: Any, or_api_key: str) -> None:
|
|
context.or_registry = ProviderRegistry(
|
|
settings=_make_or_settings(openrouter=or_api_key)
|
|
)
|
|
|
|
|
|
@given(r"I have an openrouter registry with no API keys")
|
|
def step_or_registry_no_keys(context: Any) -> None:
|
|
context.or_registry = ProviderRegistry(settings=_make_or_settings())
|
|
|
|
|
|
@given(r"I have the openrouter ProviderRegistry class")
|
|
def step_or_registry_class(context: Any) -> None:
|
|
context.or_registry_class = ProviderRegistry
|
|
|
|
|
|
@given(r"the openrouter registry LangChain ChatOpenAI client is stubbed")
|
|
def step_or_stub_chat_openai(context: Any) -> None:
|
|
_stub_chat_openai(context)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when(
|
|
r'I call create_llm on the openrouter registry for provider "openrouter" with model "(?P<or_model>[^"]+)"'
|
|
)
|
|
def step_or_create_llm_with_model(context: Any, or_model: str) -> None:
|
|
context.or_error = None
|
|
try:
|
|
context.or_result = context.or_registry.create_llm(
|
|
provider_type="openrouter", model_id=or_model
|
|
)
|
|
except ValueError as exc:
|
|
context.or_error = exc
|
|
|
|
|
|
@when(r"I call create_llm on the openrouter registry without specifying a provider")
|
|
def step_or_create_llm_default(context: Any) -> None:
|
|
context.or_error = None
|
|
try:
|
|
context.or_result = context.or_registry.create_llm()
|
|
except ValueError as exc:
|
|
context.or_error = exc
|
|
|
|
|
|
@when(
|
|
r'I call _create_provider_llm on the openrouter registry for provider "openrouter" with model None'
|
|
)
|
|
def step_or_private_llm_none_model(context: Any) -> None:
|
|
context.or_error = None
|
|
try:
|
|
context.or_result = context.or_registry._create_provider_llm(
|
|
ProviderType.OPENROUTER, None
|
|
)
|
|
except ValueError as exc:
|
|
context.or_error = exc
|
|
|
|
|
|
@when(
|
|
r'I call _create_provider_llm on the openrouter registry for provider "openrouter" with model "(?P<or_model>[^"]+)" and default_headers "(?P<or_headers_string>[^"]+)"'
|
|
)
|
|
def step_or_private_llm_with_headers(
|
|
context: Any, or_model: str, or_headers_string: str
|
|
) -> None:
|
|
headers = _parse_headers_string(or_headers_string)
|
|
context.or_error = None
|
|
try:
|
|
context.or_result = context.or_registry._create_provider_llm(
|
|
ProviderType.OPENROUTER, or_model, default_headers=headers
|
|
)
|
|
except ValueError as exc:
|
|
context.or_error = exc
|
|
|
|
|
|
@when(
|
|
r'I call _create_provider_llm on the openrouter registry for provider "openrouter" with model "(?P<or_model>[^"]+)"'
|
|
)
|
|
def step_or_private_llm_with_model(context: Any, or_model: str) -> None:
|
|
context.or_error = None
|
|
try:
|
|
context.or_result = context.or_registry._create_provider_llm(
|
|
ProviderType.OPENROUTER, or_model
|
|
)
|
|
except ValueError as exc:
|
|
context.or_error = exc
|
|
|
|
|
|
@when(r'I try to call create_llm on the openrouter registry for provider "openrouter"')
|
|
def step_or_try_create_llm_no_key(context: Any) -> None:
|
|
context.or_error = None
|
|
try:
|
|
context.or_registry.create_llm(provider_type="openrouter")
|
|
except ValueError as exc:
|
|
context.or_error = exc
|
|
|
|
|
|
@when(r"I check the openrouter FALLBACK_ORDER")
|
|
def step_or_check_fallback_order(context: Any) -> None:
|
|
context.or_fallback_order = context.or_registry_class.FALLBACK_ORDER
|
|
|
|
|
|
@when(r"I check the openrouter DEFAULT_MODELS")
|
|
def step_or_check_default_models(context: Any) -> None:
|
|
context.or_default_models = context.or_registry_class.DEFAULT_MODELS
|
|
|
|
|
|
@when(r"I check the openrouter DEFAULT_CAPABILITIES for OPENROUTER")
|
|
def step_or_check_capabilities(context: Any) -> None:
|
|
context.or_capabilities = context.or_registry_class.DEFAULT_CAPABILITIES.get(
|
|
ProviderType.OPENROUTER
|
|
)
|
|
|
|
|
|
@when(r"I call get_provider_info for OPENROUTER on the openrouter registry")
|
|
def step_or_get_provider_info(context: Any) -> None:
|
|
context.or_provider_info = context.or_registry.get_provider_info(
|
|
ProviderType.OPENROUTER
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then(
|
|
r'the openrouter registry ChatOpenAI should be called with model "(?P<or_expected_model>[^"]+)"'
|
|
)
|
|
def step_or_assert_model(context: Any, or_expected_model: str) -> None:
|
|
calls = getattr(context, "_or_chat_openai_calls", [])
|
|
assert calls, "Expected ChatOpenAI to be instantiated"
|
|
actual_model = calls[-1].get("model")
|
|
assert actual_model == or_expected_model, (
|
|
f"Expected model={or_expected_model!r}, got {actual_model!r}"
|
|
)
|
|
|
|
|
|
@then(
|
|
r'the openrouter registry ChatOpenAI should use base url "(?P<or_expected_url>[^"]+)"'
|
|
)
|
|
def step_or_assert_base_url(context: Any, or_expected_url: str) -> None:
|
|
calls = getattr(context, "_or_chat_openai_calls", [])
|
|
assert calls, "Expected ChatOpenAI to be instantiated"
|
|
actual_url = calls[-1].get("openai_api_base")
|
|
assert actual_url == or_expected_url, (
|
|
f"Expected openai_api_base={or_expected_url!r}, got {actual_url!r}"
|
|
)
|
|
|
|
|
|
@then(
|
|
r'the openrouter registry ChatOpenAI should use api key "(?P<or_expected_key>[^"]+)"'
|
|
)
|
|
def step_or_assert_api_key(context: Any, or_expected_key: str) -> None:
|
|
calls = getattr(context, "_or_chat_openai_calls", [])
|
|
assert calls, "Expected ChatOpenAI to be instantiated"
|
|
actual_key = calls[-1].get("openai_api_key")
|
|
assert actual_key == or_expected_key, (
|
|
f"Expected openai_api_key={or_expected_key!r}, got {actual_key!r}"
|
|
)
|
|
|
|
|
|
@then(
|
|
r'the openrouter registry ChatOpenAI should include header "(?P<or_header_name>[^"]+)" with value "(?P<or_expected_value>[^"]+)"'
|
|
)
|
|
def step_or_assert_header(
|
|
context: Any, or_header_name: str, or_expected_value: str
|
|
) -> None:
|
|
calls = getattr(context, "_or_chat_openai_calls", [])
|
|
assert calls, "Expected ChatOpenAI to be instantiated"
|
|
headers = calls[-1].get("default_headers")
|
|
assert isinstance(headers, dict), (
|
|
f"Expected default_headers to be a dict, got {type(headers)}"
|
|
)
|
|
actual_value = headers.get(or_header_name)
|
|
assert actual_value == or_expected_value, (
|
|
f"Expected header {or_header_name}={or_expected_value!r}, got {actual_value!r}"
|
|
)
|
|
|
|
|
|
@then(r"an openrouter registry ValueError should be raised")
|
|
def step_or_assert_value_error(context: Any) -> None:
|
|
assert context.or_error is not None, "Expected a ValueError to be raised"
|
|
assert isinstance(context.or_error, ValueError), (
|
|
f"Expected ValueError, got {type(context.or_error).__name__}"
|
|
)
|
|
|
|
|
|
@then(r'the openrouter registry error should mention "(?P<or_fragment>[^"]+)"')
|
|
def step_or_assert_error_contains(context: Any, or_fragment: str) -> None:
|
|
assert context.or_error is not None, "Expected an error to be set"
|
|
message = str(context.or_error)
|
|
assert or_fragment in message, (
|
|
f"Expected error to contain {or_fragment!r}, got: {message!r}"
|
|
)
|
|
|
|
|
|
@then(r"ProviderType\.OPENROUTER should be in the fallback order")
|
|
def step_or_assert_in_fallback(context: Any) -> None:
|
|
assert ProviderType.OPENROUTER in context.or_fallback_order, (
|
|
"Expected ProviderType.OPENROUTER to be in FALLBACK_ORDER"
|
|
)
|
|
|
|
|
|
@then(r'the openrouter default model should be "(?P<or_expected_model>[^"]+)"')
|
|
def step_or_assert_default_model(context: Any, or_expected_model: str) -> None:
|
|
actual = context.or_default_models.get(ProviderType.OPENROUTER)
|
|
assert actual == or_expected_model, (
|
|
f"Expected default model={or_expected_model!r}, got {actual!r}"
|
|
)
|
|
|
|
|
|
@then(r"the openrouter provider supports_streaming should be (?P<or_expected>\w+)")
|
|
def step_or_assert_streaming(context: Any, or_expected: str) -> None:
|
|
assert context.or_capabilities is not None
|
|
assert context.or_capabilities.supports_streaming is (or_expected == "True")
|
|
|
|
|
|
@then(r"the openrouter provider supports_tool_calls should be (?P<or_expected>\w+)")
|
|
def step_or_assert_tool_calls(context: Any, or_expected: str) -> None:
|
|
assert context.or_capabilities is not None
|
|
assert context.or_capabilities.supports_tool_calls is (or_expected == "True")
|
|
|
|
|
|
@then(r"the openrouter provider supports_vision should be (?P<or_expected>\w+)")
|
|
def step_or_assert_vision(context: Any, or_expected: str) -> None:
|
|
assert context.or_capabilities is not None
|
|
assert context.or_capabilities.supports_vision is (or_expected == "True")
|
|
|
|
|
|
@then(r"the openrouter provider max_context_length should be (?P<or_expected>\d+)")
|
|
def step_or_assert_context_length(context: Any, or_expected: str) -> None:
|
|
assert context.or_capabilities is not None
|
|
expected_int = int(or_expected)
|
|
assert context.or_capabilities.max_context_length == expected_int, (
|
|
f"Expected max_context_length={expected_int}, got {context.or_capabilities.max_context_length}"
|
|
)
|
|
|
|
|
|
@then(r"the openrouter provider supports_json_mode should be (?P<or_expected>\w+)")
|
|
def step_or_assert_json_mode(context: Any, or_expected: str) -> None:
|
|
assert context.or_capabilities is not None
|
|
assert context.or_capabilities.supports_json_mode is (or_expected == "True")
|
|
|
|
|
|
@then(r"the openrouter provider info should be configured")
|
|
def step_or_assert_provider_configured(context: Any) -> None:
|
|
info = context.or_provider_info
|
|
assert info is not None, "Expected provider info to exist"
|
|
assert info.is_configured is True, "Expected provider to be configured"
|
|
|
|
|
|
@then(r"the openrouter provider info should not be configured")
|
|
def step_or_assert_provider_not_configured(context: Any) -> None:
|
|
info = context.or_provider_info
|
|
assert info is not None, "Expected provider info to exist"
|
|
assert info.is_configured is False, "Expected provider to not be configured"
|
|
|
|
|
|
# Restore the default step matcher so that step files loaded after this one
|
|
# are not affected by the regex matcher set above.
|
|
use_step_matcher("parse")
|