diff --git a/docs/reference/providers.md b/docs/reference/providers.md new file mode 100644 index 000000000..2d542ba29 --- /dev/null +++ b/docs/reference/providers.md @@ -0,0 +1,120 @@ +# Provider Configuration Reference + +CleverAgents supports multiple AI provider backends through the `ProviderRegistry`. +Each provider is configured via environment variables or the `agents config set` command. + +## Supported Providers + +| Provider | `ProviderType` | API Key Environment Variable | +|----------|---------------|------------------------------| +| OpenAI | `openai` | `OPENAI_API_KEY` | +| Anthropic | `anthropic` | `ANTHROPIC_API_KEY` | +| Google / Gemini | `google` / `gemini` | `GOOGLE_API_KEY` / `GEMINI_API_KEY` | +| Azure OpenAI | `azure` | `AZURE_OPENAI_API_KEY` | +| **OpenRouter** | **`openrouter`** | **`OPENROUTER_API_KEY`** | +| Groq | `groq` | `GROQ_API_KEY` | +| Together AI | `together` | `TOGETHER_API_KEY` | +| Cohere | `cohere` | `COHERE_API_KEY` | + +--- + +## OpenRouter + +[OpenRouter](https://openrouter.ai) provides a unified API gateway to hundreds of +models from different providers (Anthropic, OpenAI, Google, Meta, Mistral, and more). +This makes it ideal for cost optimisation and model diversity without managing +multiple API keys. + +### Configuration + +Set the following environment variables: + +```bash +# Required +export OPENROUTER_API_KEY="sk-or-v1-..." + +# Optional: default model (defaults to anthropic/claude-sonnet-4-20250514) +export CLEVERAGENTS_DEFAULT_MODEL="openai/gpt-4o" + +# Optional: set OpenRouter as the default provider +export CLEVERAGENTS_DEFAULT_PROVIDER="openrouter" + +# Optional: organisation identifier sent in HTTP headers +export OPENROUTER_ORGANIZATION="myapp.example.com" +``` + +Or use the CLI: + +```bash +agents config set provider.openrouter.api_key "sk-or-v1-..." +agents config set provider.openrouter.model "anthropic/claude-3-haiku" +``` + +### Config Keys + +| Config Key | Environment Variable | Description | +|------------|---------------------|-------------| +| `provider.openrouter.api_key` | `OPENROUTER_API_KEY` | OpenRouter API key (required) | +| `provider.openrouter.model` | `CLEVERAGENTS_DEFAULT_MODEL` | Default model slug (optional) | +| `provider.openrouter.organization` | `OPENROUTER_ORGANIZATION` | Organisation name sent in `HTTP-Referer` / `X-Title` headers (optional) | + +### Capabilities + +| Feature | Supported | +|---------|-----------| +| Streaming | ✅ Yes | +| Tool calls | ✅ Yes | +| Vision / image input | ✅ Yes (model-dependent) | +| JSON mode | ✅ Yes (model-dependent) | +| Max context length | 128 000 tokens (varies by model) | + +### Selecting a Model + +OpenRouter model slugs follow the format `/`. Examples: + +```bash +# Anthropic via OpenRouter +agents config set provider.openrouter.model "anthropic/claude-sonnet-4-20250514" + +# OpenAI via OpenRouter +agents config set provider.openrouter.model "openai/gpt-4o" + +# Meta Llama via OpenRouter +agents config set provider.openrouter.model "meta-llama/llama-3.1-70b-instruct" +``` + +Browse the full model catalogue at . + +### Fallback Priority + +When no explicit provider is configured, CleverAgents selects the first available +provider in the following fallback order: + +1. OpenAI +2. Anthropic +3. Google +4. Azure +5. **OpenRouter** ← position 5 +6. Groq +7. Together AI +8. Cohere + +### Example Usage + +```python +from cleveragents.providers.registry import ProviderRegistry, ProviderType + +registry = ProviderRegistry() + +# Create a LangChain LLM directly +llm = registry.create_llm( + provider_type=ProviderType.OPENROUTER, + model_id="anthropic/claude-3-haiku", +) + +# Or create a full AIProviderInterface adapter +provider = registry.create_ai_provider( + provider_type="openrouter", + model_id="openai/gpt-4o", +) +``` diff --git a/features/openrouter_provider_registry.feature b/features/openrouter_provider_registry.feature new file mode 100644 index 000000000..eb9c9cd2d --- /dev/null +++ b/features/openrouter_provider_registry.feature @@ -0,0 +1,87 @@ +Feature: OpenRouter provider support in ProviderRegistry + As a developer using CleverAgents + I want ProviderRegistry to handle ProviderType.OPENROUTER in create_llm and _create_provider_llm + So that OpenRouter models are accessible without raising ValueError + + # Covers the core acceptance criterion: + # ProviderRegistry.create_llm handles ProviderType.OPENROUTER without raising ValueError + + @unit @providers @openrouter @registry + Scenario: create_llm dispatches to OpenRouter via _create_provider_llm + Given I have an openrouter registry with API key "sk-or-test-key" + And the openrouter registry LangChain ChatOpenAI client is stubbed + When I call create_llm on the openrouter registry for provider "openrouter" with model "anthropic/claude-3-haiku" + Then the openrouter registry ChatOpenAI should be called with model "anthropic/claude-3-haiku" + And the openrouter registry ChatOpenAI should use base url "https://openrouter.ai/api/v1" + And the openrouter registry ChatOpenAI should use api key "sk-or-test-key" + + @unit @providers @openrouter @registry + Scenario: _create_provider_llm builds ChatOpenAI for OpenRouter with default model + Given I have an openrouter registry with API key "sk-or-default" + And the openrouter registry LangChain ChatOpenAI client is stubbed + When I call _create_provider_llm on the openrouter registry for provider "openrouter" with model None + Then the openrouter registry ChatOpenAI should be called with model "anthropic/claude-sonnet-4-20250514" + And the openrouter registry ChatOpenAI should use base url "https://openrouter.ai/api/v1" + + @unit @providers @openrouter @registry + Scenario: _create_provider_llm injects organization headers when configured + Given I have an openrouter registry with API key "sk-or-org" and organization "myapp.example.com" + And the openrouter registry LangChain ChatOpenAI client is stubbed + When I call _create_provider_llm on the openrouter registry for provider "openrouter" with model "openai/gpt-4o" + Then the openrouter registry ChatOpenAI should include header "HTTP-Referer" with value "myapp.example.com" + And the openrouter registry ChatOpenAI should include header "X-Title" with value "myapp.example.com" + + @unit @providers @openrouter @registry + Scenario: _create_provider_llm passes custom default_headers through + Given I have an openrouter registry with API key "sk-or-headers" + And the openrouter registry LangChain ChatOpenAI client is stubbed + When I call _create_provider_llm on the openrouter registry for provider "openrouter" with model "openai/gpt-4o" and default_headers "X-Custom=trace-123" + Then the openrouter registry ChatOpenAI should include header "X-Custom" with value "trace-123" + + @unit @providers @openrouter @registry + Scenario: create_llm raises ValueError when OpenRouter API key is missing + Given I have an openrouter registry with no API keys + When I try to call create_llm on the openrouter registry for provider "openrouter" + Then an openrouter registry ValueError should be raised + And the openrouter registry error should mention "OPENROUTER_API_KEY" + + @unit @providers @openrouter @registry + Scenario: ProviderType.OPENROUTER is in the registry fallback order + Given I have the openrouter ProviderRegistry class + When I check the openrouter FALLBACK_ORDER + Then ProviderType.OPENROUTER should be in the fallback order + + @unit @providers @openrouter @registry + Scenario: OpenRouter default model is configured correctly + Given I have the openrouter ProviderRegistry class + When I check the openrouter DEFAULT_MODELS + Then the openrouter default model should be "anthropic/claude-sonnet-4-20250514" + + @unit @providers @openrouter @registry + Scenario: OpenRouter capabilities are configured correctly + Given I have the openrouter ProviderRegistry class + When I check the openrouter DEFAULT_CAPABILITIES for OPENROUTER + Then the openrouter provider supports_streaming should be True + And the openrouter provider supports_tool_calls should be True + And the openrouter provider supports_vision should be True + And the openrouter provider max_context_length should be 128000 + And the openrouter provider supports_json_mode should be True + + @unit @providers @openrouter @registry + Scenario: OpenRouter provider is discovered when API key is set + Given I have an openrouter registry with API key "sk-or-discover" + When I call get_provider_info for OPENROUTER on the openrouter registry + Then the openrouter provider info should be configured + + @unit @providers @openrouter @registry + Scenario: OpenRouter provider is not discovered when API key is missing + Given I have an openrouter registry with no API keys + When I call get_provider_info for OPENROUTER on the openrouter registry + Then the openrouter provider info should not be configured + + @unit @providers @openrouter @registry + Scenario: create_llm selects OpenRouter as default when it is the only configured provider + Given I have an openrouter registry with API key "sk-or-only" + And the openrouter registry LangChain ChatOpenAI client is stubbed + When I call create_llm on the openrouter registry without specifying a provider + Then the openrouter registry ChatOpenAI should be called with model "anthropic/claude-sonnet-4-20250514" diff --git a/features/steps/openrouter_provider_registry_steps.py b/features/steps/openrouter_provider_registry_steps.py new file mode 100644 index 000000000..854a48936 --- /dev/null +++ b/features/steps/openrouter_provider_registry_steps.py @@ -0,0 +1,376 @@ +"""Step definitions for openrouter_provider_registry.feature. + +Tests that ProviderRegistry.create_llm and _create_provider_llm correctly +dispatch to the OpenRouter ChatOpenAI backend without raising ValueError. +""" + +from __future__ import annotations + +import sys +import types +from typing import Any +from unittest.mock import MagicMock + +from behave import given, then, use_step_matcher, when # type: ignore[import-untyped] + +from cleveragents.providers.registry import ( + ProviderRegistry, + ProviderType, +) + +_OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_or_settings( + openrouter: str | None = None, + organization: str | None = None, +) -> MagicMock: + """Return a minimal fake Settings object for OpenRouter tests.""" + settings = MagicMock() + settings.openai_api_key = None + settings.anthropic_api_key = None + settings.google_api_key = None + settings.gemini_api_key = None + settings.azure_api_key = None + settings.openrouter_api_key = openrouter + settings.cohere_api_key = None + settings.groq_api_key = None + settings.together_api_key = None + settings.default_provider = None + settings.default_model = None + settings.azure_openai_endpoint = None + settings.azure_openai_api_version = None + settings.azure_openai_deployment = None + settings.openrouter_organization = organization + return settings + + +def _stub_chat_openai(context: Any) -> None: + """Replace langchain_openai.ChatOpenAI with a recording stub.""" + original = sys.modules.get("langchain_openai") + calls: list[dict[str, Any]] = [] + + class FakeChatOpenAI: + def __init__(self, **kwargs: Any) -> None: + calls.append(kwargs) + + class FakeAzureChatOpenAI: + def __init__(self, **kwargs: Any) -> None: + pass + + stub = types.ModuleType("langchain_openai") + stub.ChatOpenAI = FakeChatOpenAI # type: ignore[attr-defined] + stub.AzureChatOpenAI = FakeAzureChatOpenAI # type: ignore[attr-defined] + sys.modules["langchain_openai"] = stub + + context._or_chat_openai_calls = calls + context._or_chat_openai_cls = FakeChatOpenAI + + if not hasattr(context, "_or_cleanup"): + context._or_cleanup = [] + + def _restore() -> None: + if original is None: + sys.modules.pop("langchain_openai", None) + else: + sys.modules["langchain_openai"] = original + + context._or_cleanup.append(_restore) + + +def _parse_headers_string(headers_string: str) -> dict[str, str]: + """Parse 'Key=Value,Key2=Value2' into a dict.""" + result: dict[str, str] = {} + for entry in headers_string.split(","): + entry = entry.strip() + if "=" in entry: + key, value = entry.split("=", 1) + result[key.strip()] = value.strip() + return result + + +# --------------------------------------------------------------------------- +# Given +# --------------------------------------------------------------------------- + +use_step_matcher("re") + + +@given( + r'I have an openrouter registry with API key "(?P[^"]+)" and organization "(?P[^"]+)"' +) +def step_or_registry_with_key_and_org( + context: Any, or_api_key: str, or_org: str +) -> None: + context.or_registry = ProviderRegistry( + settings=_make_or_settings(openrouter=or_api_key, organization=or_org) + ) + + +@given(r'I have an openrouter registry with API key "(?P[^"]+)"') +def step_or_registry_with_key(context: Any, or_api_key: str) -> None: + context.or_registry = ProviderRegistry( + settings=_make_or_settings(openrouter=or_api_key) + ) + + +@given(r"I have an openrouter registry with no API keys") +def step_or_registry_no_keys(context: Any) -> None: + context.or_registry = ProviderRegistry(settings=_make_or_settings()) + + +@given(r"I have the openrouter ProviderRegistry class") +def step_or_registry_class(context: Any) -> None: + context.or_registry_class = ProviderRegistry + + +@given(r"the openrouter registry LangChain ChatOpenAI client is stubbed") +def step_or_stub_chat_openai(context: Any) -> None: + _stub_chat_openai(context) + + +# --------------------------------------------------------------------------- +# When +# --------------------------------------------------------------------------- + + +@when( + r'I call create_llm on the openrouter registry for provider "openrouter" with model "(?P[^"]+)"' +) +def step_or_create_llm_with_model(context: Any, or_model: str) -> None: + context.or_error = None + try: + context.or_result = context.or_registry.create_llm( + provider_type="openrouter", model_id=or_model + ) + except ValueError as exc: + context.or_error = exc + + +@when(r"I call create_llm on the openrouter registry without specifying a provider") +def step_or_create_llm_default(context: Any) -> None: + context.or_error = None + try: + context.or_result = context.or_registry.create_llm() + except ValueError as exc: + context.or_error = exc + + +@when( + r'I call _create_provider_llm on the openrouter registry for provider "openrouter" with model None' +) +def step_or_private_llm_none_model(context: Any) -> None: + context.or_error = None + try: + context.or_result = context.or_registry._create_provider_llm( + ProviderType.OPENROUTER, None + ) + except ValueError as exc: + context.or_error = exc + + +@when( + r'I call _create_provider_llm on the openrouter registry for provider "openrouter" with model "(?P[^"]+)" and default_headers "(?P[^"]+)"' +) +def step_or_private_llm_with_headers( + context: Any, or_model: str, or_headers_string: str +) -> None: + headers = _parse_headers_string(or_headers_string) + context.or_error = None + try: + context.or_result = context.or_registry._create_provider_llm( + ProviderType.OPENROUTER, or_model, default_headers=headers + ) + except ValueError as exc: + context.or_error = exc + + +@when( + r'I call _create_provider_llm on the openrouter registry for provider "openrouter" with model "(?P[^"]+)"' +) +def step_or_private_llm_with_model(context: Any, or_model: str) -> None: + context.or_error = None + try: + context.or_result = context.or_registry._create_provider_llm( + ProviderType.OPENROUTER, or_model + ) + except ValueError as exc: + context.or_error = exc + + +@when( + r'I try to call create_llm on the openrouter registry for provider "openrouter"' +) +def step_or_try_create_llm_no_key(context: Any) -> None: + context.or_error = None + try: + context.or_registry.create_llm(provider_type="openrouter") + except ValueError as exc: + context.or_error = exc + + +@when(r"I check the openrouter FALLBACK_ORDER") +def step_or_check_fallback_order(context: Any) -> None: + context.or_fallback_order = context.or_registry_class.FALLBACK_ORDER + + +@when(r"I check the openrouter DEFAULT_MODELS") +def step_or_check_default_models(context: Any) -> None: + context.or_default_models = context.or_registry_class.DEFAULT_MODELS + + +@when(r"I check the openrouter DEFAULT_CAPABILITIES for OPENROUTER") +def step_or_check_capabilities(context: Any) -> None: + context.or_capabilities = context.or_registry_class.DEFAULT_CAPABILITIES.get( + ProviderType.OPENROUTER + ) + + +@when(r"I call get_provider_info for OPENROUTER on the openrouter registry") +def step_or_get_provider_info(context: Any) -> None: + context.or_provider_info = context.or_registry.get_provider_info( + ProviderType.OPENROUTER + ) + + +# --------------------------------------------------------------------------- +# Then +# --------------------------------------------------------------------------- + + +@then( + r'the openrouter registry ChatOpenAI should be called with model "(?P[^"]+)"' +) +def step_or_assert_model(context: Any, or_expected_model: str) -> None: + calls = getattr(context, "_or_chat_openai_calls", []) + assert calls, "Expected ChatOpenAI to be instantiated" + actual_model = calls[-1].get("model") + assert actual_model == or_expected_model, ( + f"Expected model={or_expected_model!r}, got {actual_model!r}" + ) + + +@then( + r'the openrouter registry ChatOpenAI should use base url "(?P[^"]+)"' +) +def step_or_assert_base_url(context: Any, or_expected_url: str) -> None: + calls = getattr(context, "_or_chat_openai_calls", []) + assert calls, "Expected ChatOpenAI to be instantiated" + actual_url = calls[-1].get("openai_api_base") + assert actual_url == or_expected_url, ( + f"Expected openai_api_base={or_expected_url!r}, got {actual_url!r}" + ) + + +@then( + r'the openrouter registry ChatOpenAI should use api key "(?P[^"]+)"' +) +def step_or_assert_api_key(context: Any, or_expected_key: str) -> None: + calls = getattr(context, "_or_chat_openai_calls", []) + assert calls, "Expected ChatOpenAI to be instantiated" + actual_key = calls[-1].get("openai_api_key") + assert actual_key == or_expected_key, ( + f"Expected openai_api_key={or_expected_key!r}, got {actual_key!r}" + ) + + +@then( + r'the openrouter registry ChatOpenAI should include header "(?P[^"]+)" with value "(?P[^"]+)"' +) +def step_or_assert_header( + context: Any, or_header_name: str, or_expected_value: str +) -> None: + calls = getattr(context, "_or_chat_openai_calls", []) + assert calls, "Expected ChatOpenAI to be instantiated" + headers = calls[-1].get("default_headers") + assert isinstance(headers, dict), ( + f"Expected default_headers to be a dict, got {type(headers)}" + ) + actual_value = headers.get(or_header_name) + assert actual_value == or_expected_value, ( + f"Expected header {or_header_name}={or_expected_value!r}, got {actual_value!r}" + ) + + +@then(r"an openrouter registry ValueError should be raised") +def step_or_assert_value_error(context: Any) -> None: + assert context.or_error is not None, "Expected a ValueError to be raised" + assert isinstance(context.or_error, ValueError), ( + f"Expected ValueError, got {type(context.or_error).__name__}" + ) + + +@then(r'the openrouter registry error should mention "(?P[^"]+)"') +def step_or_assert_error_contains(context: Any, or_fragment: str) -> None: + assert context.or_error is not None, "Expected an error to be set" + message = str(context.or_error) + assert or_fragment in message, ( + f"Expected error to contain {or_fragment!r}, got: {message!r}" + ) + + +@then(r"ProviderType\.OPENROUTER should be in the fallback order") +def step_or_assert_in_fallback(context: Any) -> None: + assert ProviderType.OPENROUTER in context.or_fallback_order, ( + "Expected ProviderType.OPENROUTER to be in FALLBACK_ORDER" + ) + + +@then(r'the openrouter default model should be "(?P[^"]+)"') +def step_or_assert_default_model(context: Any, or_expected_model: str) -> None: + actual = context.or_default_models.get(ProviderType.OPENROUTER) + assert actual == or_expected_model, ( + f"Expected default model={or_expected_model!r}, got {actual!r}" + ) + + +@then(r"the openrouter provider supports_streaming should be (?P\w+)") +def step_or_assert_streaming(context: Any, or_expected: str) -> None: + assert context.or_capabilities is not None + assert context.or_capabilities.supports_streaming is (or_expected == "True") + + +@then(r"the openrouter provider supports_tool_calls should be (?P\w+)") +def step_or_assert_tool_calls(context: Any, or_expected: str) -> None: + assert context.or_capabilities is not None + assert context.or_capabilities.supports_tool_calls is (or_expected == "True") + + +@then(r"the openrouter provider supports_vision should be (?P\w+)") +def step_or_assert_vision(context: Any, or_expected: str) -> None: + assert context.or_capabilities is not None + assert context.or_capabilities.supports_vision is (or_expected == "True") + + +@then(r"the openrouter provider max_context_length should be (?P\d+)") +def step_or_assert_context_length(context: Any, or_expected: str) -> None: + assert context.or_capabilities is not None + expected_int = int(or_expected) + assert context.or_capabilities.max_context_length == expected_int, ( + f"Expected max_context_length={expected_int}, got {context.or_capabilities.max_context_length}" + ) + + +@then(r"the openrouter provider supports_json_mode should be (?P\w+)") +def step_or_assert_json_mode(context: Any, or_expected: str) -> None: + assert context.or_capabilities is not None + assert context.or_capabilities.supports_json_mode is (or_expected == "True") + + +@then(r"the openrouter provider info should be configured") +def step_or_assert_provider_configured(context: Any) -> None: + info = context.or_provider_info + assert info is not None, "Expected provider info to exist" + assert info.is_configured is True, "Expected provider to be configured" + + +@then(r"the openrouter provider info should not be configured") +def step_or_assert_provider_not_configured(context: Any) -> None: + info = context.or_provider_info + assert info is not None, "Expected provider info to exist" + assert info.is_configured is False, "Expected provider to not be configured" diff --git a/src/cleveragents/providers/registry.py b/src/cleveragents/providers/registry.py index 5aafdfe85..e4b0d2434 100644 --- a/src/cleveragents/providers/registry.py +++ b/src/cleveragents/providers/registry.py @@ -571,6 +571,34 @@ class ProviderRegistry: **kwargs, ) + if provider_type == ProviderType.OPENROUTER: + from langchain_openai import ChatOpenAI + + from cleveragents.providers.llm.openrouter_provider import ( + OpenRouterChatProvider, + ) + + key_attr = self.PROVIDER_KEY_ATTRS.get(provider_type) + api_key = getattr(self._settings, key_attr, None) if key_attr else None + organization = getattr(self._settings, "openrouter_organization", None) + default_headers: dict[str, str] | None = kwargs.pop( + "default_headers", None + ) + if organization: + if default_headers is None: + default_headers = {} + default_headers.setdefault("HTTP-Referer", organization) + default_headers.setdefault("X-Title", organization) + openrouter_kwargs: dict[str, Any] = { + "model": model_id or "anthropic/claude-sonnet-4-20250514", + "openai_api_base": OpenRouterChatProvider._BASE_URL, + "openai_api_key": api_key or "", + } + if default_headers: + openrouter_kwargs["default_headers"] = default_headers + openrouter_kwargs.update(kwargs) + return ChatOpenAI(**openrouter_kwargs) + if provider_type == ProviderType.GROQ: from langchain_groq import ChatGroq