diff --git a/CHANGELOG.md b/CHANGELOG.md index d098d8464..08306c585 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] + +### Added + +- **Robot Framework integration tests for the provider module** (#10952): Added comprehensive + Robot test suite covering the provider registry, fallback selection, capabilities metadata, + and cost tracking infrastructure. The new test suite provides deep coverage of ProviderRegistry + discovery, configuration filtering, default provider/model resolution, global singleton behavior, + AI provider creation (including MOCK provider sandboxing), LLM factory instantiation, and + named provider resolution with full error-path verification. ### Changed - **`agents session list` now displays full 26-character session ULIDs** (#10970): The Rich table diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 1b5c41879..e03bc7c4e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -31,3 +31,5 @@ Below are some of the specific details of various contributions. * HAL 9000 has contributed comprehensive milestone documentation for v3.6.0 (Advanced Concepts & Deferred Features) and v3.7.0 (TUI Implementation) (PR #9903): split into sub-documents covering context strategies, LLM backends, resource types, A2A rename, container tool execution, scope chain resolution, cost/safety budgets, E2E workflow tests, code review examples, plugin architecture, TUI layout, persona system, reference/command input, session management, configuration, and TuiMaterializer integration. * HAL 9000 has contributed the LLMTraceRepository data-integrity fix (PR #8185 / issue #7505): replaced the unconditional `session.commit()` in `LLMTraceRepository.save()` with a dual-path implementation that respects the UnitOfWork pattern — flushing only when an external session is provided, and flushing + committing + closing when operating standalone. This eliminates premature transaction commits, loss of rollback capability, and a docstring/implementation mismatch. * HAL 9000 has contributed the ACMS Index Data Model and File Traversal Engine (PR #9664 / issue #9579): foundational data structures for indexed context entries with hot/warm/cold/archive storage tier classification, tag system, and a timeout-safe chunked file traversal engine for large projects with 10,000+ files. + +* HAL 9000 has contributed comprehensive Robot Framework integration tests for the provider module (PR #10952): added 40 test cases covering registry discovery, configuration filtering, default provider/model resolution, global singleton behavior, AI provider creation with MOCK sandboxing, LLM factory instantiation, and error pathway verification. diff --git a/robot/helper_provider_integration.py b/robot/helper_provider_integration.py new file mode 100644 index 000000000..d0953d514 --- /dev/null +++ b/robot/helper_provider_integration.py @@ -0,0 +1,826 @@ +#!/usr/bin/env python3 +"""Helper script for provider integration Robot Framework tests.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_src = str(Path(__file__).resolve().parents[1] / "src") +if _src not in sys.path: + sys.path.insert(0, _src) + + +def test_discover_all_providers() -> None: + """Verify ProviderRegistry discovers all 10 ProviderType entries.""" + from cleveragents.providers.registry import ProviderRegistry + + registry = ProviderRegistry() + providers = registry.get_all_providers() + assert len(providers) == 10, f"Expected 10 providers, got {len(providers)}" + + +def test_get_all_providers_count() -> None: + """Verify get_all_providers() returns at least as many configs.""" + from cleveragents.providers.registry import ProviderRegistry + + registry = ProviderRegistry() + all_p = registry.get_all_providers() + configured_p = registry.get_configured_providers() + assert len(all_p) >= len(configured_p), "All should be >= configured count" + + +def test_get_configured_providers_filtering() -> None: + """get_configured_providers correctly filters among API-key scenarios.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + saved = os.environ.pop("OPENAI_API_KEY", None) + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + configured = registry.get_configured_providers() + provider_types = [p.provider_type.value for p in configured] + assert "openai" not in provider_types, ( + f"openai should be unconfigured, got: {provider_types}" + ) + + os.environ["OPENAI_API_KEY"] = "fake-key-for-robot" + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry2 = ProviderRegistry() + configured2 = registry2.get_configured_providers() + provider_types2 = [p.provider_type.value for p in configured2] + assert "openai" in provider_types2, f"openai should be configured: {provider_types2}" + finally: + if saved is not None: + os.environ["OPENAI_API_KEY"] = saved + else: + os.environ.pop("OPENAI_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_get_provider_info_by_enum() -> None: + """get_provider_info works when passed a ProviderType enum.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + registry = ProviderRegistry() + info = registry.get_provider_info(ProviderType.OPENAI) + assert info is not None + assert info.provider_type == ProviderType.OPENAI + + +def test_get_provider_info_by_lowercase_string() -> None: + """get_provider_info works with lowercase string lookup.""" + from cleveragents.providers.registry import ProviderRegistry + + registry = ProviderRegistry() + info = registry.get_provider_info("openai") + assert info is not None + assert info.provider_type.value == "openai" + + +def test_get_provider_info_by_uppercase_string() -> None: + """get_provider_info handles uppercase strings case-insensitively.""" + from cleveragents.providers.registry import ProviderRegistry + + registry = ProviderRegistry() + info = registry.get_provider_info("ANTHROPIC") + assert info is not None + assert info.provider_type.value == "anthropic" + + +def test_get_provider_info_unknown_returns_none() -> None: + """get_provider_info returns None for unknown provider names.""" + from cleveragents.providers.registry import ProviderRegistry + + registry = ProviderRegistry() + assert registry.get_provider_info("nonexistent") is None + assert registry.get_provider_info("FAKEPROVIDER") is None + + +def test_is_provider_configured_true() -> None: + """is_provider_configured returns True for a configured provider.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + os.environ["AZURE_API_KEY"] = "test-key-for-robot" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + assert registry.is_provider_configured("azure") is True + assert registry.is_provider_configured("openai") is False + finally: + os.environ.pop("AZURE_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_is_provider_configured_false() -> None: + """is_provider_configured returns False for an unconfigured provider.""" + from cleveragents.providers.registry import ProviderRegistry + + registry = ProviderRegistry() + assert registry.is_provider_configured("openai") is False + + +def test_get_default_provider_type_env_override() -> None: + """get_default_provider_type respects CLEVERAGENTS_DEFAULT_PROVIDER env var.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, ProviderType, reset_provider_registry, + ) + + os.environ["ANTHROPIC_API_KEY"] = "robot-anthropic-key" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None) + default = registry.get_default_provider_type() + assert default == ProviderType.ANTHROPIC, f"Expected anthropic, got {default}" + + os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = "anthropic" + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry2 = ProviderRegistry() + default2 = registry2.get_default_provider_type() + assert default2 == ProviderType.ANTHROPIC, f"Expected anthropic override, got {default2}" + finally: + os.environ.pop("ANTHROPIC_API_KEY", None) + os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_get_default_provider_type_fallback_order() -> None: + """get_default_provider_type uses fallback order when env var is absent.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, ProviderType, reset_provider_registry, + ) + + os.environ["GROQ_API_KEY"] = "robot-groq-key" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + default = registry.get_default_provider_type() + assert default == ProviderType.GROQ, f"Expected groq fallback, got {default}" + finally: + os.environ.pop("GROQ_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_get_default_provider_type_invalid_env_ignored() -> None: + """get_default_provider_type ignores invalid env var and falls back.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + os.environ["TOGETHER_API_KEY"] = "robot-together-key" + try: + os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = "nonexistent-provider" + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + default = registry.get_default_provider_type() + assert default.value == "together", f"Expected fallback to together, got {default}" + finally: + os.environ.pop("TOGETHER_API_KEY", None) + os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_get_default_provider_type_unconfigured_env_skipped() -> None: + """get_default_provider_type skips env var when provider is unconfigured.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + os.environ["TOGETHER_API_KEY"] = "robot-together-key" + try: + os.environ["CLEVERAGENTS_DEFAULT_PROVIDER"] = "openai" + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + default = registry.get_default_provider_type() + assert default.value == "together", ( + f"Should skip unconfigured openai, got {default}" + ) + finally: + os.environ.pop("TOGETHER_API_KEY", None) + os.environ.pop("CLEVERAGENTS_DEFAULT_PROVIDER", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_get_default_provider_type_none_nothing_configured() -> None: + """get_default_provider_type returns None when no provider is configured.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + api_key_vars = [ + "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", + "AZURE_OPENAI_API_KEY", "AZURE_API_KEY", "OPENROUTER_API_KEY", + "GROQ_API_KEY", "TOGETHER_API_KEY", "COHERE_API_KEY", "PERPLEXITY_API_KEY", + ] + saved: dict[str, str] = {} + for var in api_key_vars: + val = os.environ.get(var) + if val is not None: + saved[var] = val + for var in api_key_vars: + os.environ.pop(var, None) + + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + assert registry.get_default_provider_type() is None + finally: + for var in api_key_vars: + if var in saved: + os.environ[var] = saved[var] + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_get_default_model_env_override() -> None: + """get_default_model returns CLEVERAGENTS_DEFAULT_MODEL env var when set.""" + import os + + from cleveragents.providers.registry import ProviderRegistry + + os.environ["CLEVERAGENTS_DEFAULT_MODEL"] = "overridden-model-v2" + try: + registry = ProviderRegistry() + model = registry.get_default_model() + assert model == "overridden-model-v2", f"Expected overridden, got {model}" + finally: + os.environ.pop("CLEVERAGENTS_DEFAULT_MODEL", None) + + +def test_get_default_model_specific_provider() -> None: + """get_default_model returns provider-specific default when no env var.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + registry = ProviderRegistry() + model = registry.get_default_model(ProviderType.ANTHROPIC) + assert model == "claude-sonnet-4-20250514", f"Expected anthropic default, got {model}" + + +def test_get_default_model_invalid_returns_none() -> None: + """get_default_model returns None for an invalid provider type.""" + from cleveragents.providers.registry import ProviderRegistry + + registry = ProviderRegistry() + assert registry.get_default_model("totally-invalid-provider") is None + + +def test_default_capabilities_openai() -> None: + """DEFAULT_CAPABILITIES are correct for OPENAI.""" + from cleveragents.providers.registry import ( + ProviderCapabilities, ProviderRegistry, ProviderType, + ) + + caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.OPENAI] + assert caps.supports_streaming is True + assert caps.supports_tool_calls is True + assert caps.supports_vision is True + assert caps.max_context_length == 128000 + assert caps.supports_json_mode is True + + +def test_default_capabilities_anthropic() -> None: + """DEFAULT_CAPABILITIES ANTHROPIC has no json mode.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.ANTHROPIC] + assert caps.supports_json_mode is False + assert caps.max_context_length == 200000 + + +def test_default_capabilities_google() -> None: + """DEFAULT_CAPABILITIES GOOGLE has 1M context.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.GOOGLE] + assert caps.max_context_length == 1000000 + + +def test_default_capabilities_cohere() -> None: + """DEFAULT_CAPABILITIES COHERE lacks vision.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.COHERE] + assert caps.supports_vision is False + + +def test_default_capabilities_mock() -> None: + """DEFAULT_CAPABILITIES MOCK is very limited.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + caps = ProviderRegistry.DEFAULT_CAPABILITIES[ProviderType.MOCK] + assert caps.supports_streaming is False + assert caps.supports_tool_calls is False + assert caps.max_context_length == 4096 + + +def test_default_models_all_providers() -> None: + """DEFAULT_MODELS have entries for every provider type.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + for pt in ProviderType: + model = ProviderRegistry.DEFAULT_MODELS.get(pt) + assert model is not None and model != "unknown", ( + f"Missing DEFAULT_MODEL for {pt.value}" + ) + + +def test_fallback_order_sequence() -> None: + """FALLBACK_ORDER has the correct provider priority sequence.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + order = ProviderRegistry.FALLBACK_ORDER + assert len(order) == 8 + assert order[0] == ProviderType.OPENAI + assert order[1] == ProviderType.ANTHROPIC + assert order[2] == ProviderType.GOOGLE + + +def test_global_singleton_get_provider_registry() -> None: + """get_provider_registry returns a singleton instance.""" + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + get_provider_registry, reset_provider_registry, + ) + + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry1 = get_provider_registry() + registry2 = get_provider_registry() + assert registry1 is registry2, "Expected singleton behavior" + + +def test_global_reset_clears_singleton() -> None: + """reset_provider_registry clears the global registry instance.""" + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + get_provider_registry, reset_provider_registry, + ) + + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + r1 = get_provider_registry() + reset_provider_registry() + r2 = get_provider_registry() + assert r1 is not r2, "reset should create a new instance" + + +def test_create_ai_provider_configured_openai() -> None: + """create_ai_provider succeeds with a configured provider.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, reset_provider_registry, + ) + + os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + ai = registry.create_ai_provider() + assert ai is not None + assert ai.name == "openai" + finally: + os.environ.pop("OPENAI_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_create_ai_provider_explicit_model_override() -> None: + """create_ai_provider accepts an explicit model_id override.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + ai = registry.create_ai_provider(model_id="gpt-4o-mini") + assert ai is not None + assert ai.model_id == "gpt-4o-mini" + finally: + os.environ.pop("OPENAI_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_create_ai_provider_mock_rejected_without_env() -> None: + """create_ai_provider rejects MOCK provider without env var.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + registry = ProviderRegistry() + try: + registry.create_ai_provider(provider_type=ProviderType.MOCK) + assert False, "Should have raised ValueError" + except ValueError as e: + assert "Mock provider is not allowed" in str(e) + + +def test_create_ai_provider_mock_succeeds_with_env() -> None: + """create_ai_provider accepts MOCK when env var is set.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, ProviderType, reset_provider_registry, + ) + + os.environ["CLEVERAGENTS_ALLOW_MOCK_PROVIDER"] = "true" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + ai = registry.create_ai_provider(provider_type=ProviderType.MOCK) + assert ai is not None + finally: + os.environ.pop("CLEVERAGENTS_ALLOW_MOCK_PROVIDER", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_create_llm_no_provider_errors() -> None: + """create_llm raises ValueError when no provider is configured.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + api_key_vars = [ + "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", + "AZURE_OPENAI_API_KEY", "AZURE_API_KEY", "OPENROUTER_API_KEY", + "GROQ_API_KEY", "TOGETHER_API_KEY", "COHERE_API_KEY", + ] + saved: dict[str, str] = {} + for var in api_key_vars: + val = os.environ.get(var) + if val is not None: + saved[var] = val + for var in api_key_vars: + os.environ.pop(var, None) + + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + try: + registry.create_llm() + assert False, "Should have raised ValueError" + except ValueError as e: + assert "No AI provider configured" in str(e) + finally: + for var in api_key_vars: + if var in saved: + os.environ[var] = saved[var] + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_create_llm_configured_succeeds() -> None: + """create_llm succeeds with a configured provider.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + os.environ["GROQ_API_KEY"] = "robot-groq-key-fake" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + llm = registry.create_llm() + assert llm is not None + finally: + os.environ.pop("GROQ_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_create_llm_mock_rejected() -> None: + """create_llm rejects MOCK provider without env var.""" + from cleveragents.providers.registry import ProviderRegistry, ProviderType + + registry = ProviderRegistry() + try: + registry.create_llm(provider_type=ProviderType.MOCK) + assert False, "Should have raised ValueError" + except ValueError: + pass + + +def test_create_llm_mock_succeeds_with_env() -> None: + """create_llm accepts MOCK when env var is set.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, ProviderType, reset_provider_registry, + ) + + os.environ["CLEVERAGENTS_ALLOW_MOCK_PROVIDER"] = "true" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + llm = registry.create_llm(provider_type=ProviderType.MOCK) + assert llm is not None + finally: + os.environ.pop("CLEVERAGENTS_ALLOW_MOCK_PROVIDER", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_create_llm_explicit_model_id() -> None: + """create_llm uses the explicit model_id when provided.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ProviderRegistry, reset_provider_registry + + os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + llm = registry.create_llm(model_id="gpt-4o-mini") + assert llm is not None + finally: + os.environ.pop("OPENAI_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_resolve_provider_by_name_success() -> None: + """resolve_provider_by_name works with a configured provider.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, resolve_provider_by_name, reset_provider_registry, + ) + + os.environ["OPENAI_API_KEY"] = "robot-openai-key-fake-12345" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + provider = resolve_provider_by_name("openai") + assert provider is not None + finally: + os.environ.pop("OPENAI_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_resolve_provider_by_name_case_insensitive() -> None: + """resolve_provider_by_name handles case-insensitive names.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, resolve_provider_by_name, reset_provider_registry, + ) + + os.environ["GROQ_API_KEY"] = "robot-groq-key-fake" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + provider = resolve_provider_by_name("Groq") + assert provider is not None + finally: + os.environ.pop("GROQ_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_resolve_provider_by_name_unknown_raises() -> None: + """resolve_provider_by_name raises ValueError for unknown provider names.""" + from cleveragents.providers.registry import resolve_provider_by_name + + try: + resolve_provider_by_name("totally-unregistered-fake-provider") + assert False, "Should have raised ValueError" + except ValueError as e: + string = str(e).lower() + assert "unknown provider" in string or "not configured" in string + + +def test_resolve_provider_by_name_unconfigured_raises() -> None: + """resolve_provider_by_name raises ValueError when provider has no API key.""" + from cleveragents.providers.registry import resolve_provider_by_name + + try: + resolve_provider_by_name("openai") # Not configured here + assert False, "Should have raised ValueError" + except ValueError as e: + string = str(e).lower() + assert "not configured" in string + + +def test_create_provider_instance_groq_type() -> None: + """_create_provider_instance builds the correct LLM type for Groq.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, reset_provider_registry, ProviderType, + ) + + api_key_set = "GROQ_API_KEY" + os.environ[api_key_set] = "robot-groq-key-12345" + + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + from langchain_groq import ChatGroq + llm = registry._create_provider_instance( + ProviderType.GROQ, "llama-3.1-70b-versatile", + ) + assert isinstance(llm, ChatGroq), f"Expected ChatGroq, got {type(llm)}" + finally: + os.environ.pop(api_key_set, None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_provider_capabilities_immutability() -> None: + """ProviderCapabilities is a frozen dataclass and cannot be mutated.""" + from cleveragents.providers.registry import ProviderCapabilities + + caps = ProviderCapabilities() + try: + caps.supports_streaming = False # type: ignore[assignment] + assert False, "Should not be able to mutate frozen dataclass" + except Exception: + pass + + +def test_azure_provider_config_fields() -> None: + """Azure provider _create_provider_instance deploys with correct fields.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, reset_provider_registry, + ) + + azure_key = "AZURE_API_KEY" + azure_endpoint = "https://robot-test-svc.openai.azure.com" + os.environ[azure_key] = "robot-azure-key-12345" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + from langchain_openai import AzureChatOpenAI + llm = registry._create_provider_instance( + "azure", "test-deployment", + azure_endpoint=azure_endpoint, + ) + assert isinstance(llm, AzureChatOpenAI), ( + f"Expected AzureChatOpenAI, got {type(llm)}" + ) + finally: + os.environ.pop(azure_key, None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +def test_azure_missing_endpoint_raises() -> None: + """Azure _create_provider_instance raises when endpoint is missing.""" + import os + + from cleveragents.config.settings import Settings, get_settings + from cleveragents.providers.registry import ( + ProviderRegistry, reset_provider_registry, + ) + + os.environ["AZURE_API_KEY"] = "robot-azure-key-12345" + try: + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + registry = ProviderRegistry() + try: + registry._create_provider_instance("azure", "my-deploy") + assert False, "Should have raised ValueError" + except ValueError as e: + assert "Azure OpenAI endpoint not configured" in str(e) + finally: + os.environ.pop("AZURE_API_KEY", None) + Settings._instance = None # type: ignore[attr-defined] + reset_provider_registry() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: helper_provider_integration.py ", file=sys.stderr) + sys.exit(1) + + test_name = sys.argv[1] + tests = { + "discover-all-providers": test_discover_all_providers, + "get-all-providers-count": test_get_all_providers_count, + "get-configured-providers-filtering": test_get_configured_providers_filtering, + "get-provider-info-by-enum": test_get_provider_info_by_enum, + "get-provider-info-by-lowercase-string": test_get_provider_info_by_lowercase_string, + "get-provider-info-by-uppercase-string": test_get_provider_info_by_uppercase_string, + "get-provider-info-unknown-returns-none": test_get_provider_info_unknown_returns_none, + "is-provider-configured-true": test_is_provider_configured_true, + "is-provider-configured-false": test_is_provider_configured_false, + "get-default-provider-type-env-override": test_get_default_provider_type_env_override, + "get-default-provider-type-fallback-order": test_get_default_provider_type_fallback_order, + "get-default-provider-type-invalid-env-ignored": test_get_default_provider_type_invalid_env_ignored, + "get-default-provider-type-unconfigured-env-skipped": test_get_default_provider_type_unconfigured_env_skipped, + "get-default-provider-type-none": test_get_default_provider_type_none_nothing_configured, + "get-default-model-env-override": test_get_default_model_env_override, + "get-default-model-specific-provider": test_get_default_model_specific_provider, + "get-default-model-invalid-returns-none": test_get_default_model_invalid_returns_none, + "default-capabilities-openai": test_default_capabilities_openai, + "default-capabilities-anthropic": test_default_capabilities_anthropic, + "default-capabilities-google": test_default_capabilities_google, + "default-capabilities-cohere": test_default_capabilities_cohere, + "default-capabilities-mock": test_default_capabilities_mock, + "default-models-all-providers": test_default_models_all_providers, + "fallback-order-sequence": test_fallback_order_sequence, + "global-singleton-get-provider-registry": test_global_singleton_get_provider_registry, + "global-reset-clears-singleton": test_global_reset_clears_singleton, + "create-ai-provider-configured-openai": test_create_ai_provider_configured_openai, + "create-ai-provider-explicit-model-override": test_create_ai_provider_explicit_model_override, + "create-ai-provider-mock-rejected-without-env": test_create_ai_provider_mock_rejected_without_env, + "create-ai-provider-mock-succeeds-with-env": test_create_ai_provider_mock_succeeds_with_env, + "create-llm-no-provider-errors": test_create_llm_no_provider_errors, + "create-llm-configured-succeeds": test_create_llm_configured_succeeds, + "create-llm-mock-rejected": test_create_llm_mock_rejected, + "create-llm-mock-succeeds-with-env": test_create_llm_mock_succeeds_with_env, + "create-llm-explicit-model-id": test_create_llm_explicit_model_id, + "resolve-provider-by-name-success": test_resolve_provider_by_name_success, + "resolve-provider-by-name-case-insensitive": test_resolve_provider_by_name_case_insensitive, + "resolve-provider-by-name-unknown-raises": test_resolve_provider_by_name_unknown_raises, + "resolve-provider-by-name-unconfigured-raises": test_resolve_provider_by_name_unconfigured_raises, + "create-provider-instance-groq-type": test_create_provider_instance_groq_type, + "provider-capabilities-immutability": test_provider_capabilities_immutability, + "azure-provider-config-fields": test_azure_provider_config_fields, + "azure-missing-endpoint-raises": test_azure_missing_endpoint_raises, + } + + test_fn = tests.get(test_name) + if test_fn is None: + print(f"Unknown test: {test_name}", file=sys.stderr) + sys.exit(1) + test_fn() diff --git a/robot/provider_integration.robot b/robot/provider_integration.robot new file mode 100644 index 000000000..4f5db2a80 --- /dev/null +++ b/robot/provider_integration.robot @@ -0,0 +1,427 @@ +*** Settings *** +Resource ${CURDIR}/common.resource +Library OperatingSystem +Library Process + +Suite Setup Setup Test Environment With Database Isolation +Suite Teardown Cleanup Test Environment + +*** Variables *** +${SRC_DIR} ${CURDIR}/.. +${HELPER} ${CURDIR}/helper_provider_integration.py + +*** Test Cases *** +Discover All Providers Registered + [Documentation] Verify ProviderRegistry discovers all 10 ProviderType entries. + [Tags] provider registry discovery subtags:count + ${result}= Run Process ${PYTHON} robot/${HELPER} discover-all-providers + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} discover-all-providers-ok + +Get All Providers Count Consistent + [Documentation] Verify get_all_providers() returns at least as many as configured. + [Tags] provider registry subtags:count + ${result}= Run Process ${PYTHON} robot/${HELPER} get-all-providers-count + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-all-providers-count-ok + +Configured Providers Filtering With No OpenAI Key + [Documentation] get_configured_providers() correctly omits providers without API keys. + [Tags] provider registry configuration subtags:filtering + ${result}= Run Process ${PYTHON} robot/${HELPER} get-configured-providers-filtering + ... cwd=${SRC_DIR} + ... env:OPENAI_API_KEY= + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-configured-providers-filtering-ok + +Provider Info Lookup By Enum Value + [Documentation] get_provider_info() works when passed a ProviderType enum. + [Tags] provider registry lookup subtags:enum + ${result}= Run Process ${PYTHON} robot/${HELPER} get-provider-info-by-enum + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-provider-info-by-enum-ok + +Provider Info Lookup By Lowercase String + [Documentation] get_provider_info() works with lowercase string lookup. + [Tags] provider registry lookup subtags:string + ${result}= Run Process ${PYTHON} robot/${HELPER} get-provider-info-by-lowercase-string + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-provider-info-by-lowercase-string-ok + +Provider Info Lookup By Uppercase String Case-Insensitive + [Documentation] get_provider_info() handles uppercase strings case-insensitively. + [Tags] provider registry lookup subtags:string case-insensitive + ${result}= Run Process ${PYTHON} robot/${HELPER} get-provider-info-by-uppercase-string + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-provider-info-by-uppercase-string-ok + +Provider Info Returns None For Unknown Name + [Documentation] get_provider_info() returns None for unknown provider names. + [Tags] provider registry lookup subtags:none-return + ${result}= Run Process ${PYTHON} robot/${HELPER} get-provider-info-unknown-returns-none + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-provider-info-unknown-returns-none-ok + +Is Provider Configured Returns True For Azure With Key + [Documentation] is_provider_configured() returns True when API key is set. + [Tags] provider registry configuration subtags:configured + ${result}= Run Process ${PYTHON} robot/${HELPER} is-provider-configured-true + ... cwd=${SRC_DIR} + ... env:AZURE_API_KEY=test-key-for-robot + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} is-provider-configured-true-ok + +Is Provider Configured Returns False Without Key + [Documentation] is_provider_configured() returns False when no API key. + [Tags] provider registry configuration subtags:unconfigured + ${result}= Run Process ${PYTHON} robot/${HELPER} is-provider-configured-false + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} is-provider-configured-false-ok + +Default Provider Type Env Override Takes Precedence + [Documentation] get_default_provider_type() prioritizes CLEVERAGENTS_DEFAULT_PROVIDER env var. + [Tags] provider registry default subtags:env-override + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-provider-type-env-override + ... cwd=${SRC_DIR} + ... env:ANTHROPIC_API_KEY=robot-anthropic-key + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-provider-type-env-override-ok + +Default Provider Type Falls Back To Ordered Configured Provider + [Documentation] get_default_provider_type() falls back to priority-order configured provider. + [Tags] provider registry default subtags:fallback-order + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-provider-type-fallback-order + ... cwd=${SRC_DIR} + ... env:GROQ_API_KEY=robot-groq-key + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-provider-type-fallback-order-ok + +Default Provider Type With Invalid Env Var Ignores And Falls Back Together + [Documentation] get_default_provider_type() ignores invalid env var and falls back. + [Tags] provider registry default subtags:invalid-env + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-provider-type-invalid-env-ignored + ... cwd=${SRC_DIR} + ... env:TOGETHER_API_KEY=robot-together-key + ... env:CLEVERAGENTS_DEFAULT_PROVIDER=nonexistent-provider + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-provider-type-invalid-env-ignored-ok + +Default Provider Type Skips Unconfigured Env Var Provider + [Documentation] get_default_provider_type() skips env-specified provider if unconfigured. + [Tags] provider registry default subtags:unconfigured-env + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-provider-type-unconfigured-env-skipped + ... cwd=${SRC_DIR} + ... env:TOGETHER_API_KEY=robot-together-key + ... env:CLEVERAGENTS_DEFAULT_PROVIDER=openai + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-provider-type-unconfigured-env-skipped-ok + +Default Provider Type Returns None When Nothing Configured + [Documentation] get_default_provider_type() returns None when no API keys present. + [Tags] provider registry default subtags:none-configured + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-provider-type-none + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-provider-type-none-ok + +Default Model Env Override Takes Precedence + [Documentation] get_default_model() returns CLEVERAGENTS_DEFAULT_MODEL env var. + [Tags] provider registry default-model subtags:env-override + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-model-env-override + ... cwd=${SRC_DIR} + ... env:CLEVERAGENTS_DEFAULT_MODEL=overridden-model-v2 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-model-env-override-ok + +Default Model Returns Provider-Specific Default Without Env Var + [Documentation] get_default_model() returns provider-specific default when no env var. + [Tags] provider registry default-model subtags:provider-default + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-model-specific-provider + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-model-specific-provider-ok + +Default Model Returns None For Invalid Provider Type + [Documentation] get_default_model() returns None for non-existent provider names. + [Tags] provider registry default-model subtags:invalid-input + ${result}= Run Process ${PYTHON} robot/${HELPER} get-default-model-invalid-returns-none + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} get-default-model-invalid-returns-none-ok + +Default Capabilities OpenAI Has Full Feature Set + [Documentation] DEFAULT_CAPABILITIES for OPENAI has streaming, tools, vision, JSON mode. + [Tags] provider registry capabilities subtags:openai + ${result}= Run Process ${PYTHON} robot/${HELPER} default-capabilities-openai + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} default-capabilities-openai-ok + +Default Capabilities Anthropic No Json Mode Support + [Documentation] DEFAULT_CAPABILITIES for ANTHROPIC correctly lacks json_mode. + [Tags] provider registry capabilities subtags:anthropic + ${result}= Run Process ${PYTHON} robot/${HELPER} default-capabilities-anthropic + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} default-capabilities-anthropic-ok + +Default Capabilities Google Has 1M Context Window + [Documentation] DEFAULT_CAPABILITIES for GOOGLE has max_context_length of 1000000. + [Tags] provider registry capabilities subtags:google + ${result}= Run Process ${PYTHON} robot/${HELPER} default-capabilities-google + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} default-capabilities-google-ok + +Default Capabilities Cohere Lacks Vision Support + [Documentation] DEFAULT_CAPABILITIES for COHERE correctly sets supports_vision=False. + [Tags] provider registry capabilities subtags:cohere + ${result}= Run Process ${PYTHON} robot/${HELPER} default-capabilities-cohere + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} default-capabilities-cohere-ok + +Default Capabilities Mock Has No Streaming Or Tool Calls + [Documentation] DEFAULT_CAPABILITIES for MOCK is intentionally very limited. + [Tags] provider registry capabilities subtags:mock + ${result}= Run Process ${PYTHON} robot/${HELPER} default-capabilities-mock + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} default-capabilities-mock-ok + +Default Models Coverage All Provider Types + [Documentation] DEFAULT_MODELS dict has non-empty entry for each ProviderType. + [Tags] provider registry defaults subtags:model-coverage + ${result}= Run Process ${PYTHON} robot/${HELPER} default-models-all-providers + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} default-models-all-providers-ok + +Fallback Order Has Correct Priority Sequence + [Documentation] FALLBACK_ORDER starts with OpenAI, then Anthropic, then Google. + [Tags] provider registry fallback subtags:order-verification + ${result}= Run Process ${PYTHON} robot/${HELPER} fallback-order-sequence + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} fallback-order-sequence-ok + +Global Singleton Get Provider Registry Returns Same Instance + [Documentation] get_provider_registry() returns the same instance on repeated calls. + [Tags] provider registry global subtags:singleton + ${result}= Run Process ${PYTHON} robot/${HELPER} global-singleton-get-provider-registry + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} global-singleton-get-provider-registry-ok + +Global Reset Cleared Singleton Instance + [Documentation] reset_provider_registry() invalidates the cached singleton instance. + [Tags] provider registry global subtags:reset + ${result}= Run Process ${PYTHON} robot/${HELPER} global-reset-clears-singleton + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} global-reset-clears-singleton-ok + +Create AI Provider Succeeds With OpenAI Key + [Documentation] create_ai_provider() returns an AIProvider with a configured OpenAI key. + [Tags] provider registry creation subtags:openai + ${result}= Run Process ${PYTHON} robot/${HELPER} create-ai-provider-configured-openai + ... cwd=${SRC_DIR} + ... env:OPENAI_API_KEY=robot-openai-key-fake-12345 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-ai-provider-configured-openai-ok + +Create AI Provider With Explicit Model Override + [Documentation] create_ai_provider() accepts a custom model_id parameter. + [Tags] provider registry creation subtags:model-override + ${result}= Run Process ${PYTHON} robot/${HELPER} create-ai-provider-explicit-model-override + ... cwd=${SRC_DIR} + ... env:OPENAI_API_KEY=robot-openai-key-fake-12345 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-ai-provider-explicit-model-override-ok + +Create AI Provider Rejects Mock Without Env Var + [Documentation] create_ai_provider() raises ValueError for MOCK provider without flag. + [Tags] provider registry creation subtags:mock-security + ${result}= Run Process ${PYTHON} robot/${HELPER} create-ai-provider-mock-rejected-without-env + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-ai-provider-mock-rejected-without-env-ok + +Create AI Provider Accepts Mock With Env Flag + [Documentation] create_ai_provider() accepts MOCK when CLEVERAGENTS_ALLOW_MOCK_PROVIDER=true. + [Tags] provider registry creation subtags:mock-sandboxed + ${result}= Run Process ${PYTHON} robot/${HELPER} create-ai-provider-mock-succeeds-with-env + ... cwd=${SRC_DIR} + ... env:CLEVERAGENTS_ALLOW_MOCK_PROVIDER=true + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-ai-provider-mock-succeeds-with-env-ok + +Create LLM Fails When No Provider Configured + [Documentation] create_llm() raises ValueError when no API keys are available. + [Tags] provider registry creation subtags:no-key-error + ${result}= Run Process ${PYTHON} robot/${HELPER} create-llm-no-provider-errors + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-llm-no-provider-errors-ok + +Create LLM Succeeds With Groq Key + [Documentation] create_llm() works when at least one provider has an API key. + [Tags] provider registry creation subtags:groq-key + ${result}= Run Process ${PYTHON} robot/${HELPER} create-llm-configured-succeeds + ... cwd=${SRC_DIR} + ... env:GROQ_API_KEY=robot-groq-key-fake + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-llm-configured-succeeds-ok + +Create LLM Rejects Mock Without Env Var + [Documentation] create_llm() rejects MOCK provider without CLEVERAGENTS_ALLOW_MOCK_PROVIDER. + [Tags] provider registry creation subtags:mock-security + ${result}= Run Process ${PYTHON} robot/${HELPER} create-llm-mock-rejected + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-llm-mock-rejected-ok + +Create LLM Accepts Mock With Env Flag + [Documentation] create_llm() accepts MOCK when CLEVERAGENTS_ALLOW_MOCK_PROVIDER=true. + [Tags] provider registry creation subtags:mock-sandboxed + ${result}= Run Process ${PYTHON} robot/${HELPER} create-llm-mock-succeeds-with-env + ... cwd=${SRC_DIR} + ... env:CLEVERAGENTS_ALLOW_MOCK_PROVIDER=true + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-llm-mock-succeeds-with-env-ok + +Create LLM With Explicit Model ID Returns Instance + [Documentation] create_llm(model_id=...) uses the provided model identifier. + [Tags] provider registry creation subtags:model-id-param + ${result}= Run Process ${PYTHON} robot/${HELPER} create-llm-explicit-model-id + ... cwd=${SRC_DIR} + ... env:OPENAI_API_KEY=robot-openai-key-fake-12345 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-llm-explicit-model-id-ok + +Resolve Provider By Name Succeeds With OpenAI Key + [Documentation] resolve_provider_by_name() returns provider when configured. + [Tags] provider registry resolution subtags:configured + ${result}= Run Process ${PYTHON} robot/${HELPER} resolve-provider-by-name-success + ... cwd=${SRC_DIR} + ... env:OPENAI_API_KEY=robot-openai-key-fake-12345 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} resolve-provider-by-name-success-ok + +Resolve Provider By Name Case Insensitive With Groq + [Documentation] resolve_provider_by_name() handles mixed-case provider names. + [Tags] provider registry resolution subtags:case-insensitive + ${result}= Run Process ${PYTHON} robot/${HELPER} resolve-provider-by-name-case-insensitive + ... cwd=${SRC_DIR} + ... env:GROQ_API_KEY=robot-groq-key-fake + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} resolve-provider-by-name-case-insensitive-ok + +Resolve Provider By Name Raises For Unknown Provider + [Documentation] resolve_provider_by_name() raises ValueError for unregistered names. + [Tags] provider registry resolution subtags:error-handling + ${result}= Run Process ${PYTHON} robot/${HELPER} resolve-provider-by-name-unknown-raises + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} resolve-provider-by-name-unknown-raises-ok + +Resolve Provider By Name Raises For Unconfigured Provider + [Documentation] resolve_provider_by_name() raises ValueError when provider has no key. + [Tags] provider registry resolution subtags:unconfigured-error + ${result}= Run Process ${PYTHON} robot/${HELPER} resolve-provider-by-name-unconfigured-raises + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} resolve-provider-by-name-unconfigured-raises-ok + +Create Provider Instance Builds Correct Groq Type + [Documentation] _create_provider_instance() returns ChatGroq for provider type GROQ. + [Tags] provider registry creation subtags:instance-type + ${result}= Run Process ${PYTHON} robot/${HELPER} create-provider-instance-groq-type + ... cwd=${SRC_DIR} + ... env:GROQ_API_KEY=robot-groq-key-12345 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} create-provider-instance-groq-type-ok + +Provider Capabilities Frozen Dataclass Immutability + [Documentation] ProviderCapabilities cannot be mutated after creation. + [Tags] provider registry capabilities subtags:immutability + ${result}= Run Process ${PYTHON} robot/${HELPER} provider-capabilities-immutability + ... cwd=${SRC_DIR} + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} provider-capabilities-immutability-ok + +Azure Provider Config Fields Correctly Built + [Documentation] _create_provider_instance() constructs AzureChatOpenAI with endpoint + key. + [Tags] provider registry creation subtags:azure-config + ${result}= Run Process ${PYTHON} robot/${HELPER} azure-provider-config-fields + ... cwd=${SRC_DIR} + ... env:AZURE_API_KEY=robot-azure-key-12345 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} azure-provider-config-fields-ok + +Azure Provider Missing Endpoint Raises ValueError + [Documentation] Azure _create_provider_instance() fails fast with helpful error on missing endpoint. + [Tags] provider registry creation subtags:azure-error + ${result}= Run Process ${PYTHON} robot/${HELPER} azure-missing-endpoint-raises + ... cwd=${SRC_DIR} + ... env:AZURE_API_KEY=robot-azure-key-12345 + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} azure-missing-endpoint-raises-ok + +*** Keywords *** +Log Process Failure + [Arguments] ${result} + Run Keyword If ${result.rc} == 0 Return From Keyword + Log To Console Process failed with rc=${result.rc} + Log To Console STDOUT:${\n}${result.stdout} + Log To Console STDERR:${\n}${result.stderr}