From 2917aa7ddb02f5aaf517da6b0a8d8d8da7d02a8b Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Sun, 5 Apr 2026 18:07:36 +0000 Subject: [PATCH] fix(cli): extend agents diagnostics to check all 9 supported providers Extend _check_providers() in system.py to report diagnostic status for all 9 providers supported by ProviderRegistry: OpenAI, Anthropic, Google, Azure, OpenRouter, Gemini, Cohere, Groq, and Together AI. Previously only 4 providers (OpenAI, Anthropic, Google, OpenRouter) were checked, leaving users of Groq, Together AI, Cohere, Azure, and Gemini with no diagnostic feedback about their provider configuration. Changes: - Add Azure (AZURE_OPENAI_API_KEY), Gemini (GEMINI_API_KEY), Cohere (COHERE_API_KEY), Groq (GROQ_API_KEY), and Together AI (TOGETHER_API_KEY) to the provider_checks list - Add Behave feature file with 11 scenarios covering all 9 providers (presence, OK status when configured, WARN with recommendation when not) ISSUES CLOSED: #3422 --- .../diagnostics_provider_coverage.feature | 101 +++++++++++ features/mocks/settings_mock.py | 38 ++++ .../diagnostics_provider_coverage_steps.py | 170 ++++++++++++++++++ src/cleveragents/cli/commands/system.py | 5 + 4 files changed, 314 insertions(+) create mode 100644 features/diagnostics_provider_coverage.feature create mode 100644 features/mocks/settings_mock.py create mode 100644 features/steps/diagnostics_provider_coverage_steps.py diff --git a/features/diagnostics_provider_coverage.feature b/features/diagnostics_provider_coverage.feature new file mode 100644 index 000000000..312b8a261 --- /dev/null +++ b/features/diagnostics_provider_coverage.feature @@ -0,0 +1,101 @@ +@diagnostics +Feature: Diagnostics provider coverage — all 9 providers checked + As a developer ensuring thorough diagnostic coverage + I want _check_providers() to report status for all 9 supported providers + So that users of Groq, Together AI, Cohere, Azure, and Gemini see diagnostic feedback + + # =========================================================================== + # _check_providers — all 9 providers present + # =========================================================================== + + Scenario: check_providers returns results for all 9 supported providers + Given the diagnostics provider check module is loaded + When I call check_providers with no providers configured + Then the provider check results should contain exactly 9 entries + And the provider check results should include provider "openai" + And the provider check results should include provider "anthropic" + And the provider check results should include provider "google" + And the provider check results should include provider "azure" + And the provider check results should include provider "openrouter" + And the provider check results should include provider "gemini" + And the provider check results should include provider "cohere" + And the provider check results should include provider "groq" + And the provider check results should include provider "together" + + # =========================================================================== + # _check_providers — configured provider shows OK + # =========================================================================== + + Scenario: check_providers shows OK status for a configured provider + Given the diagnostics provider check module is loaded + When I call check_providers with "groq" configured + Then the provider check result for "groq" should have status "ok" + And the provider check result for "groq" should have details "configured" + And the provider check result for "groq" should have no recommendation + + Scenario: check_providers shows OK status for azure when configured + Given the diagnostics provider check module is loaded + When I call check_providers with "azure" configured + Then the provider check result for "azure" should have status "ok" + And the provider check result for "azure" should have details "configured" + And the provider check result for "azure" should have no recommendation + + Scenario: check_providers shows OK status for together when configured + Given the diagnostics provider check module is loaded + When I call check_providers with "together" configured + Then the provider check result for "together" should have status "ok" + And the provider check result for "together" should have details "configured" + And the provider check result for "together" should have no recommendation + + Scenario: check_providers shows OK status for cohere when configured + Given the diagnostics provider check module is loaded + When I call check_providers with "cohere" configured + Then the provider check result for "cohere" should have status "ok" + And the provider check result for "cohere" should have details "configured" + And the provider check result for "cohere" should have no recommendation + + Scenario: check_providers shows OK status for gemini when configured + Given the diagnostics provider check module is loaded + When I call check_providers with "gemini" configured + Then the provider check result for "gemini" should have status "ok" + And the provider check result for "gemini" should have details "configured" + And the provider check result for "gemini" should have no recommendation + + # =========================================================================== + # _check_providers — unconfigured provider shows WARN with recommendation + # =========================================================================== + + Scenario: check_providers shows WARN for groq when not configured + Given the diagnostics provider check module is loaded + When I call check_providers with no providers configured + Then the provider check result for "groq" should have status "warn" + And the provider check result for "groq" should have details "missing" + And the provider check result for "groq" should have a recommendation mentioning "GROQ_API_KEY" + + Scenario: check_providers shows WARN for together when not configured + Given the diagnostics provider check module is loaded + When I call check_providers with no providers configured + Then the provider check result for "together" should have status "warn" + And the provider check result for "together" should have details "missing" + And the provider check result for "together" should have a recommendation mentioning "TOGETHER_API_KEY" + + Scenario: check_providers shows WARN for cohere when not configured + Given the diagnostics provider check module is loaded + When I call check_providers with no providers configured + Then the provider check result for "cohere" should have status "warn" + And the provider check result for "cohere" should have details "missing" + And the provider check result for "cohere" should have a recommendation mentioning "COHERE_API_KEY" + + Scenario: check_providers shows WARN for azure when not configured + Given the diagnostics provider check module is loaded + When I call check_providers with no providers configured + Then the provider check result for "azure" should have status "warn" + And the provider check result for "azure" should have details "missing" + And the provider check result for "azure" should have a recommendation mentioning "AZURE_OPENAI_API_KEY" + + Scenario: check_providers shows WARN for gemini when not configured + Given the diagnostics provider check module is loaded + When I call check_providers with no providers configured + Then the provider check result for "gemini" should have status "warn" + And the provider check result for "gemini" should have details "missing" + And the provider check result for "gemini" should have a recommendation mentioning "GEMINI_API_KEY" diff --git a/features/mocks/settings_mock.py b/features/mocks/settings_mock.py new file mode 100644 index 000000000..8efbaf878 --- /dev/null +++ b/features/mocks/settings_mock.py @@ -0,0 +1,38 @@ +"""Mock Settings object for testing diagnostics and provider configuration. + +This mock is used in Behave step definitions to simulate the Settings object +returned by ``cleveragents.config.settings.get_settings()``. + +Following the mock placement rule (ADR-022), all mocking code must exist only +within the ``features/mocks/`` directory and never in step definition files. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + + +def make_settings_mock(configured_providers: set[str] | None = None) -> MagicMock: + """Build a mock Settings object for use in Behave step definitions. + + Args: + configured_providers: Set of provider names that should appear configured. + If None, no providers are configured. + + Returns: + A MagicMock that mimics the Settings interface, with + ``has_provider_configured()`` returning True only for providers in + ``configured_providers``. + """ + if configured_providers is None: + configured_providers = set() + + s = MagicMock() + + def has_provider_configured(provider: str | None = None) -> bool: + if provider is None: + return bool(configured_providers) + return provider in configured_providers + + s.has_provider_configured = MagicMock(side_effect=has_provider_configured) + return s diff --git a/features/steps/diagnostics_provider_coverage_steps.py b/features/steps/diagnostics_provider_coverage_steps.py new file mode 100644 index 000000000..aaedea1c9 --- /dev/null +++ b/features/steps/diagnostics_provider_coverage_steps.py @@ -0,0 +1,170 @@ +"""Step definitions for diagnostics_provider_coverage.feature. + +Tests that _check_providers() in system.py reports diagnostic status for +all 9 supported providers: openai, anthropic, google, azure, openrouter, +gemini, cohere, groq, together. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from behave import given, then, when +from behave.runner import Context + +from features.mocks.settings_mock import make_settings_mock + +# --------------------------------------------------------------------------- +# Given +# --------------------------------------------------------------------------- + + +@given("the diagnostics provider check module is loaded") +def step_provider_check_module_loaded(context: Context) -> None: + context.provider_results = None + context.configured_providers: set[str] = set() + + +# --------------------------------------------------------------------------- +# When +# --------------------------------------------------------------------------- + + +@when("I call check_providers with no providers configured") +def step_check_providers_none_configured(context: Context) -> None: + from cleveragents.cli.commands.system import _check_providers + + ms = make_settings_mock(configured_providers=set()) + with patch("cleveragents.config.settings.get_settings", return_value=ms): + context.provider_results = _check_providers() + + +@when('I call check_providers with "{provider}" configured') +def step_check_providers_one_configured(context: Context, provider: str) -> None: + from cleveragents.cli.commands.system import _check_providers + + ms = make_settings_mock(configured_providers={provider}) + with patch("cleveragents.config.settings.get_settings", return_value=ms): + context.provider_results = _check_providers() + + +# --------------------------------------------------------------------------- +# Then — count assertions +# --------------------------------------------------------------------------- + + +@then("the provider check results should contain exactly {count:d} entries") +def step_assert_result_count(context: Context, count: int) -> None: + results = context.provider_results + assert results is not None, "provider_results is None" + actual = len(results) + assert actual == count, ( + f"Expected exactly {count} provider check entries, got {actual}. " + f"Entries: {[r['name'] for r in results]}" + ) + + +# --------------------------------------------------------------------------- +# Then — provider presence assertions +# --------------------------------------------------------------------------- + + +@then('the provider check results should include provider "{provider}"') +def step_assert_provider_present(context: Context, provider: str) -> None: + results = context.provider_results + assert results is not None, "provider_results is None" + names_lower = [r["name"].lower() for r in results] + assert any(provider.lower() in name for name in names_lower), ( + f"Expected provider '{provider}' in results, but got: " + f"{[r['name'] for r in results]}" + ) + + +def _find_provider_result(results: list[dict], provider: str) -> dict | None: + """Find the result entry for a given provider name.""" + for r in results: + if provider.lower() in r["name"].lower(): + return r + return None + + +# --------------------------------------------------------------------------- +# Then — per-provider status/details/recommendation assertions +# --------------------------------------------------------------------------- + + +@then( + 'the provider check result for "{provider}" should have status "{expected_status}"' +) +def step_assert_provider_status( + context: Context, provider: str, expected_status: str +) -> None: + results = context.provider_results + assert results is not None, "provider_results is None" + entry = _find_provider_result(results, provider) + assert entry is not None, ( + f"No result found for provider '{provider}'. " + f"Available: {[r['name'] for r in results]}" + ) + actual = str(entry["status"]) + assert actual == expected_status, ( + f"Provider '{provider}': expected status '{expected_status}', got '{actual}'" + ) + + +@then( + 'the provider check result for "{provider}" should have details "{expected_details}"' +) +def step_assert_provider_details( + context: Context, provider: str, expected_details: str +) -> None: + results = context.provider_results + assert results is not None, "provider_results is None" + entry = _find_provider_result(results, provider) + assert entry is not None, ( + f"No result found for provider '{provider}'. " + f"Available: {[r['name'] for r in results]}" + ) + actual = entry["details"] + assert actual == expected_details, ( + f"Provider '{provider}': expected details '{expected_details}', got '{actual}'" + ) + + +@then('the provider check result for "{provider}" should have no recommendation') +def step_assert_provider_no_recommendation(context: Context, provider: str) -> None: + results = context.provider_results + assert results is not None, "provider_results is None" + entry = _find_provider_result(results, provider) + assert entry is not None, ( + f"No result found for provider '{provider}'. " + f"Available: {[r['name'] for r in results]}" + ) + rec = entry.get("recommendation") + assert rec is None, ( + f"Provider '{provider}': expected no recommendation, got '{rec}'" + ) + + +@then( + 'the provider check result for "{provider}" should have a recommendation mentioning "{env_var}"' +) +def step_assert_provider_recommendation_mentions( + context: Context, provider: str, env_var: str +) -> None: + results = context.provider_results + assert results is not None, "provider_results is None" + entry = _find_provider_result(results, provider) + assert entry is not None, ( + f"No result found for provider '{provider}'. " + f"Available: {[r['name'] for r in results]}" + ) + rec = entry.get("recommendation") + assert rec is not None, ( + f"Provider '{provider}': expected a recommendation mentioning '{env_var}', " + f"but recommendation is None" + ) + assert env_var in rec, ( + f"Provider '{provider}': expected recommendation to mention '{env_var}', " + f"got '{rec}'" + ) diff --git a/src/cleveragents/cli/commands/system.py b/src/cleveragents/cli/commands/system.py index 97c3dd9f3..1e4f02eb6 100644 --- a/src/cleveragents/cli/commands/system.py +++ b/src/cleveragents/cli/commands/system.py @@ -250,7 +250,12 @@ def _check_providers() -> list[dict[str, Any]]: ("openai", "OPENAI_API_KEY"), ("anthropic", "ANTHROPIC_API_KEY"), ("google", "GOOGLE_API_KEY"), + ("azure", "AZURE_OPENAI_API_KEY"), ("openrouter", "OPENROUTER_API_KEY"), + ("gemini", "GEMINI_API_KEY"), + ("cohere", "COHERE_API_KEY"), + ("groq", "GROQ_API_KEY"), + ("together", "TOGETHER_API_KEY"), ] for provider_name, env_var in provider_checks: -- 2.52.0