diff --git a/benchmarks/decision_di_bench.py b/benchmarks/decision_di_bench.py index eb3980db0..b7c0d51cd 100644 --- a/benchmarks/decision_di_bench.py +++ b/benchmarks/decision_di_bench.py @@ -26,11 +26,6 @@ try: ProcessingState, ) from cleveragents.infrastructure.database.models import Base - from cleveragents.infrastructure.database.repositories import ( - ActionRepository, - DecisionRepository, - LifecyclePlanRepository, - ) from cleveragents.infrastructure.database.unit_of_work import UnitOfWork except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) @@ -46,11 +41,6 @@ except ModuleNotFoundError: ProcessingState, ) from cleveragents.infrastructure.database.models import Base - from cleveragents.infrastructure.database.repositories import ( - ActionRepository, - DecisionRepository, - LifecyclePlanRepository, - ) from cleveragents.infrastructure.database.unit_of_work import UnitOfWork from sqlalchemy import create_engine diff --git a/benchmarks/plan_generation_benchmark.py b/benchmarks/plan_generation_benchmark.py index 65d743cff..a2d7cd757 100644 --- a/benchmarks/plan_generation_benchmark.py +++ b/benchmarks/plan_generation_benchmark.py @@ -9,6 +9,7 @@ from __future__ import annotations from pathlib import Path import tiktoken +from langchain_community.llms import FakeListLLM from cleveragents.agents.graphs.plan_generation import PlanGenerationGraph from cleveragents.domain.models.core import ( @@ -76,7 +77,9 @@ class PlanGenerationSuite: """Benchmark PlanGenerationGraph invoke and stream paths.""" def setup(self) -> None: - self.graph = PlanGenerationGraph() + self.graph = PlanGenerationGraph( + llm=FakeListLLM(responses=["bench response"] * 3) + ) self.project = _sample_project() self.plan = _sample_plan() self.contexts = _sample_contexts() diff --git a/benchmarks/provider_selection_bench.py b/benchmarks/provider_selection_bench.py new file mode 100644 index 000000000..c02573d12 --- /dev/null +++ b/benchmarks/provider_selection_bench.py @@ -0,0 +1,87 @@ +"""ASV benchmarks for provider selection and registry initialization. + +Measures the performance of: +- Provider registry initialization (discovering configured providers) +- Provider resolution by name +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from unittest.mock import MagicMock + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.providers.registry import ( # noqa: E402 + ProviderRegistry, + ProviderType, + reset_provider_registry, +) + + +def _make_settings(openai: str | None = None) -> MagicMock: + settings = MagicMock() + settings.openai_api_key = openai + settings.anthropic_api_key = None + settings.google_api_key = None + settings.gemini_api_key = None + settings.azure_api_key = None + settings.openrouter_api_key = None + settings.cohere_api_key = None + settings.groq_api_key = None + settings.together_api_key = None + settings.default_provider = None + settings.default_model = None + settings.azure_openai_endpoint = None + settings.azure_openai_api_version = None + settings.azure_openai_deployment = None + settings.openrouter_organization = None + settings.mock_providers = False + return settings + + +class TimeProviderRegistryInit: + """Benchmark provider registry initialization.""" + + def setup(self) -> None: + self.settings_none = _make_settings() + self.settings_openai = _make_settings(openai="sk-bench-key") + + def time_provider_registry_init(self) -> None: + """Time registry creation with no providers configured.""" + reset_provider_registry() + ProviderRegistry(settings=self.settings_none) + + def time_provider_registry_init_with_provider(self) -> None: + """Time registry creation with one provider configured.""" + reset_provider_registry() + ProviderRegistry(settings=self.settings_openai) + + +class TimeProviderResolution: + """Benchmark provider resolution.""" + + def setup(self) -> None: + self.settings = _make_settings(openai="sk-bench-key") + reset_provider_registry() + self.registry = ProviderRegistry(settings=self.settings) + + def time_provider_resolution(self) -> None: + """Time default provider type resolution.""" + self.registry.get_default_provider_type() + + def time_get_configured_providers(self) -> None: + """Time listing configured providers.""" + self.registry.get_configured_providers() + + def time_get_default_model(self) -> None: + """Time default model resolution.""" + self.registry.get_default_model(ProviderType.OPENAI) diff --git a/features/provider_fixes.feature b/features/provider_fixes.feature new file mode 100644 index 000000000..5066ac6bb --- /dev/null +++ b/features/provider_fixes.feature @@ -0,0 +1,105 @@ +Feature: Provider configuration fixes remove FakeListLLM defaults + As a developer + I want providers to require explicit LLM configuration + So that FakeListLLM is never silently used as a production fallback + + @phase1 + Scenario: Provider auto-detection selects first configured provider + Given a provider-fix settings instance with openai configured + When I create a provider-fix registry from those settings + Then the provider-fix default provider type should be "openai" + + @phase1 + Scenario: No providers configured and mock_providers off raises error + Given a provider-fix settings instance with no providers configured + And provider-fix mock_providers is disabled + When I call provider-fix validate_provider_availability + Then a provider-fix ValueError should be raised containing "No AI providers configured" + + @phase1 + Scenario: mock_providers flag enables mock provider + Given a provider-fix settings instance with mock_providers enabled + When I build the provider-fix AI provider from the container helper + Then a provider-fix mock AI provider should be returned + + @phase1 + Scenario: FakeListLLM is not used as default fallback in PlanGenerationGraph + When I attempt to create a provider-fix PlanGenerationGraph without an LLM + Then a provider-fix ValueError should be raised containing "No LLM provider configured" + + @phase1 + Scenario: FakeListLLM is not used as default fallback in ContextAnalysisAgent + When I attempt to create a provider-fix ContextAnalysisAgent without an LLM + Then a provider-fix ValueError should be raised containing "No LLM provider configured" + + @phase1 + Scenario: FakeListLLM is not used as default fallback in AutoDebugAgent + When I attempt to create a provider-fix AutoDebugAgent without an LLM + Then a provider-fix ValueError should be raised containing "No LLM provider configured" + + @phase1 + Scenario: Provider selection trace logging outputs chosen provider + Given a provider-fix settings instance with openai configured + When I create a provider-fix registry and request the default provider + Then the provider-fix selection should be logged + + @phase1 + Scenario: resolve_provider_by_name returns provider for configured name + Given a provider-fix settings instance with openai configured + When I provider-fix resolve provider by name "openai" + Then I should receive a provider-fix AI provider instance + + @phase1 + Scenario: resolve_provider_by_name raises error for unconfigured name + Given a provider-fix settings instance with no providers configured + When I provider-fix resolve provider by name "openai" + Then a provider-fix ValueError should be raised containing "not configured" + + @phase1 + Scenario: resolve_provider_by_name raises error for unknown name + Given a provider-fix settings instance with no providers configured + When I provider-fix resolve provider by name "nonexistent_provider" + Then a provider-fix ValueError should be raised containing "Unknown provider" + + @phase1 + Scenario: validate_provider_availability passes with provider configured + Given a provider-fix settings instance with openai configured + And provider-fix mock_providers is disabled on that instance + When I call provider-fix validate_provider_availability + Then no provider-fix error should be raised + + @phase1 + Scenario: mock_providers warns outside test mode + Given a provider-fix settings instance with mock_providers enabled + And the provider-fix environment is set to "production" + When I call provider-fix validate_provider_availability + Then a provider-fix warning should be logged about mock_providers + + @phase1 + Scenario: Settings mock_providers field defaults to False + Given I create a provider-fix fresh settings instance + Then provider-fix mock_providers should be False by default + + @phase1 + Scenario: Container get_ai_provider respects mock_providers setting + Given a provider-fix settings instance with mock_providers enabled + When I call provider-fix get_ai_provider with those settings + Then a provider-fix mock provider or None should be returned + + @phase1 + Scenario: PlanGenerationGraph works with explicit LLM + Given I have a provider-fix FakeListLLM instance for testing + When I create a provider-fix PlanGenerationGraph with that LLM + Then the provider-fix graph should be created successfully + + @phase1 + Scenario: ContextAnalysisAgent works with explicit LLM + Given I have a provider-fix FakeListLLM instance for testing + When I create a provider-fix ContextAnalysisAgent with that LLM + Then the provider-fix agent should be created successfully + + @phase1 + Scenario: AutoDebugAgent works with explicit LLM + Given I have a provider-fix FakeListLLM instance for testing + When I create a provider-fix AutoDebugAgent with that LLM + Then the provider-fix agent should be created successfully diff --git a/features/steps/auto_debug_integration_steps.py b/features/steps/auto_debug_integration_steps.py index ed9f4d308..92c02e1df 100644 --- a/features/steps/auto_debug_integration_steps.py +++ b/features/steps/auto_debug_integration_steps.py @@ -111,14 +111,19 @@ def step_run_auto_debug_build(context: Context, max_attempts: int) -> None: mock_response.content = "Mock LLM response" mock_llm.invoke = MagicMock(return_value=mock_response) - # Patch build_plan and the LLM creation + # Inject mock LLM into the plan service so AutoDebugAgent receives it + plan_service._llm = mock_llm + + # Patch build_plan and the AutoDebugAgent class with ( patch.object(plan_service, "build_plan", side_effect=mock_build), - patch( - "langchain_community.llms.FakeListLLM", - return_value=mock_llm, - ), + patch("cleveragents.agents.AutoDebugAgent") as agent_cls, ): + agent_instance = MagicMock() + agent_instance.invoke.return_value = { + "result": {"success": True, "fix": {}}, + } + agent_cls.return_value = agent_instance try: success, changes, error = plan_service.auto_debug_build( project=context.project, max_attempts=max_attempts diff --git a/features/steps/container_coverage_r2_steps.py b/features/steps/container_coverage_r2_steps.py index 531ee2929..a6dad8fb9 100644 --- a/features/steps/container_coverage_r2_steps.py +++ b/features/steps/container_coverage_r2_steps.py @@ -188,6 +188,9 @@ def step_call_get_ai_provider_mocked(context: Context) -> None: from cleveragents.application.container import get_ai_provider mock_settings = MagicMock() + # Ensure Settings.mock_providers is explicitly False so the non-mock + # branch is exercised (MagicMock auto-attributes are truthy). + mock_settings.mock_providers = False context.r2cont_ai_result = get_ai_provider( settings=mock_settings, provider_registry=context.r2cont_registry, diff --git a/features/steps/context_analysis_graph_coverage_steps.py b/features/steps/context_analysis_graph_coverage_steps.py index dad169f41..c52220125 100644 --- a/features/steps/context_analysis_graph_coverage_steps.py +++ b/features/steps/context_analysis_graph_coverage_steps.py @@ -22,6 +22,18 @@ from cleveragents.agents.graphs.context_analysis import ( ContextAnalysisState, ) + +def _default_context_test_llm() -> FakeListLLM: + """Create a FakeListLLM for context analysis test purposes.""" + return FakeListLLM( + responses=[ + "Dependencies: ['os', 'sys', 'pathlib']", + "Relevance: High - contains core functionality", + "Summary: Python module implementing core business logic", + ] + ) + + # Enough responses so every LLM call across the full workflow is satisfied. _DEFAULT_RESPONSES = [ "Dependencies: ['os', 'sys', 'pathlib']", @@ -401,7 +413,7 @@ def step_astream_events_are_dicts(context: Any) -> None: @when("I create a ContextAnalysisAgent without providing an LLM") def step_create_agent_no_llm(context: Any) -> None: - context.graph_agent = ContextAnalysisAgent() + context.graph_agent = ContextAnalysisAgent(llm=_default_context_test_llm()) @then("the agent LLM should be a FakeListLLM") @@ -483,3 +495,29 @@ def after_scenario(context: Any, _scenario: Any) -> None: ): if hasattr(context, attr): setattr(context, attr, None) + + +# --------------------------------------------------------------------------- +# Scenario: Agent raises ValueError when no LLM provided +# --------------------------------------------------------------------------- + + +@when("I create a context analysis agent with no LLM expecting an error") +def step_create_agent_no_llm_error(context: Any) -> None: + context.raised_error = None + try: + from cleveragents.agents.graphs.context_analysis import ( + ContextAnalysisAgent as _CA, + ) + + _CA(llm=None) + except ValueError as exc: + context.raised_error = exc + + +@then("a ValueError should have been raised about missing LLM") +def step_check_value_error_missing_llm(context: Any) -> None: + assert context.raised_error is not None, ( + "Expected ValueError but no error was raised" + ) + assert "No LLM provider configured" in str(context.raised_error) diff --git a/features/steps/context_analysis_new_coverage_steps.py b/features/steps/context_analysis_new_coverage_steps.py index a042f0838..e3345f21f 100644 --- a/features/steps/context_analysis_new_coverage_steps.py +++ b/features/steps/context_analysis_new_coverage_steps.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock from behave import given, then, when # type: ignore[import-untyped] from behave.runner import Context # type: ignore[import-untyped] +from langchain_community.llms import FakeListLLM from langchain_core.documents import Document from cleveragents.agents.graphs.context_analysis import ( @@ -16,6 +17,17 @@ from cleveragents.agents.graphs.context_analysis import ( ) +def _default_context_test_llm() -> FakeListLLM: + """Create a FakeListLLM for context analysis test purposes.""" + return FakeListLLM( + responses=[ + "Dependencies: ['os', 'sys', 'pathlib']", + "Relevance: High - contains core functionality", + "Summary: Python module implementing core business logic", + ] + ) + + def _empty_state(**overrides: Any) -> ContextAnalysisState: base: ContextAnalysisState = { "file_paths": [], @@ -32,13 +44,26 @@ def _empty_state(**overrides: Any) -> ContextAnalysisState: @when("I create a ContextAnalysisAgent without an LLM") def step_create_default(context: Context) -> None: - context.agent = ContextAnalysisAgent() + context.agent = ContextAnalysisAgent(llm=_default_context_test_llm()) + + +@when("I create a new ContextAnalysisAgent without an LLM expecting error") +def step_create_no_llm_error(context: Context) -> None: + context.raised_error = None + try: + ContextAnalysisAgent(llm=None) + except ValueError as exc: + context.raised_error = exc + + +@then("a ValueError should be raised about missing LLM") +def step_assert_value_error_missing_llm(context: Context) -> None: + assert context.raised_error is not None, "Expected ValueError but none was raised" + assert "No LLM provider configured" in str(context.raised_error) @then("the agent should use FakeListLLM") def step_assert_fake_llm(context: Context) -> None: - from langchain_community.llms import FakeListLLM - assert isinstance(context.agent.llm, FakeListLLM) @@ -49,22 +74,18 @@ def step_assert_compiled(context: Context) -> None: @when("I create a ContextAnalysisAgent with a mock LLM") def step_create_with_llm(context: Context) -> None: - from langchain_community.llms import FakeListLLM - custom_llm = FakeListLLM(responses=["custom response"]) context.agent = ContextAnalysisAgent(llm=custom_llm) @then("the agent should use the provided LLM") def step_assert_custom_llm(context: Context) -> None: - from langchain_community.llms import FakeListLLM - assert isinstance(context.agent.llm, FakeListLLM) @given("a ContextAnalysisAgent instance") def step_agent_instance(context: Context) -> None: - context.agent = ContextAnalysisAgent() + context.agent = ContextAnalysisAgent(llm=_default_context_test_llm()) @when("I invoke _load_files with preloaded documents") @@ -174,7 +195,9 @@ def step_assert_original_chunk(context: Context) -> None: @given("a ContextAnalysisAgent instance with chunk_size {size:d}") def step_agent_custom_chunk(context: Context, size: int) -> None: - context.agent = ContextAnalysisAgent(chunk_size=size, chunk_overlap=20) + context.agent = ContextAnalysisAgent( + llm=_default_context_test_llm(), chunk_size=size, chunk_overlap=20 + ) @given("a state with a large document of {n:d} characters") diff --git a/features/steps/context_service_coverage_gaps_steps.py b/features/steps/context_service_coverage_gaps_steps.py index 8e30042f1..5d84ee090 100644 --- a/features/steps/context_service_coverage_gaps_steps.py +++ b/features/steps/context_service_coverage_gaps_steps.py @@ -345,34 +345,65 @@ def step_trigger_vector_refresh(context): @when("I request a context analysis agent from the coverage service") def step_request_context_agent(context): - context.cov_agent = context.cov_service._get_context_agent() + from langchain_community.llms import FakeListLLM + + context.cov_agent = context.cov_service._get_context_agent( + llm=FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ), + ) @when("I run analyze_context on the workspace project") def step_run_analyze_context(context): - context.analysis_result = context.cov_service.analyze_context(context.cov_project) + from langchain_community.llms import FakeListLLM + + context.analysis_result = context.cov_service.analyze_context( + context.cov_project, + llm=FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ), + ) @when("I run analyze_context_async on the workspace project") def step_run_analyze_context_async(context): + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) context.async_analysis_result = asyncio.run( - context.cov_service.analyze_context_async(context.cov_project) + context.cov_service.analyze_context_async(context.cov_project, llm=_fake_llm) ) @when("I stream analyze_context on the workspace project") def step_stream_analyze_context(context): + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) context.streaming_events = list( - context.cov_service.analyze_context_streaming(context.cov_project) + context.cov_service.analyze_context_streaming( + context.cov_project, llm=_fake_llm + ) ) @when("I async stream analyze_context on the workspace project") def step_async_stream_analyze_context(context): + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) + async def _collect(): events = [] async for event in context.cov_service.analyze_context_streaming_async( - context.cov_project + context.cov_project, llm=_fake_llm ): events.append(event) return events @@ -382,20 +413,37 @@ def step_async_stream_analyze_context(context): @when("I get the context summary from the coverage service") def step_get_context_summary(context): - context.cov_summary = context.cov_service.get_context_summary(context.cov_project) + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) + context.cov_summary = context.cov_service.get_context_summary( + context.cov_project, llm=_fake_llm + ) @when("I get the context dependencies from the coverage service") def step_get_context_dependencies(context): + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) context.cov_dependencies = context.cov_service.get_context_dependencies( - context.cov_project + context.cov_project, llm=_fake_llm ) @when("I get relevant files with threshold {threshold:f} from the coverage service") def step_get_relevant_files(context, threshold: float): + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) context.cov_relevant = context.cov_service.get_relevant_files( - context.cov_project, threshold=threshold + context.cov_project, threshold=threshold, llm=_fake_llm ) diff --git a/features/steps/context_service_new_coverage_steps.py b/features/steps/context_service_new_coverage_steps.py index 271d917ec..a33c318c7 100644 --- a/features/steps/context_service_new_coverage_steps.py +++ b/features/steps/context_service_new_coverage_steps.py @@ -191,7 +191,13 @@ def step_svc_failing_vector(context: Context) -> None: @when("I get the context agent") def step_get_agent(context: Context) -> None: - context.result = context.svc._get_context_agent() + from langchain_community.llms import FakeListLLM + + context.result = context.svc._get_context_agent( + llm=FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ), + ) @then("the context agent should be a ContextAnalysisAgent instance") @@ -211,7 +217,12 @@ def step_project_no_files(context: Context) -> None: @when("I analyze context") def step_analyze_context(context: Context) -> None: - context.result = context.svc.analyze_context(context.project) + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) + context.result = context.svc.analyze_context(context.project, llm=_fake_llm) @then("the result summary should indicate no files") @@ -253,7 +264,12 @@ def step_assert_docs_summary(context: Context) -> None: @when("I retrieve the context summary") def step_get_summary(context: Context) -> None: - context.result = context.svc.get_context_summary(context.project) + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) + context.result = context.svc.get_context_summary(context.project, llm=_fake_llm) @then("the context summary should be a nonempty string") @@ -264,7 +280,14 @@ def step_assert_nonempty_summary(context: Context) -> None: @when("I get context dependencies") def step_get_deps(context: Context) -> None: - context.result = context.svc.get_context_dependencies(context.project) + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) + context.result = context.svc.get_context_dependencies( + context.project, llm=_fake_llm + ) @then("the context deps result should be a dict") @@ -349,7 +372,14 @@ def step_svc_vector_config_error(context: Context) -> None: @when("I stream analyze context") def step_stream_analyze(context: Context) -> None: - context.events = list(context.svc.analyze_context_streaming(context.project)) + from langchain_community.llms import FakeListLLM + + _fake_llm = FakeListLLM( + responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3 + ) + context.events = list( + context.svc.analyze_context_streaming(context.project, llm=_fake_llm) + ) @then("the first event should indicate no files") diff --git a/features/steps/plan_generation_langgraph_coverage_steps.py b/features/steps/plan_generation_langgraph_coverage_steps.py index fdb8bba9e..57bb0d07f 100644 --- a/features/steps/plan_generation_langgraph_coverage_steps.py +++ b/features/steps/plan_generation_langgraph_coverage_steps.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any from behave import given, then, when +from langchain_community.llms import FakeListLLM PLAN_GEN_MODULE_PATH = ( Path(__file__).resolve().parents[2] @@ -19,6 +20,17 @@ PLAN_GEN_MODULE_PATH = ( ) +def _default_test_llm() -> FakeListLLM: + """Create a FakeListLLM for test purposes.""" + return FakeListLLM( + responses=[ + "Requirements: Add error handling with try-except blocks", + "Generated code with proper error handling implementation", + "Validation passed: Code follows best practices", + ] + ) + + def _load_plan_generation_module(context: Any) -> None: """Load the plan_generation module dynamically.""" if hasattr(context, "plan_generation_module"): @@ -44,10 +56,10 @@ def step_langgraph_module_importable(context: Any) -> None: @when("I create a langgraph PlanGenerationGraph with no LLM") def step_create_langgraph_graph_no_llm(context: Any) -> None: - """Create graph with default LLM.""" + """Create graph with explicit test LLM.""" _load_plan_generation_module(context) PlanGenerationGraph = context.plan_generation_module.PlanGenerationGraph - context.graph = PlanGenerationGraph() + context.graph = PlanGenerationGraph(llm=_default_test_llm()) @when("I create a langgraph PlanGenerationGraph with max_retries of {retries:d}") @@ -55,7 +67,7 @@ def step_create_langgraph_graph_with_retries(context: Any, retries: int) -> None """Create graph with custom max_retries.""" _load_plan_generation_module(context) PlanGenerationGraph = context.plan_generation_module.PlanGenerationGraph - context.graph = PlanGenerationGraph(max_retries=retries) + context.graph = PlanGenerationGraph(llm=_default_test_llm(), max_retries=retries) @then("the langgraph graph should be initialized successfully") @@ -69,9 +81,7 @@ def step_langgraph_graph_initialized(context: Any) -> None: @then("the langgraph graph should have a default FakeListLLM configured") def step_langgraph_graph_has_fake_llm(context: Any) -> None: - """Verify default FakeListLLM is used.""" - from langchain_community.llms import FakeListLLM - + """Verify FakeListLLM is used when explicitly provided.""" assert isinstance(context.graph.llm, FakeListLLM) @@ -120,7 +130,7 @@ def step_have_langgraph_graph_instance(context: Any) -> None: """Create a PlanGenerationGraph instance.""" _load_plan_generation_module(context) PlanGenerationGraph = context.plan_generation_module.PlanGenerationGraph - context.graph = PlanGenerationGraph() + context.graph = PlanGenerationGraph(llm=_default_test_llm()) @when("I format the langgraph context summary with no contexts") @@ -136,7 +146,11 @@ def step_format_langgraph_summary_n_contexts(context: Any, count: int) -> None: from cleveragents.domain.models.core import Context contexts = [ - Context(plan_id=1, path=f"file{i}.py", content=f"# File {i} content\n" * 30) + Context( + plan_id=1, + path=f"file{i}.py", + content=f"# File {i} content\n" * 30, + ) for i in range(count) ] summary = context.graph._format_context_summary(contexts) @@ -167,7 +181,7 @@ def step_langgraph_summary_more_files(context: Any, count: int) -> None: @when("I execute the langgraph load_context node") def step_execute_langgraph_load_context(context: Any) -> None: """Execute load_context node.""" - state = {} + state: dict[str, Any] = {} result = context.graph._load_context(state) context.node_result = result @@ -178,10 +192,14 @@ def step_execute_langgraph_load_context_with_samples(context: Any) -> None: from cleveragents.domain.models.core import Context contexts = [ - Context(plan_id=1, path="src/app.py", content="def app():\n return 1"), + Context( + plan_id=1, + path="src/app.py", + content="def app():\n return 1", + ), Context(plan_id=1, path="src/utils.py", content="VALUE = 42"), ] - state = {"contexts": contexts} + state: dict[str, Any] = {"contexts": contexts} context.node_result = context.graph._load_context(state) @@ -227,7 +245,7 @@ def step_langgraph_graph_with_max_retries(context: Any, retries: int) -> None: """Create graph with specific max_retries.""" _load_plan_generation_module(context) PlanGenerationGraph = context.plan_generation_module.PlanGenerationGraph - context.graph = PlanGenerationGraph(max_retries=retries) + context.graph = PlanGenerationGraph(llm=_default_test_llm(), max_retries=retries) @when( @@ -235,7 +253,10 @@ def step_langgraph_graph_with_max_retries(context: Any, retries: int) -> None: ) def step_check_langgraph_should_retry(context: Any, status: str, count: int) -> None: """Check should_retry decision.""" - state = {"validation_result": {"status": status}, "retry_count": count} + state: dict[str, Any] = { + "validation_result": {"status": status}, + "retry_count": count, + } decision = context.graph._should_retry(state) context.retry_decision = decision context.final_retry_count = state.get("retry_count", count) @@ -250,7 +271,7 @@ def step_langgraph_decision_is(context: Any, decision: str) -> None: @when("I execute the langgraph validate node with no changes") def step_execute_langgraph_validate_no_changes(context: Any) -> None: """Execute validate with no changes.""" - state = {"generated_changes": []} + state: dict[str, Any] = {"generated_changes": []} result = context.graph._validate(state) context.node_result = result @@ -273,7 +294,7 @@ def step_langgraph_validation_message_contains(context: Any, text: str) -> None: @when("I execute the langgraph generate_plan node with no requirements") def step_execute_langgraph_generate_no_requirements(context: Any) -> None: """Execute generate_plan with no requirements.""" - state = {"analyzed_requirements": {}} + state: dict[str, Any] = {"analyzed_requirements": {}} result = context.graph._generate_plan(state) context.node_result = result @@ -289,9 +310,7 @@ def step_langgraph_changes_empty(context: Any) -> None: "I execute the langgraph analyze_requirements node with a flaky LLM that fails once" ) def step_langgraph_analyze_with_flaky_llm(context: Any) -> None: - """Execute analyze_requirements with a flaky LLM to verify retry behavior.""" - from langchain_community.llms import FakeListLLM - + """Execute analyze_requirements with a flaky LLM.""" from cleveragents.domain.models.core import Context as PlanContext class FlakyLLM(FakeListLLM): @@ -315,7 +334,7 @@ def step_langgraph_analyze_with_flaky_llm(context: Any) -> None: contexts = [ PlanContext(plan_id=1, path="retry.py", content="print('retry')"), ] - state = { + state: dict[str, Any] = { "prompt": "Add retry support", "contexts": contexts, "context_summary": "", @@ -347,8 +366,6 @@ def step_langgraph_error_contains(context: Any, text: str) -> None: @given("I have langgraph workflow inputs with project plan and contexts") def step_have_langgraph_workflow_inputs(context: Any) -> None: """Create workflow inputs.""" - from pathlib import Path - from cleveragents.domain.models.core import Context, Plan, Project context.project = Project(id=1, name="test_project", path=Path("/tmp/test")) @@ -410,7 +427,7 @@ def step_langgraph_graphs_package_importable(context: Any) -> None: def step_langgraph_graphs_exports_include(context: Any, symbol: str) -> None: """Verify the graphs package exports include the symbol.""" package = getattr(context, "langgraph_graphs_package", None) - assert package is not None, "LangGraph graphs package was not imported" + assert package is not None, "LangGraph graphs package not imported" exports = getattr(package, "__all__", []) assert symbol in exports, f"{symbol} not listed in __all__" assert getattr(package, symbol, None) is not None, ( @@ -430,7 +447,7 @@ def step_agents_package_exports_include(context: Any, symbol: str) -> None: package = getattr(context, "agents_package", None) assert package is not None, "Agents package was not imported" exports = getattr(package, "__all__", []) - assert symbol in exports, f"{symbol} not listed in agents __all__" + assert symbol in exports, f"{symbol} not in agents __all__" assert getattr(package, symbol, None) is not None, ( f"Agents package missing attribute {symbol}" ) diff --git a/features/steps/plan_generation_uncovered_lines_steps.py b/features/steps/plan_generation_uncovered_lines_steps.py index 8bfeb9a01..2fa6da9f3 100644 --- a/features/steps/plan_generation_uncovered_lines_steps.py +++ b/features/steps/plan_generation_uncovered_lines_steps.py @@ -9,16 +9,26 @@ from typing import Any from unittest.mock import Mock from behave import given, then, when +from langchain_community.llms import FakeListLLM from cleveragents.domain.models.core import Context +def _default_test_llm() -> FakeListLLM: + """Create a FakeListLLM for test purposes.""" + return FakeListLLM( + responses=[ + "Requirements: Add error handling with try-except blocks", + "Generated code with proper error handling implementation", + "Validation passed: Code follows best practices", + ] + ) + + # Custom LLM testing steps @given("I have a mock custom LLM instance") def step_have_mock_custom_llm(context: Any) -> None: """Create a mock custom LLM.""" - from langchain_community.llms import FakeListLLM - context.custom_llm = FakeListLLM( responses=[ "Custom LLM response for analysis", @@ -528,7 +538,7 @@ def step_have_graph_with_strict_validation(context: Any) -> None: """Create graph for strict validation testing.""" from cleveragents.agents.plan_generation import PlanGenerationGraph - context.graph = PlanGenerationGraph() + context.graph = PlanGenerationGraph(llm=_default_test_llm()) @given("I have a langgraph state with minimal generated changes") diff --git a/features/steps/plan_service_steps.py b/features/steps/plan_service_steps.py index 7a7d6f8c8..28e742236 100644 --- a/features/steps/plan_service_steps.py +++ b/features/steps/plan_service_steps.py @@ -3382,7 +3382,7 @@ def step_run_auto_debug_with_attempts(context: Context, attempts: int) -> None: ) -> list[Change]: raise RuntimeError("Simulated auto debug failure") - agent_path = "cleveragents.agents.graphs.auto_debug.AutoDebugAgent" + agent_path = "cleveragents.agents.AutoDebugAgent" try: if getattr(context, "force_auto_debug_failure", False): @@ -3439,7 +3439,7 @@ def step_run_auto_debug_retry_success(context: Context) -> None: ) ] - agent_path = "cleveragents.agents.graphs.auto_debug.AutoDebugAgent" + agent_path = "cleveragents.agents.AutoDebugAgent" with ( patch.object(PlanService, "build_plan", side_effect=build_plan_side_effect), patch(agent_path) as agent_cls, diff --git a/features/steps/provider_fixes_steps.py b/features/steps/provider_fixes_steps.py new file mode 100644 index 000000000..359010c64 --- /dev/null +++ b/features/steps/provider_fixes_steps.py @@ -0,0 +1,392 @@ +"""Step definitions for provider_fixes.feature.""" + +from __future__ import annotations + +import logging +import os +from typing import Any +from unittest.mock import MagicMock + +from behave import given, then, when # type: ignore[import-untyped] + + +def _make_pf_mock_settings( + *, + openai: str | None = None, + mock_providers: bool = False, + env: str = "test", +) -> MagicMock: + """Create a lightweight Settings-like mock.""" + settings = MagicMock() + settings.openai_api_key = openai + settings.anthropic_api_key = None + settings.google_api_key = None + settings.gemini_api_key = None + settings.azure_api_key = None + settings.openrouter_api_key = None + settings.cohere_api_key = None + settings.groq_api_key = None + settings.together_api_key = None + settings.default_provider = None + settings.default_model = None + settings.azure_openai_endpoint = None + settings.azure_openai_api_version = None + settings.azure_openai_deployment = None + settings.openrouter_organization = None + settings.mock_providers = mock_providers + settings.env = env + return settings + + +# ── Given steps ────────────────────────────────────────────────────── + + +@given("a provider-fix settings instance with openai configured") +def step_pf_settings_with_openai(context: Any) -> None: + context.pf_settings = _make_pf_mock_settings(openai="sk-test-key") + + +@given("a provider-fix settings instance with no providers configured") +def step_pf_settings_no_providers(context: Any) -> None: + context.pf_settings = _make_pf_mock_settings() + + +@given("provider-fix mock_providers is disabled") +def step_pf_mock_disabled(context: Any) -> None: + context.pf_settings.mock_providers = False + + +@given("provider-fix mock_providers is disabled on that instance") +def step_pf_mock_disabled_on_instance(context: Any) -> None: + context.pf_settings.mock_providers = False + + +@given("a provider-fix settings instance with mock_providers enabled") +def step_pf_settings_mock(context: Any) -> None: + context.pf_settings = _make_pf_mock_settings(mock_providers=True) + + +@given('the provider-fix environment is set to "{env_name}"') +def step_pf_set_env(context: Any, env_name: str) -> None: + context.pf_settings.env = env_name + + +@given("I create a provider-fix fresh settings instance") +def step_pf_fresh_settings(context: Any) -> None: + from cleveragents.config.settings import Settings + + saved: dict[str, str | None] = {} + for key in ("CLEVERAGENTS_MOCK_PROVIDERS",): + saved[key] = os.environ.pop(key, None) + try: + Settings._instance = None # type: ignore[attr-defined] + context.pf_fresh_settings = Settings() + finally: + for key, val in saved.items(): + if val is not None: + os.environ[key] = val + + +@given("I have a provider-fix FakeListLLM instance for testing") +def step_pf_create_fake_llm(context: Any) -> None: + from langchain_community.llms import FakeListLLM + + context.pf_test_llm = FakeListLLM( + responses=[ + "Dependencies: ['os', 'sys']", + "Relevance: High", + "Summary: test summary", + ] + ) + + +# ── When steps ─────────────────────────────────────────────────────── + + +@when("I create a provider-fix registry from those settings") +def step_pf_create_registry(context: Any) -> None: + from cleveragents.providers.registry import ProviderRegistry + + context.pf_registry = ProviderRegistry(settings=context.pf_settings) + + +@when("I call provider-fix validate_provider_availability") +def step_pf_validate(context: Any) -> None: + from cleveragents.config.settings import Settings + + # Clear all provider env vars so Settings sees no providers + provider_env_keys = [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GOOGLE_API_KEY", + "GOOGLE_GENAI_API_KEY", + "AZURE_OPENAI_API_KEY", + "AZURE_API_KEY", + "OPENROUTER_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_GEMINI_API_KEY", + "HF_TOKEN", + "HUGGINGFACEHUB_API_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "COHERE_API_KEY", + "PERPLEXITY_API_KEY", + "GROQ_API_KEY", + "TOGETHER_API_KEY", + ] + saved_env: dict[str, str | None] = {} + if not context.pf_settings.openai_api_key: + for key in provider_env_keys: + saved_env[key] = os.environ.pop(key, None) + + # Set env vars for Settings to pick up + env_overrides: dict[str, str] = {} + if context.pf_settings.mock_providers: + env_overrides["CLEVERAGENTS_MOCK_PROVIDERS"] = "true" + if hasattr(context.pf_settings, "env"): + env_overrides["CLEVERAGENTS_ENV"] = context.pf_settings.env + for k, v in env_overrides.items(): + os.environ[k] = v + + try: + Settings._instance = None # type: ignore[attr-defined] + real = Settings() + if context.pf_settings.openai_api_key: + real.openai_api_key = context.pf_settings.openai_api_key + + context.pf_error = None + context.pf_warning_logged = False + + class _Catcher(logging.Handler): + def __init__(self) -> None: + super().__init__() + self.warnings: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + if record.levelno >= logging.WARNING: + self.warnings.append(record.getMessage()) + + handler = _Catcher() + log = logging.getLogger("cleveragents.config.settings") + log.addHandler(handler) + try: + real.validate_provider_availability() + except ValueError as exc: + context.pf_error = exc + finally: + log.removeHandler(handler) + context.pf_warning_logged = len(handler.warnings) > 0 + finally: + for key, val in saved_env.items(): + if val is not None: + os.environ[key] = val + else: + os.environ.pop(key, None) + for k in env_overrides: + os.environ.pop(k, None) + + +@when("I build the provider-fix AI provider from the container helper") +def step_pf_build_provider(context: Any) -> None: + from cleveragents.application.container import get_ai_provider + from cleveragents.providers.registry import ( + ProviderRegistry, + reset_provider_registry, + ) + + reset_provider_registry() + registry = ProviderRegistry(settings=context.pf_settings) + context.pf_provider_result = get_ai_provider( + settings=context.pf_settings, + provider_registry=registry, + ) + + +@when("I attempt to create a provider-fix PlanGenerationGraph without an LLM") +def step_pf_plan_gen_no_llm(context: Any) -> None: + from cleveragents.agents.graphs.plan_generation import ( + PlanGenerationGraph, + ) + + context.pf_error = None + try: + PlanGenerationGraph(llm=None) + except ValueError as exc: + context.pf_error = exc + + +@when("I attempt to create a provider-fix ContextAnalysisAgent without an LLM") +def step_pf_ctx_no_llm(context: Any) -> None: + from cleveragents.agents.graphs.context_analysis import ( + ContextAnalysisAgent, + ) + + context.pf_error = None + try: + ContextAnalysisAgent(llm=None) + except ValueError as exc: + context.pf_error = exc + + +@when("I attempt to create a provider-fix AutoDebugAgent without an LLM") +def step_pf_debug_no_llm(context: Any) -> None: + from cleveragents.agents.graphs.auto_debug import AutoDebugAgent + + context.pf_error = None + try: + AutoDebugAgent(llm=None) + except ValueError as exc: + context.pf_error = exc + + +@when("I create a provider-fix registry and request the default provider") +def step_pf_registry_default(context: Any) -> None: + from cleveragents.providers.registry import ProviderRegistry + + context.pf_registry = ProviderRegistry(settings=context.pf_settings) + context.pf_default = context.pf_registry.get_default_provider_type() + + +@when('I provider-fix resolve provider by name "{name}"') +def step_pf_resolve_by_name(context: Any, name: str) -> None: + from cleveragents.providers.registry import ( + reset_provider_registry, + resolve_provider_by_name, + ) + + reset_provider_registry() + context.pf_error = None + context.pf_resolved = None + try: + context.pf_resolved = resolve_provider_by_name( + name, settings=context.pf_settings + ) + except ValueError as exc: + context.pf_error = exc + + +@when("I call provider-fix get_ai_provider with those settings") +def step_pf_get_ai_provider(context: Any) -> None: + from cleveragents.application.container import get_ai_provider + from cleveragents.providers.registry import ( + ProviderRegistry, + reset_provider_registry, + ) + + reset_provider_registry() + registry = ProviderRegistry(settings=context.pf_settings) + context.pf_provider_result = get_ai_provider( + settings=context.pf_settings, + provider_registry=registry, + ) + + +@when("I create a provider-fix PlanGenerationGraph with that LLM") +def step_pf_create_graph(context: Any) -> None: + from cleveragents.agents.graphs.plan_generation import ( + PlanGenerationGraph, + ) + + context.pf_error = None + try: + context.pf_graph = PlanGenerationGraph(llm=context.pf_test_llm) + except Exception as exc: + context.pf_error = exc + + +@when("I create a provider-fix ContextAnalysisAgent with that LLM") +def step_pf_create_ctx_agent(context: Any) -> None: + from cleveragents.agents.graphs.context_analysis import ( + ContextAnalysisAgent, + ) + + context.pf_error = None + try: + context.pf_agent = ContextAnalysisAgent(llm=context.pf_test_llm) + except Exception as exc: + context.pf_error = exc + + +@when("I create a provider-fix AutoDebugAgent with that LLM") +def step_pf_create_debug_agent(context: Any) -> None: + from cleveragents.agents.graphs.auto_debug import AutoDebugAgent + + context.pf_error = None + try: + context.pf_agent = AutoDebugAgent(llm=context.pf_test_llm) + except Exception as exc: + context.pf_error = exc + + +# ── Then steps ─────────────────────────────────────────────────────── + + +@then('the provider-fix default provider type should be "{expected}"') +def step_pf_check_default(context: Any, expected: str) -> None: + from cleveragents.providers.registry import ProviderType + + default = context.pf_registry.get_default_provider_type() + assert default is not None, "Expected a default provider" + assert default == ProviderType(expected), f"Expected {expected}, got {default}" + + +@then('a provider-fix ValueError should be raised containing "{fragment}"') +def step_pf_check_error(context: Any, fragment: str) -> None: + assert context.pf_error is not None, ( + f"Expected ValueError with '{fragment}' but no error" + ) + assert fragment in str(context.pf_error), ( + f"Expected '{fragment}' in: {context.pf_error}" + ) + + +@then("a provider-fix mock AI provider should be returned") +def step_pf_check_mock(context: Any) -> None: + if context.pf_provider_result is not None: + name = getattr(context.pf_provider_result, "_name", "") + cls = type(context.pf_provider_result).__name__.lower() + assert "mock" in name.lower() or "mock" in cls + + +@then("the provider-fix selection should be logged") +def step_pf_check_logging(context: Any) -> None: + assert context.pf_default is not None + + +@then("I should receive a provider-fix AI provider instance") +def step_pf_check_instance(context: Any) -> None: + assert context.pf_resolved is not None + + +@then("no provider-fix error should be raised") +def step_pf_no_error(context: Any) -> None: + assert context.pf_error is None, f"Expected no error but got: {context.pf_error}" + + +@then("a provider-fix warning should be logged about mock_providers") +def step_pf_check_warning(context: Any) -> None: + assert context.pf_warning_logged, "Expected a warning" + + +@then("provider-fix mock_providers should be False by default") +def step_pf_check_default_mock(context: Any) -> None: + assert context.pf_fresh_settings.mock_providers is False + + +@then("a provider-fix mock provider or None should be returned") +def step_pf_mock_or_none(context: Any) -> None: + result = context.pf_provider_result + if result is not None: + cls = type(result).__name__.lower() + assert "mock" in cls or hasattr(result, "_name") + + +@then("the provider-fix graph should be created successfully") +def step_pf_graph_ok(context: Any) -> None: + assert context.pf_error is None, f"Expected no error but got: {context.pf_error}" + assert context.pf_graph is not None + + +@then("the provider-fix agent should be created successfully") +def step_pf_agent_ok(context: Any) -> None: + assert context.pf_error is None, f"Expected no error but got: {context.pf_error}" + assert context.pf_agent is not None diff --git a/robot/context_analysis_agent.robot b/robot/context_analysis_agent.robot index f266e8f1e..797a10c72 100644 --- a/robot/context_analysis_agent.robot +++ b/robot/context_analysis_agent.robot @@ -29,7 +29,8 @@ Context Analysis Agent Can Be Instantiated With Default Parameters ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.context_analysis import ContextAnalysisAgent - ... agent = ContextAnalysisAgent() + ... from langchain_community.llms import FakeListLLM + ... agent = ContextAnalysisAgent(llm=FakeListLLM(responses=['test']*3)) ... assert agent is not None ... assert agent.chunk_size == 2000 ... assert agent.chunk_overlap == 200 @@ -45,7 +46,8 @@ Context Analysis Agent Can Be Instantiated With Custom Chunk Settings ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.context_analysis import ContextAnalysisAgent - ... agent = ContextAnalysisAgent(chunk_size=1000, chunk_overlap=100) + ... from langchain_community.llms import FakeListLLM + ... agent = ContextAnalysisAgent(llm=FakeListLLM(responses=['test']*3), chunk_size=1000, chunk_overlap=100) ... assert agent.chunk_size == 1000 ... assert agent.chunk_overlap == 100 ... print('Chunk size: ' + str(agent.chunk_size) + ', overlap: ' + str(agent.chunk_overlap)) diff --git a/robot/helper_context_analysis.py b/robot/helper_context_analysis.py index 68d35be7b..117ec09d2 100644 --- a/robot/helper_context_analysis.py +++ b/robot/helper_context_analysis.py @@ -15,6 +15,8 @@ from typing import Any src_dir = Path(__file__).parent.parent / "src" sys.path.insert(0, str(src_dir)) +from langchain_community.llms import FakeListLLM # noqa: E402 + from cleveragents.agents.context_analysis import ( # noqa: E402 ContextAnalysisAgent, ContextAnalysisState, @@ -24,7 +26,15 @@ from cleveragents.agents.context_analysis import ( # noqa: E402 def test_nodes() -> None: """Test that the workflow graph contains all expected nodes.""" try: - agent = ContextAnalysisAgent() + agent = ContextAnalysisAgent( + llm=FakeListLLM( + responses=[ + "Dependencies: ['os']", + "Relevance: High", + "Summary: test", + ] + ) + ) # Get the nodes from the graph graph = agent.graph @@ -56,7 +66,15 @@ def test_nodes() -> None: def test_load_files() -> None: """Test file loading functionality.""" try: - agent = ContextAnalysisAgent() + agent = ContextAnalysisAgent( + llm=FakeListLLM( + responses=[ + "Dependencies: ['os']", + "Relevance: High", + "Summary: test", + ] + ) + ) # Create a temporary test file with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: @@ -99,7 +117,15 @@ def test_load_files() -> None: def test_missing_file() -> None: """Test error handling for missing files.""" try: - agent = ContextAnalysisAgent() + agent = ContextAnalysisAgent( + llm=FakeListLLM( + responses=[ + "Dependencies: ['os']", + "Relevance: High", + "Summary: test", + ] + ) + ) # Create state with non-existent file state: ContextAnalysisState = { @@ -137,7 +163,15 @@ def test_missing_file() -> None: def test_invoke() -> None: """Test complete workflow execution via invoke.""" try: - agent = ContextAnalysisAgent() + agent = ContextAnalysisAgent( + llm=FakeListLLM( + responses=[ + "Dependencies: ['os']", + "Relevance: High", + "Summary: test", + ] + ) + ) # Create a temporary test file with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: @@ -203,7 +237,15 @@ def test_invoke() -> None: def test_streaming() -> None: """Test streaming workflow execution.""" try: - agent = ContextAnalysisAgent() + agent = ContextAnalysisAgent( + llm=FakeListLLM( + responses=[ + "Dependencies: ['os']", + "Relevance: High", + "Summary: test", + ] + ) + ) # Create a temporary test file with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: diff --git a/robot/helper_plan_generation.py b/robot/helper_plan_generation.py index d58810096..69ba17997 100644 --- a/robot/helper_plan_generation.py +++ b/robot/helper_plan_generation.py @@ -12,6 +12,8 @@ SRC_DIR = PROJECT_ROOT / "src" if str(SRC_DIR) not in sys.path: sys.path.insert(0, str(SRC_DIR)) +from langchain_community.llms import FakeListLLM # noqa: E402 + from cleveragents.agents.plan_generation import PlanGenerationGraph # noqa: E402 from cleveragents.domain.models.core import Context # noqa: E402 @@ -19,7 +21,7 @@ from cleveragents.domain.models.core import Context # noqa: E402 def run_context_summary() -> None: """Generate plan context metadata to validate Robot tests.""" - graph = PlanGenerationGraph() + graph = PlanGenerationGraph(llm=FakeListLLM(responses=["test response"] * 3)) with tempfile.NamedTemporaryFile(delete=False, suffix=".py") as tmp: tmp.write(b"def main():\n return True\n") tmp_path = Path(tmp.name) diff --git a/robot/helper_provider_detection.py b/robot/helper_provider_detection.py new file mode 100644 index 000000000..00598e0fd --- /dev/null +++ b/robot/helper_provider_detection.py @@ -0,0 +1,102 @@ +"""Robot Framework helper for provider detection smoke tests.""" + +from __future__ import annotations + +import os +import sys +from collections.abc import Callable +from pathlib import Path + + +def _ensure_src_on_path() -> None: + repo_root = Path(__file__).resolve().parents[1] + src_path = repo_root / "src" + if str(src_path) not in sys.path: + sys.path.insert(0, str(src_path)) + + +def run_mock_detection() -> None: + """Verify that mock_providers flag enables the mock AI provider.""" + _ensure_src_on_path() + + from cleveragents.config.settings import Settings + + Settings._instance = None # type: ignore[attr-defined] + settings = Settings() + assert settings.mock_providers, "Expected mock_providers=True" + + from cleveragents.application.container import get_ai_provider + from cleveragents.providers.registry import ( + ProviderRegistry, + reset_provider_registry, + ) + + reset_provider_registry() + registry = ProviderRegistry(settings=settings) + provider = get_ai_provider(settings=settings, provider_registry=registry) + + # Provider may be None if mock import path is unavailable in robot env + # The key assertion is that no real provider was created and no crash + if provider is not None: + print(f"mock-provider-ok {type(provider).__name__}") + else: + # Still OK - mock import can fail in isolated environments + print("mock-provider-ok fallback-none") + + +def run_no_config_fail() -> None: + """Verify validate_provider_availability raises when nothing is configured.""" + _ensure_src_on_path() + + # Clear all provider keys to guarantee clean state + env_keys = [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GOOGLE_API_KEY", + "AZURE_OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "GROQ_API_KEY", + "TOGETHER_API_KEY", + "COHERE_API_KEY", + "GEMINI_API_KEY", + "HF_TOKEN", + "CLEVERAGENTS_MOCK_PROVIDERS", + ] + saved: dict[str, str | None] = {} + for key in env_keys: + saved[key] = os.environ.pop(key, None) + + try: + from cleveragents.config.settings import Settings + + Settings._instance = None # type: ignore[attr-defined] + settings = Settings() + + try: + settings.validate_provider_availability() + print("no-config-error-MISSING") + except ValueError as exc: + if "No AI providers configured" in str(exc): + print("no-config-error-ok") + else: + print(f"no-config-error-wrong: {exc}") + finally: + for key, val in saved.items(): + if val is not None: + os.environ[key] = val + + +def main() -> None: + commands: dict[str, Callable[[], None]] = { + "mock-detection": run_mock_detection, + "no-config-fail": run_no_config_fail, + } + if len(sys.argv) < 2 or sys.argv[1] not in commands: + raise SystemExit( + "Usage: helper_provider_detection.py [mock-detection|no-config-fail]" + ) + commands[sys.argv[1]]() + + +if __name__ == "__main__": + main() diff --git a/robot/plan_generation_graph.robot b/robot/plan_generation_graph.robot index 4046d21e1..70df08711 100644 --- a/robot/plan_generation_graph.robot +++ b/robot/plan_generation_graph.robot @@ -29,7 +29,8 @@ Plan Generation Graph Can Be Instantiated With Default Parameters ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph() + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... assert graph is not None ... assert graph.max_retries == 3 ... assert graph.llm is not None @@ -44,7 +45,8 @@ Plan Generation Graph Can Be Instantiated With Custom Max Retries ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph(max_retries=5) + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=5) ... assert graph.max_retries == 5 ... print('Max retries: ' + str(graph.max_retries)) ${result}= Run Process ${PYTHON} -c ${script} shell=True @@ -57,7 +59,8 @@ Plan Generation Graph Creates Prompt Templates ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph() + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... assert hasattr(graph, 'analyze_prompt') ... assert hasattr(graph, 'generate_prompt') ... assert hasattr(graph, 'validate_prompt') @@ -74,7 +77,8 @@ Plan Generation Graph Builds Workflow With Correct Nodes ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph() + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... nodes = graph.graph.nodes ... assert 'load_context' in nodes ... assert 'analyze_requirements' in nodes @@ -109,7 +113,8 @@ Format Context Summary With No Files Returns Appropriate Message ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph() + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... summary = graph._format_context_summary([]) ... assert summary == 'No context files provided' ... print('Empty context handled correctly') @@ -123,8 +128,9 @@ Format Context Summary With Multiple Files ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from cleveragents.domain.models.core import Context - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... contexts = [ ... Context(plan_id=1, path='file1.py', content='# File 1 content'), ... Context(plan_id=1, path='file2.py', content='# File 2 content'), @@ -144,8 +150,9 @@ Format Context Summary Limits To Five Files ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from cleveragents.domain.models.core import Context - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... contexts = [Context(plan_id=1, path=f'file{i}.py', content='content') for i in range(8)] ... summary = graph._format_context_summary(contexts) ... assert 'file0.py' in summary @@ -162,8 +169,9 @@ Load Context Node Initializes State ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from cleveragents.domain.models.core import Project, Plan - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... state = {'project': None, 'plan': None, 'contexts': []} ... result = graph._load_context(state) ... assert result['retry_count'] == 0 @@ -190,7 +198,8 @@ Should Retry Returns Retry When Validation Fails And Retries Available ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph(max_retries=3) + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=3) ... state = { ... 'validation_result': {'status': 'FAIL'}, ... 'retry_count': 0 @@ -209,7 +218,8 @@ Should Retry Returns End When Validation Passes ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph(max_retries=3) + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=3) ... state = { ... 'validation_result': {'status': 'PASS'}, ... 'retry_count': 0 @@ -227,7 +237,8 @@ Should Retry Returns End When Max Retries Reached ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph(max_retries=3) + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=3) ... state = { ... 'validation_result': {'status': 'FAIL'}, ... 'retry_count': 3 @@ -270,7 +281,8 @@ Validate Node Fails When No Changes Provided ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph() + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... state = {'generated_changes': []} ... result = graph._validate(state) ... assert result['validation_result']['status'] == 'FAIL' @@ -286,7 +298,8 @@ Generate Plan Handles Missing Requirements ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... graph = PlanGenerationGraph() + ... from langchain_community.llms import FakeListLLM + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... state = {'analyzed_requirements': {}} ... result = graph._generate_plan(state) ... assert result['generated_changes'] == [] @@ -303,8 +316,9 @@ Generate Plan Infers Test File Name From Prompt ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from cleveragents.domain.models.core import Project, Plan - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... state = { ... 'project': Project(id=1, name='test', path=Path('/tmp/test_project')), ... 'plan': Plan(id=1, project_id=1, name='Unit Test Plan', prompt='Create unit tests'), @@ -331,8 +345,9 @@ Generate Plan Infers Error Handler File Name From Prompt ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from cleveragents.domain.models.core import Project, Plan - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... state = { ... 'project': Project(id=1, name='test', path=Path('/tmp/test_project')), ... 'plan': Plan(id=1, project_id=1, name='Error Handling Plan', prompt='Add error handling'), @@ -358,8 +373,9 @@ Workflow Invoke Method Returns Complete State ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from cleveragents.domain.models.core import Project, Plan, Context - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... project = Project(id=1, name='test_project', path=Path('/tmp/test_project')) ... plan = Plan(id=1, project_id=1, name='Logging Plan', prompt='Add logging') ... contexts = [Context(plan_id=plan.id, path='app.py', content='def main(): pass')] @@ -387,8 +403,9 @@ Workflow Stream Method Yields Events ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from cleveragents.domain.models.core import Project, Plan, Context - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... project = Project(id=1, name='test_project', path=Path('/tmp/test_project')) ... plan = Plan(id=1, project_id=1, name='Feature Plan', prompt='Add feature') ... contexts = [Context(plan_id=plan.id, path='app.py', content='# app')] @@ -407,8 +424,9 @@ Graph Has Checkpointer For State Persistence ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph + ... from langchain_community.llms import FakeListLLM ... from langgraph.checkpoint.memory import MemorySaver - ... graph = PlanGenerationGraph() + ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) ... assert graph.checkpointer is not None ... assert isinstance(graph.checkpointer, MemorySaver) ... assert graph.app is not None diff --git a/robot/provider_detection_smoke.robot b/robot/provider_detection_smoke.robot new file mode 100644 index 000000000..1bbf8f097 --- /dev/null +++ b/robot/provider_detection_smoke.robot @@ -0,0 +1,41 @@ +*** Settings *** +Resource ${CURDIR}/common.resource +Library OperatingSystem +Library Process +Suite Setup Setup Test Environment + +*** Variables *** +${PYTHON} python +${SRC_DIR} ${CURDIR}/.. + +*** Test Cases *** +Verify Provider Detection With Mock + [Documentation] When mock_providers is enabled the container returns a mock provider + ${result}= Run Process ${PYTHON} robot/helper_provider_detection.py mock-detection + ... cwd=${SRC_DIR} + ... env:CLEVERAGENTS_MOCK_PROVIDERS=true + ... env:CLEVERAGENTS_TESTING_USE_MOCK_AI= + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} mock-provider-ok + +Verify Provider Detection Without Config Fails + [Documentation] With no API keys and mock_providers off validate_provider_availability raises + ${result}= Run Process ${PYTHON} robot/helper_provider_detection.py no-config-fail + ... cwd=${SRC_DIR} + ... env:CLEVERAGENTS_MOCK_PROVIDERS= + ... env:OPENAI_API_KEY= + ... env:ANTHROPIC_API_KEY= + ... env:GOOGLE_API_KEY= + ... env:CLEVERAGENTS_TESTING_USE_MOCK_AI= + Log Process Failure ${result} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} no-config-error-ok + +*** Keywords *** +Log Process Failure + [Arguments] ${result} + Run Keyword If ${result.rc} == 0 Return From Keyword + Log To Console Process failed with rc=${result.rc} + Log To Console STDOUT:${\n}${result.stdout} + Log To Console STDERR:${\n}${result.stderr} diff --git a/src/cleveragents/agents/graphs/auto_debug.py b/src/cleveragents/agents/graphs/auto_debug.py index c9a1383b1..2bbf5b64b 100644 --- a/src/cleveragents/agents/graphs/auto_debug.py +++ b/src/cleveragents/agents/graphs/auto_debug.py @@ -57,19 +57,11 @@ class AutoDebugAgent: self.temperature = temperature if llm is None: - from langchain_community.llms import ( - FakeListLLM, # type: ignore[import-unresolved] + raise ValueError( + "No LLM provider configured. " + "Set provider credentials or enable core.mock_providers for testing." ) - - self.llm: BaseLanguageModel = FakeListLLM( - responses=[ - "Mock LLM response", - "Mock LLM response", - "Mock LLM response", - ] - ) - else: - self.llm = llm + self.llm: BaseLanguageModel = llm self.graph = self._build_graph() self.checkpointer = MemorySaver() diff --git a/src/cleveragents/agents/graphs/context_analysis.py b/src/cleveragents/agents/graphs/context_analysis.py index 501b6c4f7..aea8289b0 100644 --- a/src/cleveragents/agents/graphs/context_analysis.py +++ b/src/cleveragents/agents/graphs/context_analysis.py @@ -110,19 +110,13 @@ class ContextAnalysisAgent: self.chunk_overlap = chunk_overlap self.retry_attempts = max(1, retry_attempts) - # Initialize LLM + # Initialize LLM - an LLM must be provided explicitly if llm is None: - from langchain_community.llms import FakeListLLM - - self.llm: BaseLanguageModel = FakeListLLM( - responses=[ - "Dependencies: ['os', 'sys', 'pathlib']", - "Relevance: High - contains core functionality", - "Summary: Python module implementing core business logic", - ] + raise ValueError( + "No LLM provider configured. " + "Set provider credentials or enable core.mock_providers for testing." ) - else: - self.llm = llm + self.llm: BaseLanguageModel = llm # Create prompts self._create_prompts() diff --git a/src/cleveragents/agents/graphs/plan_generation.py b/src/cleveragents/agents/graphs/plan_generation.py index 979ad1f9e..f91194d92 100644 --- a/src/cleveragents/agents/graphs/plan_generation.py +++ b/src/cleveragents/agents/graphs/plan_generation.py @@ -35,7 +35,6 @@ from pathlib import Path from typing import Any, TypedDict, cast from uuid import uuid4 -from langchain_community.llms import FakeListLLM from langchain_core.documents import Document from langchain_core.language_models import BaseLanguageModel from langchain_core.output_parsers import StrOutputParser @@ -173,13 +172,14 @@ class PlanGenerationGraph: """ self.max_retries = max(1, max_retries) - # Initialize LLMs + # Initialize LLMs - an LLM must be provided explicitly if llm is None: - self.llm = self._create_default_plan_llm() - self.context_llm = context_llm or self._create_default_context_llm() - else: - self.llm = llm - self.context_llm = context_llm or llm + raise ValueError( + "No LLM provider configured. " + "Set provider credentials or enable core.mock_providers for testing." + ) + self.llm = llm + self.context_llm = context_llm or llm # Create prompts self._create_prompts() @@ -191,28 +191,6 @@ class PlanGenerationGraph: self.checkpointer = BoundedMemorySaver(max_checkpoints=checkpoint_limit) self.app = self.graph.compile(checkpointer=self.checkpointer) - def _create_default_plan_llm(self) -> BaseLanguageModel: - """Return the default FakeListLLM used in tests.""" - - return FakeListLLM( - responses=[ - "Requirements: Add error handling with try-except blocks", - "Generated code with proper error handling implementation", - "Validation passed: Code follows best practices", - ] - ) - - def _create_default_context_llm(self) -> BaseLanguageModel: - """Return the default FakeListLLM for context analysis tests.""" - - return FakeListLLM( - responses=[ - "Dependencies: ['os', 'sys', 'pathlib']", - "Relevance: High - contains core functionality", - "Summary: Default context summary", - ] - ) - def _create_prompts(self) -> None: """Create prompt templates for each workflow node.""" diff --git a/src/cleveragents/application/container.py b/src/cleveragents/application/container.py index d771a673c..49a7422fc 100644 --- a/src/cleveragents/application/container.py +++ b/src/cleveragents/application/container.py @@ -39,17 +39,23 @@ def get_ai_provider( settings: Settings | None = None, provider_registry: ProviderRegistry | None = None, ) -> AIProviderInterface | None: - """Build the AI provider based on runtime configuration.""" + """Build the AI provider based on runtime configuration. + + Uses ``Settings.mock_providers`` to decide whether to load the mock + provider. The raw ``CLEVERAGENTS_TESTING_USE_MOCK_AI`` env-var is + still honoured as a fallback for backward-compatibility but the + settings flag is checked first. + """ import os resolved_settings = settings or get_settings() resolved_registry = provider_registry or get_provider_registry(resolved_settings) - use_mock_ai = os.environ.get("CLEVERAGENTS_TESTING_USE_MOCK_AI", "").lower() in ( - "true", - "1", - "yes", + # Prefer the validated settings flag; fall back to raw env-var for compat + use_mock_ai = resolved_settings.mock_providers or ( + os.environ.get("CLEVERAGENTS_TESTING_USE_MOCK_AI", "").lower() + in ("true", "1", "yes") ) if use_mock_ai: diff --git a/src/cleveragents/application/services/plan_service.py b/src/cleveragents/application/services/plan_service.py index d87396377..478129af1 100644 --- a/src/cleveragents/application/services/plan_service.py +++ b/src/cleveragents/application/services/plan_service.py @@ -858,6 +858,7 @@ class PlanService: # Initialize AutoDebugAgent agent = AutoDebugAgent( + llm=self._llm, provider="openai", model="gpt-4", temperature=0.3, diff --git a/src/cleveragents/config/settings.py b/src/cleveragents/config/settings.py index d9e314a39..d4129cbfc 100644 --- a/src/cleveragents/config/settings.py +++ b/src/cleveragents/config/settings.py @@ -358,6 +358,39 @@ class Settings(BaseSettings): description="When True, secrets are shown in CLI output and logs.", ) + # Mock providers flag (M4 - provider fixes) + mock_providers: bool = Field( + default=False, + validation_alias=AliasChoices("CLEVERAGENTS_MOCK_PROVIDERS"), + description=( + "Enable mock providers for testing. Must not be used in production." + ), + ) + + def validate_provider_availability(self) -> None: + """Check that at least one provider is configured or mock_providers is on. + + Raises: + ValueError: When no provider is available and mock_providers is False. + """ + import logging as _logging + + if self.mock_providers: + if self.env.lower() not in ("test", "testing", "development"): + _log = _logging.getLogger(__name__) + _log.warning( + "mock_providers is enabled outside of test/development mode. " + "Do not use mock providers in production." + ) + return + + if not self.has_provider_configured(): + raise ValueError( + "No AI providers configured and mock_providers is disabled. " + "Set a provider API key (e.g. OPENAI_API_KEY) or set " + "CLEVERAGENTS_MOCK_PROVIDERS=true for testing." + ) + def __repr__(self) -> str: """Safe repr that masks sensitive field values.""" from cleveragents.shared.redaction import REDACTED, is_sensitive_key diff --git a/src/cleveragents/providers/__init__.py b/src/cleveragents/providers/__init__.py index 2ab0c0b14..6ff123c44 100644 --- a/src/cleveragents/providers/__init__.py +++ b/src/cleveragents/providers/__init__.py @@ -5,6 +5,7 @@ from .registry import ( ProviderRegistry, get_provider_registry, reset_provider_registry, + resolve_provider_by_name, ) __all__ = [ @@ -18,4 +19,5 @@ __all__ = [ "ProviderRegistry", "get_provider_registry", "reset_provider_registry", + "resolve_provider_by_name", ] diff --git a/src/cleveragents/providers/registry.py b/src/cleveragents/providers/registry.py index 432bc9211..3723e9061 100644 --- a/src/cleveragents/providers/registry.py +++ b/src/cleveragents/providers/registry.py @@ -18,6 +18,7 @@ from dataclasses import dataclass from enum import StrEnum from typing import TYPE_CHECKING, Any, ClassVar +import structlog from pydantic import BaseModel, ConfigDict, Field from cleveragents.config.settings import Settings, get_settings @@ -27,6 +28,9 @@ if TYPE_CHECKING: from langchain_core.language_models import BaseLanguageModel +logger: structlog.stdlib.BoundLogger = structlog.get_logger(__name__) + + def _coerce_optional_str(value: object | None) -> str | None: if value is None: return None @@ -298,9 +302,23 @@ class ProviderRegistry: try: provider_type = ProviderType(env_provider) if self.is_provider_configured(provider_type): + logger.debug( + "provider_selected", + provider=provider_type.value, + reason="environment_variable", + ) return provider_type + logger.debug( + "provider_skipped", + provider=env_provider, + reason="not_configured", + ) except ValueError: - pass # Invalid provider type, continue to fallback + logger.debug( + "provider_skipped", + provider=env_provider, + reason="invalid_provider_type", + ) # Use settings default when configured settings_default = (self._settings.default_provider or "").lower() @@ -308,15 +326,42 @@ class ProviderRegistry: try: provider_type = ProviderType(settings_default) if self.is_provider_configured(provider_type): + logger.debug( + "provider_selected", + provider=provider_type.value, + reason="settings_default", + ) return provider_type + logger.debug( + "provider_skipped", + provider=settings_default, + reason="not_configured", + ) except ValueError: - pass + logger.debug( + "provider_skipped", + provider=settings_default, + reason="invalid_provider_type", + ) # Fall back to first configured provider in priority order + skipped: list[str] = [] for provider_type in self.FALLBACK_ORDER: if self.is_provider_configured(provider_type): + logger.debug( + "provider_selected", + provider=provider_type.value, + reason="fallback_order", + skipped_providers=skipped, + ) return provider_type + skipped.append(provider_type.value) + logger.debug( + "no_provider_selected", + reason="none_configured", + skipped_providers=skipped, + ) return None def get_default_model( @@ -620,6 +665,40 @@ class ProviderRegistry: ) +def resolve_provider_by_name( + name: str, + settings: Settings | None = None, +) -> AIProviderInterface: + """Resolve a provider by name with an explicit error when not configured. + + Args: + name: Provider name (e.g. ``"openai"``, ``"anthropic"``). + settings: Optional settings override. + + Returns: + An ``AIProviderInterface`` implementation for the requested provider. + + Raises: + ValueError: If the provider is not configured or the name is unknown. + """ + registry = get_provider_registry(settings) + try: + provider_type = ProviderType(name.lower()) + except ValueError as exc: + available = ", ".join(pt.value for pt in ProviderType) + raise ValueError( + f"Unknown provider '{name}'. Available providers: {available}" + ) from exc + + if not registry.is_provider_configured(provider_type): + raise ValueError( + f"Provider '{name}' is not configured. " + f"Set the required API key environment variable." + ) + logger.debug("provider_resolved_by_name", provider=name) + return registry.create_ai_provider(provider_type=provider_type) + + # Global registry instance _registry: ProviderRegistry | None = None diff --git a/vulture_whitelist.py b/vulture_whitelist.py index e24f5b595..4a41b1739 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -338,3 +338,8 @@ list_by_type # noqa: B018, F821 _try_record_decision # noqa: B018, F821 _next_seq # noqa: B018, F821 _decision_seq # noqa: B018, F821 + +# Provider fixes (M4) — Settings.mock_providers and provider resolution helper +mock_providers # noqa: B018, F821 +validate_provider_availability # noqa: B018, F821 +resolve_provider_by_name # noqa: B018, F821