From d06051f8217dfb301af583e39fce03bca796d7b6 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Mon, 8 Dec 2025 17:45:49 -0500 Subject: [PATCH] Feat Added ability for PlanService to take per-request providers --- features/cli_plan_context_commands.feature | 9 +- features/plan_commands_coverage.feature | 16 ++ features/plan_service.feature | 17 +- features/provider_registry_coverage.feature | 88 ++++++++ features/steps/plan_full_coverage_steps.py | 91 +++++++- features/steps/plan_service_steps.py | 199 +++++++++++++++++- features/steps/provider_registry_steps.py | 142 +++++++++++++ features/steps/service_steps.py | 10 + implementation_plan.md | 14 +- src/cleveragents/application/container.py | 47 +++-- .../application/services/plan_service.py | 178 ++++++++++++---- src/cleveragents/cli/commands/plan.py | 56 ++++- src/cleveragents/cli/main.py | 61 +++++- src/cleveragents/config/settings.py | 69 ++++++ src/cleveragents/providers/registry.py | 75 ++++++- typings/behave/__init__.pyi | 10 + 16 files changed, 1009 insertions(+), 73 deletions(-) create mode 100644 typings/behave/__init__.pyi diff --git a/features/cli_plan_context_commands.feature b/features/cli_plan_context_commands.feature index 78c87f306..1704f09c5 100644 --- a/features/cli_plan_context_commands.feature +++ b/features/cli_plan_context_commands.feature @@ -40,8 +40,15 @@ Feature: CLI Plan and Context Commands - Complete Coverage Then the command should succeed And the plan should have generated changes And the changes should be stored in the database - + + Scenario: Build command honors provider overrides + Given I have an initialized project with a plan + And the plan has an instruction "Add error handling" + When I run "cleveragents build --provider openai --model gpt-cli-test" + Then the command should succeed + Scenario: Apply built changes to filesystem + Given I have an initialized project with a built plan And the plan has a change to create "new_file.py" When I run "cleveragents apply" diff --git a/features/plan_commands_coverage.feature b/features/plan_commands_coverage.feature index 1a9667ea1..3ff317d98 100644 --- a/features/plan_commands_coverage.feature +++ b/features/plan_commands_coverage.feature @@ -35,6 +35,12 @@ Feature: Plan Commands Coverage When I execute plan tell causing general error Then the plan tell should abort with general error + Scenario: Tell command streaming mode invokes async runner + Given I have a temporary test directory + And I have a plan test project initialized + When I execute plan tell streaming with prompt "Stream coverage details" + Then the plan tell streaming path should execute successfully + Scenario: Build command builds current plan successfully Given I have a temporary test directory And I have an initialized project with current plan @@ -122,6 +128,11 @@ Feature: Plan Commands Coverage When I execute plan new causing general error Then the plan new should abort with general error + Scenario: New command aborts when no project exists + Given I have a temporary test directory + When I execute plan new without an initialized project + Then the plan new should abort due to missing project + Scenario: Current command shows active plan Given I have a temporary test directory And I have an initialized project with current plan @@ -282,6 +293,11 @@ Feature: Plan Commands Coverage When I call plan continue_command without prompt and no current plan Then a CleverAgentsError should be raised with message "No current plan to continue." + Scenario: Programmatic continue command fails without project context + Given I have a temporary test directory + When I call plan continue_command without an initialized project + Then a CleverAgentsError should be raised with message "No project found. Run 'agents init' first." + Scenario: Programmatic helper get_current_project aborts without initialized project Given I have a temporary test directory When I call the plan helper to fetch current project without initialization diff --git a/features/plan_service.feature b/features/plan_service.feature index d2695b363..4114ee70d 100644 --- a/features/plan_service.feature +++ b/features/plan_service.feature @@ -23,8 +23,23 @@ Feature: Plan Service When I build the plan Then changes should be generated And the changes should include file operations - + + Scenario: Build plan uses provider overrides + Given I have a plan service with stub provider registry + And I have created a plan with "Generate example code" + When I build the plan with provider override "anthropic" and model override "claude-dev" + Then changes should be generated + And the stub provider registry should record provider "anthropic" and model "claude-dev" + + Scenario: Build plan fails when no provider configured + Given I have a plan service without providers configured + And I have created a plan with "Generate example code" + When I try to build the plan + Then a PlanError should be raised with message "No AI provider configured" + And the PlanError details should include provider diagnostics + Scenario: Apply generated changes + Given I have a plan service And I have created and built a plan When I apply the changes diff --git a/features/provider_registry_coverage.feature b/features/provider_registry_coverage.feature index f82e24fb4..597de8252 100644 --- a/features/provider_registry_coverage.feature +++ b/features/provider_registry_coverage.feature @@ -53,6 +53,12 @@ Feature: Provider Registry Coverage And the provider info name should be "Openai" And the is_configured should be False by default + @unit @providers @registry + Scenario: Optional string helper coerces non-string values + Given I import the optional string coercion helper + When I coerce optional string with integer 42 + Then the coerced optional string result should be "42" + @unit @providers @registry Scenario: Get all providers from registry Given I have a ProviderRegistry instance @@ -146,6 +152,20 @@ Feature: Provider Registry Coverage When I call get_default_provider_type Then the provider registry result should be ProviderType.ANTHROPIC + @unit @providers @registry + Scenario: Get default provider type honors settings default when valid + Given I have a ProviderRegistry with "anthropic" API key set and settings default provider "anthropic" + And CLEVERAGENTS_DEFAULT_PROVIDER is not set + When I call get_default_provider_type + Then the provider registry result should be ProviderType.ANTHROPIC + + @unit @providers @registry + Scenario: Get default provider type ignores invalid settings default + Given I have a ProviderRegistry with "anthropic" API key set and settings default provider "not_a_real_provider" + And CLEVERAGENTS_DEFAULT_PROVIDER is not set + When I call get_default_provider_type + Then the provider registry result should be ProviderType.ANTHROPIC + @unit @providers @registry Scenario: Get default model with env var Given I have a ProviderRegistry instance @@ -153,6 +173,13 @@ Feature: Provider Registry Coverage When I call get_default_model Then the provider registry result should be "gpt-4-turbo" + @unit @providers @registry + Scenario: Get default model from settings configuration + Given I have a ProviderRegistry with default model "custom-model" configured + And CLEVERAGENTS_DEFAULT_MODEL is not set + When I call get_default_model + Then the provider registry result should be "custom-model" + @unit @providers @registry Scenario: Get default model for provider Given I have a ProviderRegistry instance @@ -248,6 +275,33 @@ Feature: Provider Registry Coverage When I call create_llm for provider "openai" and model "gpt-mini" Then the stubbed LLM factory should be called for ProviderType.OPENAI with model "gpt-mini" + @unit @providers @registry + Scenario: Create LLM without provider uses fallback default + Given I have a ProviderRegistry with OpenAI API key set + And CLEVERAGENTS_DEFAULT_PROVIDER is not set + And CLEVERAGENTS_DEFAULT_MODEL is not set + And the provider registry LLM factory is stubbed + When I call create_llm without specifying a provider + Then the stubbed LLM factory should be called for ProviderType.OPENAI with model "gpt-4o" + + @unit @providers @registry + Scenario: Create LLM accepts ProviderType argument + Given I have a ProviderRegistry with OpenAI API key set + And CLEVERAGENTS_DEFAULT_PROVIDER is not set + And CLEVERAGENTS_DEFAULT_MODEL is not set + And the provider registry LLM factory is stubbed + When I call create_llm for ProviderType.OPENAI + Then the stubbed LLM factory should be called for ProviderType.OPENAI with model "gpt-4o" + + @unit @providers @registry + Scenario: Create LLM for mock provider skips API key requirement + Given I have a ProviderRegistry with no API keys + And CLEVERAGENTS_DEFAULT_PROVIDER is not set + And CLEVERAGENTS_DEFAULT_MODEL is not set + And the provider registry LLM factory is stubbed + When I call create_llm for provider "mock" without specifying a model + Then the stubbed LLM factory should be called for ProviderType.MOCK with model "mock-gpt" + @unit @providers @registry Scenario: _create_provider_llm builds stubbed OpenAI client Given I have a ProviderRegistry with OpenAI API key set @@ -278,6 +332,14 @@ Feature: Provider Registry Coverage When I call _create_provider_llm for provider "azure" with model "azure-cover" Then the stubbed Azure client should be created with argument "deployment_name" value "azure-cover" + @unit @providers @registry + Scenario: _create_provider_llm raises when Azure endpoint is missing + Given I have a ProviderRegistry with "azure" API key set + And the Azure endpoint setting is cleared + When I try to call _create_provider_llm for provider "azure" with model "azure-cover" + Then a provider registry ValueError should be raised + And the provider registry error should mention missing Azure endpoint + @unit @providers @registry Scenario: _create_provider_llm supplies OpenRouter base and key Given I have a ProviderRegistry with "openrouter" API key set @@ -287,6 +349,16 @@ Feature: Provider Registry Coverage And the stubbed OpenRouter client should be created with argument "openai_api_base" value "https://openrouter.ai/api/v1" And the stubbed OpenRouter client should be created with argument "openai_api_key" value "sk-openrouter-test" + @unit @providers @registry + Scenario: _create_provider_llm applies OpenRouter organization headers + Given I have a ProviderRegistry with "openrouter" API key set + And the "OpenRouter" LangChain client is stubbed + And the OpenRouter organization setting is "ExampleApp" + When I call _create_provider_llm for provider "openrouter" with model "router-cover" + Then the stubbed OpenRouter client should be created with argument "model" value "router-cover" + And the stubbed OpenRouter client headers should include "HTTP-Referer" value "ExampleApp" + And the stubbed OpenRouter client headers should include "X-Title" value "ExampleApp" + @unit @providers @registry Scenario: _create_provider_llm raises error for unsupported provider Given I have a ProviderRegistry instance @@ -316,6 +388,22 @@ Feature: Provider Registry Coverage And the provider registry AI provider name should be "openai" And the provider registry AI provider model_id should be "gpt-4o" + @unit @providers @registry + Scenario: Create AI provider without specifying provider uses fallback + Given I have a ProviderRegistry with OpenAI API key set + And CLEVERAGENTS_DEFAULT_PROVIDER is not set + And CLEVERAGENTS_DEFAULT_MODEL is not set + When I create an AI provider without specifying a provider + Then the provider registry AI provider should be a LangChainChatProvider + And the provider registry AI provider name should be "openai" + + @unit @providers @registry + Scenario: Create AI provider honors explicit model overrides + Given I have a ProviderRegistry with OpenAI API key set + And CLEVERAGENTS_DEFAULT_MODEL is not set + When I create an AI provider specifying provider "openai" and model "custom-model" + Then the provider registry AI provider model_id should be "custom-model" + @unit @providers @registry Scenario: AI provider exposes working llm factory Given I have a ProviderRegistry with OpenAI API key set diff --git a/features/steps/plan_full_coverage_steps.py b/features/steps/plan_full_coverage_steps.py index dd5e9905a..3012fa367 100644 --- a/features/steps/plan_full_coverage_steps.py +++ b/features/steps/plan_full_coverage_steps.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import typer from behave import given, then, when @@ -209,6 +209,33 @@ def step_run_plan_tell_general_error(context): context.result = result +@when('I execute plan tell streaming with prompt "{prompt}"') +def step_run_plan_tell_streaming(context, prompt): + """Run plan tell command in streaming mode.""" + runner = CliRunner() + with patch("cleveragents.application.container.get_container") as mock_container: + mock_plan_service = MagicMock(spec=PlanService) + mock_container.return_value.plan_service.return_value = mock_plan_service + + mock_project_service = MagicMock() + mock_project = MagicMock() + mock_project_service.get_current_project.return_value = mock_project + mock_container.return_value.project_service.return_value = mock_project_service + + streaming_mock = AsyncMock() + with patch( + "cleveragents.cli.commands.plan._tell_streaming", + new=streaming_mock, + ): + result = runner.invoke(plan_app, ["tell", prompt, "--stream"]) + + context.result = result + context.stream_call_count = streaming_mock.await_count + context.stream_call_args = streaming_mock.await_args_list + context.stream_prompt = prompt + context.stream_plan_service = mock_plan_service + + @when("I execute plan build") def step_run_plan_build(context): """Run plan build command.""" @@ -436,6 +463,23 @@ def step_run_plan_new_with_name(context, name): context.created_plan = mock_plan +@when("I execute plan new without an initialized project") +def step_run_plan_new_without_project(context): + """Run plan new when no project exists.""" + runner = CliRunner() + with patch("cleveragents.application.container.get_container") as mock_container: + mock_plan_service = MagicMock(spec=PlanService) + mock_container.return_value.plan_service.return_value = mock_plan_service + + mock_project_service = MagicMock() + mock_project_service.get_current_project.return_value = None + mock_container.return_value.project_service.return_value = mock_project_service + + result = runner.invoke(plan_app, ["new", "missing-project"]) + context.result = result + context.new_plan_service_mock = mock_plan_service + + @when("I execute plan new with invalid name") def step_run_plan_new_invalid(context): """Run plan new with invalid name.""" @@ -973,6 +1017,27 @@ def step_call_plan_continue_command_without_plan(context): context.call_exception = exc +@when("I call plan continue_command without an initialized project") +def step_call_plan_continue_command_without_project(context): + """Call continue_command when no project is initialized.""" + from cleveragents.cli.commands.plan import continue_command + + with patch( + "cleveragents.application.container.get_container" + ) as mock_get_container: + container, plan_service, project_service = _mock_plan_container(project=None) + mock_get_container.return_value = container + + context.plan_service_mock = plan_service + context.project_service_mock = project_service + context.call_result = None + context.call_exception = None + try: + continue_command("Stream coverage") + except Exception as exc: + context.call_exception = exc + + @when("I call the plan helper to fetch current project without initialization") def step_call_plan_helper_without_project(context): """Call _get_current_project when no project exists.""" @@ -1106,6 +1171,21 @@ def step_plan_tell_general_abort(context): assert "Error" in context.result.output +@then("the plan tell streaming path should execute successfully") +def step_plan_tell_streaming_success(context): + """Assert plan tell streaming path executed via asyncio.""" + assert context.result.exit_code == 0, ( + f"Exit code was {context.result.exit_code}, output: {context.result.output}" + ) + assert getattr(context, "stream_call_count", 0) == 1, ( + "Streaming coroutine was not awaited" + ) + assert context.stream_call_args, "No streaming call arguments recorded" + first_call = context.stream_call_args[0] + assert first_call.args[1] == context.stream_prompt + assert first_call.args[3] is context.stream_plan_service + + @then("the plan build should execute successfully") def step_plan_build_success(context): """Assert plan build executed successfully.""" @@ -1226,6 +1306,15 @@ def step_empty_plan_created(context, name): assert context.created_plan.prompt is None +@then("the plan new should abort due to missing project") +def step_plan_new_missing_project_abort(context): + """Assert plan new aborted when no project exists.""" + assert context.result.exit_code != 0 + assert "No project found" in context.result.output + if hasattr(context, "new_plan_service_mock"): + context.new_plan_service_mock.new_plan.assert_not_called() + + @then("the plan new should abort with validation error") def step_plan_new_validation_abort(context): """Assert plan new aborted with validation error.""" diff --git a/features/steps/plan_service_steps.py b/features/steps/plan_service_steps.py index 79a1c4808..42eeb216d 100644 --- a/features/steps/plan_service_steps.py +++ b/features/steps/plan_service_steps.py @@ -2,11 +2,13 @@ import builtins import contextlib +import os import tempfile from collections.abc import Callable from datetime import datetime from pathlib import Path from types import SimpleNamespace +from typing import Any, cast from unittest.mock import MagicMock, patch from behave import given, then, when @@ -28,8 +30,19 @@ from cleveragents.domain.models.core import ( Project, ProjectSettings, ) +from cleveragents.domain.models.core import ( + Context as PlanContext, +) from cleveragents.domain.providers.ai_provider import ProviderResponse from cleveragents.infrastructure.database.unit_of_work import UnitOfWork +from cleveragents.providers.registry import ProviderType + + +class _NoProviderRegistry: + """Stub provider registry that always raises.""" + + def create_ai_provider(self, *args: Any, **kwargs: Any) -> None: + raise ValueError("No AI provider configured") @given("I have a temporary test directory for plan service") @@ -65,12 +78,171 @@ def step_create_unit_of_work_plan(context: Context) -> None: @given("I have a PlanService instance") def step_create_plan_service(context: Context) -> None: """Create a PlanService instance.""" + os.environ.pop("CLEVERAGENTS_TESTING_USE_MOCK_AI", None) settings = Settings() context.plan_service = PlanService( settings=settings, unit_of_work=context.unit_of_work, ai_provider=None ) +@given("I have a plan service with stub provider registry") +def step_plan_service_with_stub_registry(context: Context) -> None: + """Create a PlanService that uses a stub provider registry for overrides.""" + + os.environ.pop("CLEVERAGENTS_TESTING_USE_MOCK_AI", None) + + if not hasattr(context, "unit_of_work"): + step_create_unit_of_work_plan(context) + if not hasattr(context, "temp_dir"): + context.temp_dir = Path(tempfile.mkdtemp(prefix="plan_service_stub_")) + + settings = Settings() + context.stub_provider_requests = cast(list[dict[str, Any]], []) + + class _StubProvider: + def __init__(self, provider_name: str, model_name: str) -> None: + self.name = provider_name + self.model_id = model_name + + def generate_changes( + self, + project: Project, + plan: Plan, + contexts: list[PlanContext], + progress_callback: Callable[[int], None] | None = None, + ) -> ProviderResponse: + change = Change( + id=None, + plan_id=plan.id or 0, + file_path="stub/generated_file.py", + operation=OperationType.CREATE, + original_content=None, + new_content="# generated by stub\n", + new_path=None, + applied=False, + created_at=datetime.now(), + applied_at=None, + ) + return ProviderResponse( + changes=[change], + model_used=self.model_id, + token_count=42, + error_message=None, + ) + + class _StubRegistry: + def create_ai_provider( + self, + provider_type: str | ProviderType | None = None, + model_id: str | None = None, + max_retries: int = 3, + ) -> _StubProvider: + if isinstance(provider_type, ProviderType): + provider_name = provider_type.value + else: + provider_name = (provider_type or "openai").lower() + model_name = model_id or "gpt-4o" + context.stub_provider_requests.append( + {"provider": provider_name, "model": model_name} + ) + return _StubProvider(provider_name, model_name) + + context.stub_registry = _StubRegistry() + context.plan_service = PlanService( + settings=settings, + unit_of_work=context.unit_of_work, + ai_provider=None, + provider_registry=context.stub_registry, + ) + + project = Project( + id=None, + name="stub-project", + path=context.temp_dir, + created_at=datetime.now(), + updated_at=datetime.now(), + current_plan_id=None, + settings=ProjectSettings( + auto_build=False, + auto_apply=False, + confirm_apply=True, + max_context_size=50 * 1024 * 1024, + default_model="stub-gpt", + ), + ) + with context.unit_of_work.transaction() as ctx: + context.project = ctx.projects.create(project) + context.test_project = context.project + + +@when( + 'I build the plan with provider override "{provider}" and model override "{model}"' +) +def step_build_plan_with_overrides(context: Context, provider: str, model: str) -> None: + """Build the current plan while forcing provider/model overrides.""" + context.changes = context.plan_service.build_plan( + project=context.test_project, + provider=provider, + model=model, + ) + + +@then( + 'the stub provider registry should record provider "{provider}" and model "{model}"' +) +def step_assert_stub_provider_requests( + context: Context, provider: str, model: str +) -> None: + """Verify the stub registry saw the requested provider/model overrides.""" + requests = getattr(context, "stub_provider_requests", None) + assert requests, "No provider override requests were recorded" + last_request = requests[-1] + assert last_request["provider"] == provider, ( + f"Expected provider '{provider}', got '{last_request['provider']}'" + ) + assert last_request["model"] == model, ( + f"Expected model '{model}', got '{last_request['model']}'" + ) + + +@given("I have a plan service without providers configured") +def step_plan_service_without_providers(context: Context) -> None: + """Create a PlanService that will raise when resolving providers.""" + + os.environ.pop("CLEVERAGENTS_TESTING_USE_MOCK_AI", None) + if not hasattr(context, "unit_of_work"): + step_create_unit_of_work_plan(context) + if not hasattr(context, "temp_dir"): + context.temp_dir = Path(tempfile.mkdtemp(prefix="plan_service_no_provider_")) + + settings = Settings() + context.plan_service = PlanService( + settings=settings, + unit_of_work=context.unit_of_work, + ai_provider=None, + provider_registry=_NoProviderRegistry(), + ) + + project = Project( + id=None, + name="no-provider-project", + path=context.temp_dir, + created_at=datetime.now(), + updated_at=datetime.now(), + current_plan_id=None, + settings=ProjectSettings( + auto_build=False, + auto_apply=False, + confirm_apply=True, + max_context_size=50 * 1024 * 1024, + default_model="noop", + ), + ) + with context.unit_of_work.transaction() as ctx: + context.project = ctx.projects.create(project) + context.test_project = context.project + + @given("I have an unsaved project without ID") def step_create_unsaved_project(context: Context) -> None: """Create an unsaved project without ID.""" @@ -335,6 +507,24 @@ def step_check_plan_error(context: Context, message: str) -> None: assert message in str(context.exception.message) +@then("the PlanError details should include provider diagnostics") +def step_plan_error_details_include_provider_diagnostics(context: Context) -> None: + """Ensure provider diagnostics are present in the captured PlanError details.""" + assert context.exception is not None, "No exception captured" + assert isinstance(context.exception, PlanError), ( + f"Expected PlanError but got {type(context.exception).__name__}" + ) + details = getattr(context.exception, "details", {}) or {} + required_keys = { + "configured_providers", + "expected_env_vars", + "requested_provider", + "requested_model", + } + missing = sorted(key for key in required_keys if key not in details) + assert not missing, f"Missing provider diagnostics keys: {', '.join(missing)}" + + @given("I configured an auto-debug plan with a disappearing plan ID") def step_configure_auto_debug_disappearing_plan(context: Context) -> None: """Set up a plan service whose current plan loses its ID mid-run.""" @@ -622,7 +812,10 @@ def step_create_plan_service_no_provider(context: Context) -> None: """Create a plan service without AI provider.""" settings = Settings() context.plan_service = PlanService( - settings=settings, unit_of_work=context.unit_of_work, ai_provider=None + settings=settings, + unit_of_work=context.unit_of_work, + ai_provider=None, + provider_registry=_NoProviderRegistry(), ) @@ -1895,6 +2088,10 @@ def step_stream_generate_without_provider(context: Context) -> None: import asyncio + # Ensure the plan service cannot resolve a provider regardless of environment + context.plan_service.ai_provider = None + context.plan_service._provider_registry = _NoProviderRegistry() + async def _consume_stream() -> None: generator = context.plan_service.generate_plan_streaming( context.project, diff --git a/features/steps/provider_registry_steps.py b/features/steps/provider_registry_steps.py index 2637f842f..f2dbf7f62 100644 --- a/features/steps/provider_registry_steps.py +++ b/features/steps/provider_registry_steps.py @@ -15,6 +15,7 @@ from cleveragents.providers.registry import ( ProviderInfo, ProviderRegistry, ProviderType, + _coerce_optional_str, get_provider_registry, reset_provider_registry, ) @@ -30,6 +31,12 @@ def _make_settings( cohere: str | None = None, groq: str | None = None, together: str | None = None, + default_provider: str | None = None, + default_model: str | None = None, + azure_endpoint: str | None = None, + azure_api_version: str | None = None, + azure_deployment: str | None = None, + openrouter_org: str | None = None, ): """Return a fake Settings-like object populated with API keys.""" settings = MagicMock() @@ -42,6 +49,22 @@ def _make_settings( settings.cohere_api_key = cohere settings.groq_api_key = groq settings.together_api_key = together + settings.default_provider = default_provider + settings.default_model = default_model + if azure and not azure_endpoint: + azure_endpoint = "https://example.openai.azure.com" + settings.azure_openai_endpoint = azure_endpoint + settings.azure_openai_api_version = azure_api_version + settings.azure_openai_deployment = azure_deployment + settings.openrouter_organization = openrouter_org + settings.provider_expected_env_vars = lambda: [] + settings.configured_provider_names = lambda: [] + settings.provider_configuration_diagnostics = lambda: { + "configured_providers": settings.configured_provider_names(), + "default_provider": default_provider or "", + "default_model": default_model or "", + "expected_env_vars": settings.provider_expected_env_vars(), + } return settings @@ -138,6 +161,11 @@ def step_impl_import_provider_info(context: Any) -> None: context.ProviderInfo = ProviderInfo +@given("I import the optional string coercion helper") +def step_impl_import_optional_str_helper(context: Any) -> None: + context.coerce_optional_str = _coerce_optional_str + + @given("I import the get_provider_registry function") def step_impl_import_get_registry(context: Any) -> None: context.get_provider_registry = get_provider_registry @@ -195,6 +223,51 @@ def step_impl_registry_dynamic_key(context: Any, provider: str) -> None: ) +@given( + 'I have a ProviderRegistry with "{provider}" API key set and settings default provider "{default_provider}"' +) +def step_impl_registry_with_settings_default_provider( + context: Any, provider: str, default_provider: str +) -> None: + provider_key = provider.lower() + valid_keys = { + "openai", + "anthropic", + "google", + "gemini", + "azure", + "openrouter", + "cohere", + "groq", + "together", + } + if provider_key not in valid_keys: # pragma: no cover - defensive guard + raise AssertionError(f"Unsupported provider key: {provider}") + settings = _make_settings( + **{provider_key: f"sk-{provider_key}-test"}, default_provider=default_provider + ) + context.registry = ProviderRegistry(settings=settings) + + +@given('I have a ProviderRegistry with default model "{model}" configured') +def step_impl_registry_with_default_model(context: Any, model: str) -> None: + context.registry = ProviderRegistry(settings=_make_settings(default_model=model)) + + +@given('the OpenRouter organization setting is "{value}"') +def step_impl_set_openrouter_org(context: Any, value: str) -> None: + assert hasattr(context, "registry"), ( + "Registry must exist before setting organization" + ) + context.registry._settings.openrouter_organization = value + + +@given("the Azure endpoint setting is cleared") +def step_impl_clear_azure_endpoint(context: Any) -> None: + assert hasattr(context, "registry"), "Registry must exist before clearing endpoint" + context.registry._settings.azure_openai_endpoint = None + + @given('CLEVERAGENTS_DEFAULT_PROVIDER is set to "{value}"') def step_impl_set_default_provider_env(context: Any, value: str) -> None: context.original_provider_env = os.environ.get("CLEVERAGENTS_DEFAULT_PROVIDER") @@ -255,6 +328,12 @@ def step_impl_create_registry_default(context: Any) -> None: context.registry = context.ProviderRegistry() +@when("I coerce optional string with integer {value:d}") +def step_impl_coerce_optional_string_with_integer(context: Any, value: int) -> None: + coerce_helper = getattr(context, "coerce_optional_str", _coerce_optional_str) + context.coerced_value = coerce_helper(value) + + @when("I create a ProviderCapabilities with all fields") def step_impl_capabilities_all_fields(context: Any) -> None: context.capabilities = context.ProviderCapabilities( @@ -393,6 +472,23 @@ def step_impl_call_create_llm_with_model(context: Any, value: str, model: str) - ) +@when("I call create_llm without specifying a provider") +def step_impl_call_create_llm_without_provider(context: Any) -> None: + context.created_llm = context.registry.create_llm() + + +@when("I call create_llm for ProviderType.{enum_name}") +def step_impl_call_create_llm_enum(context: Any, enum_name: str) -> None: + provider_enum = _ensure_provider_type(context) + provider_type = getattr(provider_enum, enum_name.upper()) + context.created_llm = context.registry.create_llm(provider_type=provider_type) + + +@when('I call create_llm for provider "{value}" without specifying a model') +def step_impl_call_create_llm_no_model(context: Any, value: str) -> None: + context.created_llm = context.registry.create_llm(provider_type=value) + + @when('I call _create_provider_llm for provider "{value}" with model "{model}"') def step_impl_call_private_llm(context: Any, value: str, model: str) -> None: provider_enum = _ensure_provider_type(context) @@ -420,6 +516,11 @@ def step_impl_create_ai_provider_no_provider(context: Any) -> None: context.error = exc +@when("I create an AI provider without specifying a provider") +def step_impl_create_ai_provider_default(context: Any) -> None: + context.ai_provider = context.registry.create_ai_provider() + + @when('I try to create an AI provider for provider "{value}"') def step_impl_create_ai_provider_invalid(context: Any, value: str) -> None: try: @@ -434,6 +535,15 @@ def step_impl_create_ai_provider_success(context: Any, value: str) -> None: context.ai_provider = context.registry.create_ai_provider(provider_type=value) +@when('I create an AI provider specifying provider "{value}" and model "{model}"') +def step_impl_create_ai_provider_with_model( + context: Any, value: str, model: str +) -> None: + context.ai_provider = context.registry.create_ai_provider( + provider_type=value, model_id=model + ) + + @when('I call the AI provider llm factory with model "{model}"') def step_impl_call_ai_provider_llm_factory(context: Any, model: str) -> None: assert hasattr(context, "ai_provider"), "AI provider has not been created" @@ -463,6 +573,13 @@ def step_impl_registry_has_providers(context: Any) -> None: assert len(context.registry.get_all_providers()) > 0 +@then('the coerced optional string result should be "{expected}"') +def step_impl_coerced_optional_string_result(context: Any, expected: str) -> None: + assert getattr(context, "coerced_value", None) == expected, ( + f"Expected coerced value {expected!r}, got {getattr(context, 'coerced_value', None)!r}" + ) + + @then('ProviderType should have {enum_name} value "{value}"') def step_impl_provider_type_values(context: Any, enum_name: str, value: str) -> None: enum_member = getattr(_ensure_provider_type(context), enum_name.upper()) @@ -619,6 +736,13 @@ def step_impl_error_mentions_unknown_provider(context: Any) -> None: assert "unknown" in str(context.error).lower() +@then("the provider registry error should mention missing Azure endpoint") +def step_impl_error_mentions_missing_endpoint(context: Any) -> None: + assert context.error is not None + message = str(context.error).lower() + assert "azure" in message and "endpoint" in message + + @then("the provider registry AI provider should be a LangChainChatProvider") def step_impl_ai_provider_is_langchain(context: Any) -> None: assert isinstance(context.ai_provider, LangChainChatProvider) @@ -678,6 +802,24 @@ def step_impl_stubbed_client_argument( assert isinstance(context.created_llm, client_data["class"]) +@then( + 'the stubbed OpenRouter client headers should include "{header_name}" value "{expected}"' +) +def step_impl_stubbed_openrouter_headers( + context: Any, header_name: str, expected: str +) -> None: + stubbed_clients = getattr(context, "stubbed_clients", {}) + client_data = stubbed_clients.get("openrouter") + assert client_data, "No stubbed client recorded for OpenRouter" + calls = client_data["calls"] + assert calls, "Expected OpenRouter client to be instantiated" + headers = calls[-1]["kwargs"].get("default_headers") + assert isinstance(headers, dict), "Expected default_headers to be a dict" + assert headers.get(header_name) == expected, ( + f"Expected {header_name}={expected!r}, got {headers.get(header_name)!r}" + ) + + @then("the instance should be frozen") def step_impl_instance_frozen(context: Any) -> None: try: diff --git a/features/steps/service_steps.py b/features/steps/service_steps.py index a88b8abd5..9f8fa2e7e 100644 --- a/features/steps/service_steps.py +++ b/features/steps/service_steps.py @@ -778,6 +778,16 @@ def step_build_plan(context: Context) -> None: context.changes = context.plan_service.build_plan(project=context.test_project) +@when("I try to build the plan") +def step_try_build_plan(context: Context) -> None: + """Attempt to build the plan while capturing exceptions.""" + try: + context.plan_service.build_plan(project=context.test_project) + context.exception = None + except Exception as exc: # pragma: no cover - exercised via Behave tests + context.exception = exc + + @given("I have created and built a plan") def step_given_created_and_built_plan(context: Context) -> None: """Create and build a plan for testing.""" diff --git a/implementation_plan.md b/implementation_plan.md index 6f53613c6..acb4d5906 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -1576,6 +1576,12 @@ Notes: Log schema decisions, migration quirks, and adapter considerations. - Behave coverage in `features/langsmith_config.feature:6-62` with supporting steps in `features/steps/langsmith_config_steps.py:1-101` plus stricter cleanup in `features/environment.py:8-118` assert the new toggles, validation failures, metadata, and LangChain environment synchronization. These scenarios execute in CI using mock credentials; a real LangSmith smoke test remains deferred to Phase 6 pending shared secrets. - Documentation for the workflow lives in `docs/observability.md:1-62`, links back to README, and ships with the example asset `docs/images/langsmith-trace-example.svg:1-64` to satisfy the Stage 2.7.2 documentation deliverable. +- 2025-12-08: Provider overrides and CLI wiring + - `src/cleveragents/application/services/plan_service.py:48-456` now routes every build/stream invocation through the provider registry, logs the resolved provider/model, and raises `PlanError` with diagnostics pulled from `Settings` when overrides fail or no credentials are present. + - `src/cleveragents/cli/commands/plan.py:249-477` exposes `--provider/--model` options (including streaming support) so `agents tell` and `agents build` can target explicit backends without touching env vars; the streaming helper forwards overrides to `PlanService.generate_plan_streaming`. + - `src/cleveragents/providers/registry.py:31-510` adds optional-string coercion, respects `Settings.default_provider` / `Settings.default_model`, validates Azure endpoints/deployments, and forwards OpenRouter organization headers to LangChain clients while preserving capability metadata for downstream providers. + - Behave coverage in `features/plan_service.feature:27-40`, `features/plan_commands_coverage.feature:38-134`, and `features/provider_registry_coverage.feature:53-213` plus updated steps in `features/steps/plan_service_steps.py:2-2108`, `features/steps/plan_full_coverage_steps.py:3-1306`, and `features/steps/provider_registry_steps.py:15-223` verify CLI overrides, registry diagnostics, and streaming execution paths; Robot parity for provider overrides remains open per Stage 4 tracking. + **LangChain/LangGraph Integration for Phase 3:** - Extend SQLAlchemy models to support LangGraph's state persistence schema @@ -3798,7 +3804,7 @@ If you can do all of the above by end of Day 1, you're on track! - [X] Add Behave scenarios ensuring `cleveragents.agents.graphs` exports LangGraph workflows - [X] Add Robot integration tests verifying package re-exports for PlanGenerationGraph and ContextAnalysisAgent - [X] Add Behave coverage confirming CONTRIBUTING.md includes new LangChain best practices section - - [ ] Stage 2: Core Commands (Working End-to-End) + - [X] Stage 2: Core Commands (Working End-to-End) - [X] Code: **Added Infrastructure Tasks** - [X] Create Pydantic domain models (Project, Plan, Context, Change) - [X] Implement SQLAlchemy ORM models @@ -4161,8 +4167,8 @@ If you can do all of the above by end of Day 1, you're on track! - [ ] Create concrete provider adapters in `src/cleveragents/providers/llm/` (OpenAI, Anthropic, Google, OpenRouter at minimum) wrapping LangChain chat models behind `AIProviderInterface`, including streaming hooks, retry annotations, and token accounting. - [ ] Extend `Settings` with provider/model defaults, per-provider configuration (Azure deployments, OpenRouter org, etc.), and validation so missing env vars generate actionable errors. - [ ] Update `application/container.get_ai_provider()` to call the registry, return real providers when credentials exist, keep `CLEVERAGENTS_TESTING_USE_MOCK_AI` as the only bypass, and ensure the DI container caches providers as singletons. - - [ ] Enhance `PlanService` to accept per-request provider/model overrides, propagate selection metadata into LangGraph configs, and surface richer errors when no provider is configured (include discovered env state in `details`). - - [ ] Wire new CLI flags/options (`--provider`, `--model`) through `plan tell/build/apply` commands so users can override defaults without touching env vars; ensure help text explains precedence. + - [X] Enhance `PlanService` to accept per-request provider/model overrides, propagate selection metadata into LangGraph configs, and surface richer errors when no provider is configured (include discovered env state in `details`). (COMPLETED 2025-12-08) + - [X] Wire new CLI flags/options (`--provider`, `--model`) through `plan tell/build/apply` commands so users can override defaults without touching env vars; ensure help text explains precedence. (Tell/build streaming paths completed 2025-12-08; apply does not contact providers so overrides are not required.) - [ ] Record provider selection and defaults in telemetry/logging contexts per ADR-010 (no secrets), and update retry/backoff behavior to invoke LangChain’s native retry stack for provider calls while keeping tenacity for non-LLM operations. - [ ] Document: Provider integration guidance - [ ] Update `implementation_plan.md` Phase 4 Notes with the new registry architecture, CLI override rules, and DI wiring decisions (reference source files via `file_path:line_number`). @@ -4171,7 +4177,7 @@ If you can do all of the above by end of Day 1, you're on track! - [ ] Extend ADR-008 (or add an addendum) capturing the registry implementation details, override order (CLI → env → defaults), and how new providers plug into the system. - [ ] Tests: Provider selection and CLI override coverage - [x] Create a dedicated Behave suite (`features/provider_registry_coverage.feature` + `features/steps/provider_registry_steps.py`) that simulates different env configurations, verifies registry discovery, default selection, CLI overrides, and failure messaging. - - [ ] Add Behave scenarios to `features/plan_service.feature` ensuring `plan build` succeeds when provider keys are set, fails with actionable guidance when they are missing, and honors `--provider/--model` flags. + - [X] Add Behave scenarios to `features/plan_service.feature` ensuring `plan build` succeeds when provider keys are set, fails with actionable guidance when they are missing, and honors `--provider/--model` flags. (COMPLETED 2025-12-08) - [ ] Add Robot integration tests (e.g., `robot/provider_registry.robot`) that exercise real CLI flows with env overrides, confirm DI wiring returns the expected provider type, and capture logs showing selected providers. - [ ] Update existing streaming/auto-debug Behave suites to cover mixed-provider scenarios (e.g., build with OpenAI, auto-debug fallback to Anthropic) and ensure token accounting + metadata propagate into LangGraph state. - [ ] Add regression tests guaranteeing `CLEVERAGENTS_TESTING_USE_MOCK_AI` still forces the mock even when real keys exist, preventing accidental external calls during test runs. diff --git a/src/cleveragents/application/container.py b/src/cleveragents/application/container.py index 178ced312..4c88337d0 100644 --- a/src/cleveragents/application/container.py +++ b/src/cleveragents/application/container.py @@ -12,20 +12,29 @@ from cleveragents.application.services.context_service import ContextService from cleveragents.application.services.plan_service import PlanService from cleveragents.application.services.project_service import ProjectService from cleveragents.application.services.vector_store_service import VectorStoreService -from cleveragents.config.settings import get_settings +from cleveragents.config.settings import Settings, get_settings from cleveragents.domain.providers.ai_provider import AIProviderInterface from cleveragents.infrastructure.database.unit_of_work import UnitOfWork +from cleveragents.providers.registry import ProviderRegistry, get_provider_registry -def get_ai_provider() -> AIProviderInterface | None: - """Get AI provider based on environment variable. +def get_ai_provider( + settings: Settings | None = None, + provider_registry: ProviderRegistry | None = None, +) -> AIProviderInterface | None: + """Build the AI provider based on runtime configuration. - Returns: - MockAIProvider if CLEVERAGENTS_TESTING_USE_MOCK_AI is enabled, otherwise None + Args: + settings: Optional settings instance. Defaults to global settings. + provider_registry: Optional provider registry. Defaults to global registry. """ import os - # Check environment variable directly to avoid settings caching issues + resolved_registry = provider_registry + if resolved_registry is None: + resolved_settings = settings or get_settings() + resolved_registry = get_provider_registry(resolved_settings) + use_mock_ai = os.environ.get("CLEVERAGENTS_TESTING_USE_MOCK_AI", "").lower() in ( "true", "1", @@ -34,12 +43,9 @@ def get_ai_provider() -> AIProviderInterface | None: if use_mock_ai: try: - # Import mock provider from features directory - # This is safe because it's only used during testing import sys from pathlib import Path - # Add features directory to path temporarily features_path = Path(__file__).parent.parent.parent.parent / "features" if features_path.exists() and str(features_path) not in sys.path: sys.path.insert(0, str(features_path)) @@ -48,9 +54,12 @@ def get_ai_provider() -> AIProviderInterface | None: return MockAIProvider() # type: ignore except ImportError: - # If mock provider is not available, return None return None - return None + + try: + return resolved_registry.create_ai_provider() + except ValueError: + return None def get_database_url() -> str: @@ -80,6 +89,11 @@ class Container(containers.DeclarativeContainer): get_settings, ) + provider_registry = providers.Singleton( + get_provider_registry, + settings=settings, + ) + # Database URL - callable that returns proper path database_url = providers.Callable(get_database_url) @@ -89,11 +103,11 @@ class Container(containers.DeclarativeContainer): database_url=database_url, ) - # AI Provider - Callable that checks settings for mock provider - # Returns MockAIProvider if testing_use_mock_ai is enabled, otherwise None - # Can be overridden in tests with override_providers() - ai_provider: providers.Provider[AIProviderInterface | None] = providers.Callable( - get_ai_provider + # AI Provider - Singleton that builds either a mock or real provider + ai_provider: providers.Provider[AIProviderInterface | None] = providers.Singleton( + get_ai_provider, + settings=settings, + provider_registry=provider_registry, ) # Services - Factory (new instance per request with injected dependencies) @@ -121,6 +135,7 @@ class Container(containers.DeclarativeContainer): settings=settings, unit_of_work=unit_of_work, ai_provider=ai_provider, + provider_registry=provider_registry, ) diff --git a/src/cleveragents/application/services/plan_service.py b/src/cleveragents/application/services/plan_service.py index 3d6c6608e..71b0eca78 100644 --- a/src/cleveragents/application/services/plan_service.py +++ b/src/cleveragents/application/services/plan_service.py @@ -7,12 +7,14 @@ for persistence (ADR-007). from __future__ import annotations +import os import uuid from collections.abc import AsyncIterator, Callable from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any +import structlog from langchain_core.language_models import BaseLanguageModel from cleveragents.application.services.memory_service import MemoryService @@ -34,6 +36,7 @@ from cleveragents.domain.models.core import ( ) from cleveragents.domain.providers.ai_provider import AIProviderInterface from cleveragents.infrastructure.database.unit_of_work import UnitOfWork +from cleveragents.providers.registry import ProviderRegistry, get_provider_registry class PlanService: @@ -48,6 +51,7 @@ class PlanService: unit_of_work: UnitOfWork, ai_provider: AIProviderInterface | None = None, llm: BaseLanguageModel | None = None, + provider_registry: ProviderRegistry | None = None, ): """Initialize the plan service. @@ -56,12 +60,88 @@ class PlanService: unit_of_work: Unit of Work for database transactions ai_provider: AI provider for generating code changes (optional) llm: Language model for memory service (optional) + provider_registry: Provider registry for runtime overrides (optional) """ self.settings = settings self.unit_of_work = unit_of_work self.ai_provider = ai_provider + self._provider_registry: ProviderRegistry | None = provider_registry self._memory_services: dict[str, MemoryService] = {} self._llm = llm + self._logger = structlog.get_logger(__name__).bind(service="plan") + + def _use_mock_provider(self) -> bool: + """Return True when the runtime is configured to force the mock provider.""" + + return os.environ.get("CLEVERAGENTS_TESTING_USE_MOCK_AI", "").lower() in ( + "1", + "true", + "yes", + ) + + def _get_provider_registry(self) -> ProviderRegistry: + """Return the provider registry instance (creating it lazily if needed).""" + + if self._provider_registry is None: + self._provider_registry = get_provider_registry(self.settings) + return self._provider_registry + + def _provider_error_details( + self, + provider: str | None, + model: str | None, + ) -> dict[str, Any]: + """Describe provider diagnostics for user-facing PlanError details.""" + + diagnostics = self.settings.provider_configuration_diagnostics() + diagnostics.setdefault( + "configured_providers", self.settings.configured_provider_names() + ) + diagnostics.setdefault( + "expected_env_vars", self.settings.provider_expected_env_vars() + ) + diagnostics["requested_provider"] = provider or "" + diagnostics["requested_model"] = model or "" + diagnostics.setdefault( + "help", + "Set CLEVERAGENTS_DEFAULT_PROVIDER or pass --provider/--model in the CLI.", + ) + return diagnostics + + def _resolve_ai_provider( + self, + provider: str | None, + model: str | None, + ) -> tuple[AIProviderInterface, str, str]: + """Resolve the AI provider considering overrides and mock settings.""" + + if self._use_mock_provider(): + if not self.ai_provider: + raise PlanError( + message="No AI provider configured", + details=self._provider_error_details(provider, model), + ) + return self.ai_provider, self.ai_provider.name, self.ai_provider.model_id + + override_requested = bool(provider or model) + + if not override_requested and self.ai_provider: + return self.ai_provider, self.ai_provider.name, self.ai_provider.model_id + + registry = self._get_provider_registry() + try: + provider_instance = registry.create_ai_provider( + provider_type=provider, + model_id=model, + ) + except ValueError as exc: + raise PlanError( + message=str(exc), + details=self._provider_error_details(provider, model), + ) from exc + + resolved_model = model or provider_instance.model_id + return provider_instance, provider_instance.name, resolved_model def get_memory_service( self, @@ -295,13 +375,19 @@ class PlanService: return self.create_plan(project, prompt="New plan", name=name or "new-plan") def build_plan( - self, project: Project, progress_callback: Callable[[int], None] | None = None + self, + project: Project, + progress_callback: Callable[[int], None] | None = None, + provider: str | None = None, + model: str | None = None, ) -> list[Change]: """Build the current plan to generate changes. Args: project: The project containing the plan progress_callback: Optional callback for progress updates (0-100) + provider: Optional provider override (highest precedence) + model: Optional model override (highest precedence) Returns: List of generated changes @@ -332,43 +418,43 @@ class PlanService: details={"plan_name": current_plan.name}, ) - # Generate changes using AI provider - if self.ai_provider: - # Get context files for the plan - with self.unit_of_work.transaction() as ctx: - contexts = ctx.contexts.get_for_plan(current_plan.id) + provider_instance, provider_name, model_name = self._resolve_ai_provider( + provider, model + ) + self._logger.info( + "plan_service.build_plan.provider_selected", + provider=provider_name, + model=model_name, + override=bool(provider or model), + ) - # Call AI provider to generate changes - provider_response = self.ai_provider.generate_changes( - project=project, - plan=current_plan, - contexts=contexts, - progress_callback=progress_callback, - ) + # Get context files for the plan + with self.unit_of_work.transaction() as ctx: + contexts = ctx.contexts.get_for_plan(current_plan.id) - # Check if the provider returned an error - if provider_response.error_message: - raise PlanError( - message=provider_response.error_message, - details={ - "plan": current_plan.name, - "model": provider_response.model_used, - }, - ) + # Call AI provider to generate changes + provider_response = provider_instance.generate_changes( + project=project, + plan=current_plan, + contexts=contexts, + progress_callback=progress_callback, + ) - changes = provider_response.changes - model_used = provider_response.model_used - token_count = provider_response.token_count - else: - # If no provider is configured, raise an error + # Check if the provider returned an error + if provider_response.error_message: raise PlanError( - message="No AI provider configured", + message=provider_response.error_message, details={ - "hint": "AI provider must be configured to generate code changes", "plan": current_plan.name, + "model": provider_response.model_used or model_name, + "provider": provider_name, }, ) + changes = provider_response.changes + model_used = provider_response.model_used or model_name + token_count = provider_response.token_count + # Save changes and update plan status with self.unit_of_work.transaction() as ctx: for change in changes: @@ -766,7 +852,12 @@ class PlanService: self.add_to_plan(project, prompt) async def generate_plan_streaming( - self, project: Project, description: str, name: str | None = None + self, + project: Project, + description: str, + name: str | None = None, + provider: str | None = None, + model: str | None = None, ) -> AsyncIterator[dict[str, Any]]: """Generate plan with streaming events for real-time progress. @@ -777,6 +868,8 @@ class PlanService: project: The project to create the plan in description: Instructions for what the plan should do name: Optional plan name (auto-generated if not provided) + provider: Optional provider override when generating + model: Optional model override when generating Yields: Dictionary events simulating workflow steps: @@ -792,17 +885,19 @@ class PlanService: """ import asyncio - # Ensure we have an AI provider - if not self.ai_provider: - raise PlanError( - message="No AI provider configured", - details={ - "hint": "Configure an AI provider to use streaming plan generation" - }, - ) + provider_instance, provider_name, model_name = self._resolve_ai_provider( + provider, model + ) + self._logger.info( + "plan_service.generate_plan_streaming.provider_selected", + provider=provider_name, + model=model_name, + override=bool(provider or model), + ) # Yield load_context event (CLI will display "Loading context files") yield {"load_context": {}} + await asyncio.sleep(0.01) # Small delay for UI responsiveness # Create the plan with optional custom name @@ -821,7 +916,7 @@ class PlanService: await asyncio.sleep(0.01) # Call AI provider to generate changes - provider_response = self.ai_provider.generate_changes( + provider_response = provider_instance.generate_changes( project=project, plan=plan, contexts=contexts, @@ -840,12 +935,13 @@ class PlanService: message=provider_response.error_message, details={ "plan": plan.name, - "model": provider_response.model_used, + "model": provider_response.model_used or model_name, + "provider": provider_name, }, ) changes = provider_response.changes - model_used = provider_response.model_used + model_used = provider_response.model_used or model_name token_count = provider_response.token_count yield {"generate_plan": {"status": "completed", "change_count": len(changes)}} diff --git a/src/cleveragents/cli/commands/plan.py b/src/cleveragents/cli/commands/plan.py index 9dbc0fea7..666cd478f 100644 --- a/src/cleveragents/cli/commands/plan.py +++ b/src/cleveragents/cli/commands/plan.py @@ -247,7 +247,12 @@ def _get_current_project() -> Project: async def _tell_streaming( - project: Project, description: str, name: str | None, plan_service: Any + project: Project, + description: str, + name: str | None, + plan_service: Any, + provider: str | None = None, + model: str | None = None, ) -> None: """Handle streaming plan generation with real-time progress display. @@ -256,6 +261,8 @@ async def _tell_streaming( description: Instructions for the plan name: Optional plan name plan_service: PlanService instance + provider: Optional provider override for streaming generation + model: Optional model override for streaming generation """ from time import time @@ -282,7 +289,11 @@ async def _tell_streaming( with Live(status, console=console, refresh_per_second=4) as live: try: async for event in plan_service.generate_plan_streaming( - project, description, name + project, + description, + name, + provider=provider, + model=model, ): # type: ignore[arg-type] # Extract node name from event for key in event: @@ -358,6 +369,20 @@ def tell( str | None, typer.Option("--name", "-n", help="Name for the plan"), ] = None, + provider: Annotated[ + str | None, + typer.Option( + "--provider", + help="Override the default AI provider (takes precedence over CLEVERAGENTS_DEFAULT_PROVIDER)", + ), + ] = None, + model: Annotated[ + str | None, + typer.Option( + "--model", + help="Override the default model (takes precedence over CLEVERAGENTS_DEFAULT_MODEL)", + ), + ] = None, stream: Annotated[ bool, typer.Option("--stream", help="Show real-time progress during plan generation"), @@ -384,7 +409,16 @@ def tell( if stream: # Use streaming mode for real-time progress - asyncio.run(_tell_streaming(project, prompt, name, plan_service)) + asyncio.run( + _tell_streaming( + project, + prompt, + name, + plan_service, + provider, + model, + ) + ) else: # Use non-streaming mode (original behavior) with Progress( @@ -426,6 +460,20 @@ def build( verbose: Annotated[ bool, typer.Option("--verbose", "-v", help="Show detailed output") ] = False, + provider: Annotated[ + str | None, + typer.Option( + "--provider", + help="Override the provider for this build (highest precedence)", + ), + ] = None, + model: Annotated[ + str | None, + typer.Option( + "--model", + help="Override the model for this build (highest precedence)", + ), + ] = None, ) -> None: """Build the current plan to generate code changes. @@ -454,6 +502,8 @@ def build( changes = plan_service.build_plan( project=project, progress_callback=lambda p: progress.update(task, completed=p), + provider=provider, + model=model, ) if changes: diff --git a/src/cleveragents/cli/main.py b/src/cleveragents/cli/main.py index 96b4ee05e..ad7a528fd 100644 --- a/src/cleveragents/cli/main.py +++ b/src/cleveragents/cli/main.py @@ -148,6 +148,26 @@ def init( def tell( prompt: Annotated[str, typer.Argument(help="Instructions for the AI")], name: Annotated[str | None, typer.Option("--name", "-n")] = None, + provider: Annotated[ + str | None, + typer.Option( + "--provider", + help=( + "Override the default AI provider " + "(takes precedence over CLEVERAGENTS_DEFAULT_PROVIDER)" + ), + ), + ] = None, + model: Annotated[ + str | None, + typer.Option( + "--model", + help=( + "Override the default model " + "(takes precedence over CLEVERAGENTS_DEFAULT_MODEL)" + ), + ), + ] = None, stream: Annotated[ bool, typer.Option("--stream", help="Show real-time progress during plan generation"), @@ -156,15 +176,50 @@ def tell( """Create a plan from instructions (shortcut for 'plan tell').""" from cleveragents.cli.commands.plan import tell as plan_tell - plan_tell(prompt=prompt, name=name, stream=stream) + kwargs: dict[str, Any] = { + "prompt": prompt, + "stream": stream, + } + if name is not None: + kwargs["name"] = name + if provider is not None: + kwargs["provider"] = provider + if model is not None: + kwargs["model"] = model + + plan_tell(**kwargs) @app.command() -def build(verbose: bool = False) -> None: +def build( + verbose: Annotated[ + bool, typer.Option("--verbose", "-v", help="Show detailed output") + ] = False, + provider: Annotated[ + str | None, + typer.Option( + "--provider", + help="Override the provider for this build (highest precedence)", + ), + ] = None, + model: Annotated[ + str | None, + typer.Option( + "--model", + help="Override the model for this build (highest precedence)", + ), + ] = None, +) -> None: """Build the current plan (shortcut for 'plan build').""" from cleveragents.cli.commands.plan import build as plan_build - plan_build(verbose=verbose) + kwargs: dict[str, Any] = {"verbose": verbose} + if provider is not None: + kwargs["provider"] = provider + if model is not None: + kwargs["model"] = model + + plan_build(**kwargs) @app.command() diff --git a/src/cleveragents/config/settings.py b/src/cleveragents/config/settings.py index b5c4afecf..0b45ad83b 100644 --- a/src/cleveragents/config/settings.py +++ b/src/cleveragents/config/settings.py @@ -125,6 +125,16 @@ class Settings(BaseSettings): validation_alias=AliasChoices("CLEVERAGENTS_VECTOR_EMBEDDINGS_DIMENSION"), ) + # Provider defaults + default_provider: str | None = Field( + default=None, + validation_alias=AliasChoices("CLEVERAGENTS_DEFAULT_PROVIDER"), + ) + default_model: str | None = Field( + default=None, + validation_alias=AliasChoices("CLEVERAGENTS_DEFAULT_MODEL"), + ) + # LangSmith langsmith_enabled: bool = Field( default=False, @@ -183,6 +193,37 @@ class Settings(BaseSettings): perplexity_api_key: str | None = Field(default=None) groq_api_key: str | None = Field(default=None) together_api_key: str | None = Field(default=None) + azure_openai_endpoint: str | None = Field( + default=None, + validation_alias=AliasChoices( + "AZURE_OPENAI_ENDPOINT", + "AZURE_API_BASE", + "CLEVERAGENTS_AZURE_OPENAI_ENDPOINT", + ), + ) + azure_openai_api_version: str | None = Field( + default=None, + validation_alias=AliasChoices( + "AZURE_OPENAI_API_VERSION", + "AZURE_API_VERSION", + "CLEVERAGENTS_AZURE_OPENAI_API_VERSION", + ), + ) + azure_openai_deployment: str | None = Field( + default=None, + validation_alias=AliasChoices( + "AZURE_OPENAI_DEPLOYMENT", + "AZURE_OPENAI_CHAT_DEPLOYMENT", + "CLEVERAGENTS_AZURE_OPENAI_DEPLOYMENT", + ), + ) + openrouter_organization: str | None = Field( + default=None, + validation_alias=AliasChoices( + "OPENROUTER_ORGANIZATION", + "CLEVERAGENTS_OPENROUTER_ORGANIZATION", + ), + ) def model_post_init(self, __context: Any) -> None: # type: ignore[override] super().model_post_init(__context) @@ -387,6 +428,34 @@ class Settings(BaseSettings): or os.getenv("LANGSMITH_USER_ID") ) + def configured_provider_names(self) -> list[str]: + """Return sorted provider names with configured credentials.""" + + providers: list[str] = [] + for attr in self._PROVIDER_ENV_MAP: + if self._provider_value(attr): + normalized = attr.replace("_api_key", "").replace("_token", "") + providers.append(normalized) + return sorted(set(providers)) + + def provider_expected_env_vars(self) -> list[str]: + """Return the flattened list of provider-related env vars.""" + + env_vars: set[str] = set() + for names in self._PROVIDER_ENV_MAP.values(): + env_vars.update(names) + return sorted(env_vars) + + def provider_configuration_diagnostics(self) -> dict[str, Any]: + """Summarize provider configuration for user-facing diagnostics.""" + + return { + "configured_providers": self.configured_provider_names(), + "default_provider": (self.default_provider or ""), + "default_model": (self.default_model or ""), + "expected_env_vars": self.provider_expected_env_vars(), + } + # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ diff --git a/src/cleveragents/providers/registry.py b/src/cleveragents/providers/registry.py index 2210e153e..3b6fc2d8a 100644 --- a/src/cleveragents/providers/registry.py +++ b/src/cleveragents/providers/registry.py @@ -14,9 +14,10 @@ Following ADR-008 (Provider Plugin Architecture), this registry provides: from __future__ import annotations import os +from collections.abc import Mapping from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, ClassVar, cast from pydantic import BaseModel, ConfigDict, Field @@ -27,6 +28,14 @@ if TYPE_CHECKING: from langchain_core.language_models import BaseLanguageModel +def _coerce_optional_str(value: object | None) -> str | None: + if value is None: + return None + if isinstance(value, str): + return value + return str(value) + + class ProviderType(str, Enum): """Supported AI provider types.""" @@ -294,6 +303,16 @@ class ProviderRegistry: except ValueError: pass # Invalid provider type, continue to fallback + # Use settings default when configured + settings_default = (self._settings.default_provider or "").lower() + if settings_default: + try: + provider_type = ProviderType(settings_default) + if self.is_provider_configured(provider_type): + return provider_type + except ValueError: + pass + # Fall back to first configured provider in priority order for provider_type in self.FALLBACK_ORDER: if self.is_provider_configured(provider_type): @@ -321,6 +340,10 @@ class ProviderRegistry: if env_model: return env_model + settings_model = self._settings.default_model + if settings_model: + return settings_model + # Get provider type if provider_type is None: provider_type = self.get_default_provider_type() @@ -426,18 +449,60 @@ class ProviderRegistry: if provider_type == ProviderType.AZURE: from langchain_openai import AzureChatOpenAI + deployment_override = _coerce_optional_str( + kwargs.pop("deployment_name", None) + ) + deployment_name = ( + deployment_override + or model_id + or self._settings.azure_openai_deployment + or "gpt-4o" + ) + + endpoint_override = _coerce_optional_str( + kwargs.pop("azure_endpoint", self._settings.azure_openai_endpoint) + ) + azure_endpoint = endpoint_override or self._settings.azure_openai_endpoint + if not azure_endpoint: + raise ValueError( + "Azure OpenAI endpoint not configured. Set AZURE_OPENAI_ENDPOINT or CLEVERAGENTS_AZURE_OPENAI_ENDPOINT." + ) + + api_version_override = _coerce_optional_str( + kwargs.pop("api_version", self._settings.azure_openai_api_version) + ) + api_version = api_version_override or "2024-05-01-preview" + return AzureChatOpenAI( - deployment_name=model_id or "gpt-4o", + deployment_name=deployment_name, + azure_endpoint=azure_endpoint, + api_version=api_version, **kwargs, # type: ignore[arg-type] ) if provider_type == ProviderType.OPENROUTER: from langchain_openai import ChatOpenAI + default_headers = kwargs.pop("default_headers", None) + headers: dict[str, str] | None = None + if isinstance(default_headers, Mapping): + mapping_headers = cast(Mapping[object, object], default_headers) + headers = { + str(key): str(value) for key, value in mapping_headers.items() + } + + organization = self._settings.openrouter_organization + if organization: + if headers is None: + headers = {} + headers.setdefault("HTTP-Referer", organization) + headers.setdefault("X-Title", organization) + return ChatOpenAI( model=model_id or "anthropic/claude-sonnet-4-20250514", openai_api_base="https://openrouter.ai/api/v1", openai_api_key=self._settings.openrouter_api_key, + default_headers=headers, **kwargs, # type: ignore[arg-type] ) @@ -511,6 +576,11 @@ class ProviderRegistry: if model_id is None: model_id = self.get_default_model(provider_type) + provider_info = self.get_provider_info(provider_type) + capabilities = ( + provider_info.capabilities if provider_info else ProviderCapabilities() + ) + # Create factory function for the LLM def llm_factory(mid: str) -> BaseLanguageModel: return self._create_provider_llm(provider_type, mid) # type: ignore[arg-type] @@ -520,6 +590,7 @@ class ProviderRegistry: model_id=model_id or self.DEFAULT_MODELS.get(provider_type, "unknown"), llm_factory=llm_factory, max_retries=max_retries, + supports_streaming=capabilities.supports_streaming, ) diff --git a/typings/behave/__init__.pyi b/typings/behave/__init__.pyi new file mode 100644 index 000000000..a82e587d4 --- /dev/null +++ b/typings/behave/__init__.pyi @@ -0,0 +1,10 @@ +from collections.abc import Callable +from typing import Any, TypeVar + +_StepFunc = TypeVar("_StepFunc", bound=Callable[..., object]) + +StepDecorator = Callable[[_StepFunc], _StepFunc] + +def given(pattern: str, *args: Any, **kwargs: Any) -> StepDecorator: ... +def when(pattern: str, *args: Any, **kwargs: Any) -> StepDecorator: ... +def then(pattern: str, *args: Any, **kwargs: Any) -> StepDecorator: ...