diff --git a/features/anthropic_provider.feature b/features/anthropic_provider.feature new file mode 100644 index 000000000..32d8fdf10 --- /dev/null +++ b/features/anthropic_provider.feature @@ -0,0 +1,69 @@ +Feature: Anthropic Claude chat provider coverage + As a maintainer integrating real provider adapters + I want unit-level Behave scenarios for the Anthropic Claude chat provider + So that the implementation stays documented and regression tested + + @unit @providers @anthropic + Scenario: Anthropic provider instantiates ChatAnthropic with provided credentials + Given I have sample provider domain inputs + When I create an Anthropic chat provider with API key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + And I request plan generation from the Anthropic provider + Then the Anthropic provider should construct ChatAnthropic with api key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + And the Anthropic provider metadata should report name "anthropic" and model "claude-sonnet-4-20250514" + + @unit @providers @anthropic + Scenario: Anthropic provider stubbed response reports metadata + Given I have sample provider domain inputs + And I create an Anthropic chat provider with API key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + When I request plan generation from the Anthropic provider + Then the Anthropic provider response should contain no generated changes + And the Anthropic provider response should report the requested model without errors + + @unit @providers @anthropic + Scenario: Anthropic provider rejects missing API key + Given I have sample provider domain inputs + When I attempt to create an Anthropic chat provider without an API key + Then the Anthropic provider creation should fail with error "Anthropic API key is required" + + @unit @providers @anthropic + Scenario: Anthropic provider forwards optional kwargs + Given I have sample provider domain inputs + And I set the Anthropic provider extra kwargs "temperature=0.35,max_tokens=2048" + When I create an Anthropic chat provider with API key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + And I request plan generation from the Anthropic provider + Then the Anthropic provider should construct ChatAnthropic with api key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + And the Anthropic provider should include kwargs "temperature=0.35,max_tokens=2048" in the ChatAnthropic call + + @unit @providers @anthropic + Scenario: Anthropic provider reports runtime errors + Given I have sample provider domain inputs + And I create an Anthropic chat provider with API key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + And the plan generation graph raises RuntimeError "deadline reached" + When I request plan generation from the Anthropic provider + Then the Anthropic provider response should report error "deadline reached" + And the Anthropic provider response should contain no generated changes + + @unit @providers @anthropic + Scenario: Anthropic provider streaming yields workflow events + Given I have sample provider domain inputs + And the plan generation graph returns a generated change for "app/stream.py" + And the plan generation graph emits streaming nodes "load_context,analyze_requirements,generate_plan,validate" + And I create an Anthropic chat provider with API key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + When I stream plan generation from the Anthropic provider + Then the Anthropic provider streaming events should include nodes "load_context,analyze_requirements,generate_plan,validate" + And the Anthropic provider streaming result should finish with a response containing 1 generated change + + @unit @providers @anthropic + Scenario: Anthropic provider surfaces plan generation errors + Given I have sample provider domain inputs + And the plan generation graph raises ValueError "graph exploded" + And I create an Anthropic chat provider with API key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + When I request plan generation from the Anthropic provider + Then the Anthropic provider response should report error "graph exploded" + And the Anthropic provider response should contain no generated changes + + @unit @providers @anthropic + Scenario: Anthropic provider supports streaming responses + Given I have sample provider domain inputs + And I create an Anthropic chat provider with API key "sk-ant-unit-test" and model "claude-sonnet-4-20250514" + Then the Anthropic provider should support streaming responses diff --git a/features/google_provider.feature b/features/google_provider.feature new file mode 100644 index 000000000..82f68cb10 --- /dev/null +++ b/features/google_provider.feature @@ -0,0 +1,69 @@ +Feature: Google Gemini chat provider coverage + As a maintainer integrating real provider adapters + I want unit-level Behave scenarios for the Google Gemini chat provider + So that the implementation stays documented and regression tested + + @unit @providers @google + Scenario: Google provider instantiates ChatGoogleGenerativeAI with provided credentials + Given I have sample provider domain inputs + When I create a Google chat provider with API key "AIzaSy-unit-test" and model "gemini-2.0-flash" + And I request plan generation from the Google provider + Then the Google provider should construct ChatGoogleGenerativeAI with api key "AIzaSy-unit-test" and model "gemini-2.0-flash" + And the Google provider metadata should report name "google" and model "gemini-2.0-flash" + + @unit @providers @google + Scenario: Google provider stubbed response reports metadata + Given I have sample provider domain inputs + And I create a Google chat provider with API key "AIzaSy-unit-test" and model "gemini-2.0-flash" + When I request plan generation from the Google provider + Then the Google provider response should contain no generated changes + And the Google provider response should report the requested model without errors + + @unit @providers @google + Scenario: Google provider rejects missing API key + Given I have sample provider domain inputs + When I attempt to create a Google chat provider without an API key + Then the Google provider creation should fail with error "Google API key is required" + + @unit @providers @google + Scenario: Google provider forwards optional kwargs + Given I have sample provider domain inputs + And I set the Google provider extra kwargs "temperature=0.35,max_output_tokens=2048" + When I create a Google chat provider with API key "AIzaSy-unit-test" and model "gemini-2.0-flash" + And I request plan generation from the Google provider + Then the Google provider should construct ChatGoogleGenerativeAI with api key "AIzaSy-unit-test" and model "gemini-2.0-flash" + And the Google provider should include kwargs "temperature=0.35,max_output_tokens=2048" in the ChatGoogleGenerativeAI call + + @unit @providers @google + Scenario: Google provider reports runtime errors + Given I have sample provider domain inputs + And I create a Google chat provider with API key "AIzaSy-unit-test" and model "gemini-2.0-flash" + And the plan generation graph raises RuntimeError "deadline reached" + When I request plan generation from the Google provider + Then the Google provider response should report error "deadline reached" + And the Google provider response should contain no generated changes + + @unit @providers @google + Scenario: Google provider streaming yields workflow events + Given I have sample provider domain inputs + And the plan generation graph returns a generated change for "app/stream.py" + And the plan generation graph emits streaming nodes "load_context,analyze_requirements,generate_plan,validate" + And I create a Google chat provider with API key "AIzaSy-unit-test" and model "gemini-2.0-flash" + When I stream plan generation from the Google provider + Then the Google provider streaming events should include nodes "load_context,analyze_requirements,generate_plan,validate" + And the Google provider streaming result should finish with a response containing 1 generated change + + @unit @providers @google + Scenario: Google provider surfaces plan generation errors + Given I have sample provider domain inputs + And the plan generation graph raises ValueError "graph exploded" + And I create a Google chat provider with API key "AIzaSy-unit-test" and model "gemini-2.0-flash" + When I request plan generation from the Google provider + Then the Google provider response should report error "graph exploded" + And the Google provider response should contain no generated changes + + @unit @providers @google + Scenario: Google provider supports streaming responses + Given I have sample provider domain inputs + And I create a Google chat provider with API key "AIzaSy-unit-test" and model "gemini-2.0-flash" + Then the Google provider should support streaming responses diff --git a/features/steps/anthropic_provider_steps.py b/features/steps/anthropic_provider_steps.py index 394bf73b3..65f88e9f6 100644 --- a/features/steps/anthropic_provider_steps.py +++ b/features/steps/anthropic_provider_steps.py @@ -1,11 +1,17 @@ from __future__ import annotations +import ast +from typing import Any from unittest.mock import MagicMock, patch -from behave import then, when +from behave import given, then, when from cleveragents.providers.llm.anthropic_provider import AnthropicChatProvider +# Shared step definitions (I have sample provider domain inputs, plan generation graph steps) +# are registered by features/steps/provider_shared_steps.py which Behave loads automatically. +# Helper functions are duplicated here to avoid cross-module import issues with Behave's exec_file. + def _register_cleanup(context, cleanup): if hasattr(context, "add_cleanup"): @@ -16,53 +22,29 @@ def _register_cleanup(context, cleanup): context._cleanup_handlers = cleanup_handlers -@when("I attempt to create an Anthropic chat provider without an API key") -def step_attempt_create_without_key(context): - context.anthropic_creation_error = None - try: - AnthropicChatProvider(api_key="", model="claude-3-opus-20240229") - except Exception as exc: # pragma: no cover - defensive test guard - context.anthropic_creation_error = exc +def _parse_kwargs_string(kwargs_string: str) -> dict[str, Any]: + if not kwargs_string: + return {} + result: dict[str, Any] = {} + for entry in kwargs_string.split(","): + entry = entry.strip() + if not entry: + continue + if "=" not in entry: + result[entry] = True + continue + key, value = entry.split("=", 1) + key = key.strip() + value = value.strip() + try: + parsed_value = ast.literal_eval(value) + except Exception: + parsed_value = value + result[key] = parsed_value + return result -@then('the Anthropic provider creation should fail with "{error_message}"') -def step_assert_creation_failure(context, error_message): - error = getattr(context, "anthropic_creation_error", None) - assert error is not None, "Anthropic provider creation should have failed" - assert isinstance(error, ValueError) - assert str(error) == error_message - - -@when( - 'I create an Anthropic chat provider with API key "{api_key}" and model "{model}" with custom overrides' -) -def step_create_anthropic_provider_with_overrides(context, api_key, model): - patcher = patch("cleveragents.providers.llm.anthropic_provider.ChatAnthropic") - context.chat_anthropic_patcher = patcher - mock_chat_class = patcher.start() - _register_cleanup(context, patcher.stop) - mock_chat_instance = MagicMock(name="ChatAnthropicInstance") - mock_chat_instance.get_num_tokens = MagicMock(return_value=0) - mock_chat_class.return_value = mock_chat_instance - - context.anthropic_overrides = { - "temperature": 0.15, - "max_tokens": 2048, - "default_request_timeout": 12, - } - - context.provider = AnthropicChatProvider( - api_key=api_key, - model=model, - max_retries=2, - **context.anthropic_overrides, - ) - context.chat_anthropic_class = mock_chat_class - context.chat_anthropic_instance = mock_chat_instance - - -@when("I request plan generation from the Anthropic provider") -def step_request_plan_generation_anthropic(context): +def _setup_plan_generation_graph(context) -> MagicMock: patcher = patch( "cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph" ) @@ -70,16 +52,79 @@ def step_request_plan_generation_anthropic(context): mock_graph_class = patcher.start() _register_cleanup(context, patcher.stop) mock_graph_instance = MagicMock(name="PlanGenerationGraphInstance") - mock_graph_instance.invoke.return_value = { + + default_state = { "generated_changes": [], "validation_result": {"status": "PASS"}, "error": None, } - mock_graph_instance.stream.return_value = iter(()) - mock_graph_class.return_value = mock_graph_instance + state_override = getattr(context, "plan_generation_state_override", None) + mock_graph_instance.invoke.return_value = state_override or default_state + invoke_side_effect = getattr(context, "plan_generation_invoke_side_effect", None) + if invoke_side_effect is not None: + mock_graph_instance.invoke.side_effect = invoke_side_effect + + stream_events = getattr(context, "plan_generation_stream_events", None) + if stream_events is None: + mock_graph_instance.stream.return_value = iter(()) + else: + events_copy = list(stream_events) + + def _stream(*_args, **_kwargs): + yield from events_copy + + mock_graph_instance.stream.side_effect = _stream + + mock_graph_class.return_value = mock_graph_instance context.plan_generation_graph = mock_graph_instance context.plan_generation_graph_class = mock_graph_class + return mock_graph_instance + + +@given('I set the Anthropic provider extra kwargs "{kwargs_string}"') +def step_set_anthropic_provider_kwargs(context, kwargs_string): + context.anthropic_provider_kwargs = _parse_kwargs_string(kwargs_string) + + +@given( + 'I create an Anthropic chat provider with API key "{api_key}" and model "{model_id}"' +) +@when( + 'I create an Anthropic chat provider with API key "{api_key}" and model "{model_id}"' +) +def step_create_anthropic_provider(context, api_key, model_id): + patcher = patch("cleveragents.providers.llm.anthropic_provider.ChatAnthropic") + context.chat_anthropic_patcher = patcher + mock_chat_anthropic_class = patcher.start() + _register_cleanup(context, patcher.stop) + mock_chat_anthropic_instance = MagicMock(name="ChatAnthropicInstance") + mock_chat_anthropic_instance.get_num_tokens = MagicMock(return_value=0) + mock_chat_anthropic_class.return_value = mock_chat_anthropic_instance + + extra_kwargs = dict(getattr(context, "anthropic_provider_kwargs", {})) + + context.provider = AnthropicChatProvider( + api_key=api_key, + model=model_id, + **extra_kwargs, + ) + context.chat_anthropic_class = mock_chat_anthropic_class + context.chat_anthropic_instance = mock_chat_anthropic_instance + + +@when("I attempt to create an Anthropic chat provider without an API key") +def step_anthropic_provider_without_api_key(context): + context.anthropic_provider_error = None + try: + AnthropicChatProvider(api_key="", model="claude-sonnet-4-20250514") + except Exception as exc: # pragma: no cover - defensive logging only + context.anthropic_provider_error = exc + + +@when("I request plan generation from the Anthropic provider") +def step_request_plan_generation(context): + _setup_plan_generation_graph(context) context.response = context.provider.generate_changes( context.project, @@ -87,43 +132,142 @@ def step_request_plan_generation_anthropic(context): context.contexts, ) context.chat_anthropic_call = context.chat_anthropic_class.call_args - context.plan_generation_graph_call = mock_graph_class.call_args + context.plan_generation_graph_call = context.plan_generation_graph_class.call_args + + +@when("I stream plan generation from the Anthropic provider") +def step_stream_plan_generation(context): + _setup_plan_generation_graph(context) + context.streamed_events = list( + context.provider.stream_changes( + context.project, + context.plan, + context.contexts, + ) + ) + context.chat_anthropic_call = context.chat_anthropic_class.call_args + context.plan_generation_graph_call = context.plan_generation_graph_class.call_args @then( - 'the Anthropic provider should construct ChatAnthropic with api key "{api_key}" and model "{model}"' + 'the Anthropic provider should construct ChatAnthropic with api key "{api_key}" and model "{model_id}"' ) -def step_assert_chat_anthropic_construction(context, api_key, model): - call = getattr(context, "chat_anthropic_call", None) - assert call is not None, "ChatAnthropic should have been constructed" - call_args, call_kwargs = call +def step_assert_chat_anthropic_constructor(context, api_key, model_id): + assert context.chat_anthropic_call is not None, ( + "ChatAnthropic should have been called" + ) + call_args, call_kwargs = context.chat_anthropic_call assert call_args == (), "ChatAnthropic should be called with keyword arguments" assert call_kwargs["api_key"] == api_key - assert call_kwargs["model"] == model + assert call_kwargs["model"] == model_id - graph_call = getattr(context, "plan_generation_graph_call", None) - assert graph_call is not None, "PlanGenerationGraph should be constructed" - _, graph_kwargs = graph_call + assert context.plan_generation_graph_call is not None, ( + "PlanGenerationGraph should receive the ChatAnthropic instance" + ) + _, graph_kwargs = context.plan_generation_graph_call assert graph_kwargs["llm"] is context.chat_anthropic_instance -@then("the Anthropic provider should apply the custom overrides to ChatAnthropic") -def step_assert_anthropic_overrides(context): - call = getattr(context, "chat_anthropic_call", None) - assert call is not None, "ChatAnthropic should have been constructed" - _, call_kwargs = call - overrides = getattr(context, "anthropic_overrides", None) - assert overrides, "Custom overrides should have been captured" - - for key, value in overrides.items(): - assert call_kwargs.get(key) == value, f"Override {key} was not forwarded" +@then( + 'the Anthropic provider should include kwargs "{kwargs_string}" in the ChatAnthropic call' +) +def step_assert_chat_anthropic_kwargs(context, kwargs_string): + assert context.chat_anthropic_call is not None, ( + "ChatAnthropic should have been called" + ) + _, call_kwargs = context.chat_anthropic_call + expected_kwargs = _parse_kwargs_string(kwargs_string) + for key, value in expected_kwargs.items(): + assert key in call_kwargs, f"Expected {key} in ChatAnthropic kwargs" + assert call_kwargs[key] == value, ( + f"Expected ChatAnthropic kwargs[{key!r}] to equal {value!r}" + ) @then( 'the Anthropic provider metadata should report name "{expected_name}" and model "{expected_model}"' ) -def step_assert_anthropic_metadata(context, expected_name, expected_model): +def step_assert_provider_metadata(context, expected_name, expected_model): provider = getattr(context, "provider", None) - assert provider is not None, "Anthropic provider should exist in context" + assert provider is not None, "Provider should exist" assert provider.name == expected_name assert provider.model_id == expected_model + response = getattr(context, "response", None) + if response is not None: + assert response.model_used == expected_model + + +@then( + "the Anthropic provider response should report the requested model without errors" +) +def step_assert_placeholder_metadata(context): + assert context.response is not None + assert context.response.model_used == context.provider.model_id + assert context.response.error_message in (None, "") + + +@then("the Anthropic provider response should contain no generated changes") +def step_assert_no_changes(context): + assert context.response is not None, "Provider response should exist" + assert len(context.response.changes) == 0, "Expected no generated changes" + + +@then( + 'the Anthropic provider response should include {count:d} generated change for "{file_path}"' +) +def step_assert_generated_change(context, count, file_path): + assert context.response is not None, "Provider response should exist" + assert len(context.response.changes) == count, ( + f"Expected {count} changes, got {len(context.response.changes)}" + ) + assert any( + getattr(change, "file_path", None) == file_path + for change in context.response.changes + ), f"Expected change for {file_path}" + + +@then('the Anthropic provider response should report error "{message}"') +def step_assert_response_error(context, message): + assert context.response is not None + assert context.response.error_message == message + + +@then('the Anthropic provider streaming events should include nodes "{node_list}"') +def step_assert_stream_events(context, node_list): + expected = [node.strip() for node in node_list.split(",") if node.strip()] + actual = [ + next(iter(event.keys())) + for event in getattr(context, "streamed_events", []) + if "__end__" not in event + ] + assert actual == expected + + +@then( + "the Anthropic provider streaming result should finish with a response containing {count:d} generated change" +) +def step_assert_stream_final_response(context, count): + events = getattr(context, "streamed_events", []) + assert events, "Expected streamed events" + final_event = events[-1] + assert "__end__" in final_event, "Expected final __end__ event" + response = final_event["__end__"].get("response") + assert response is not None, "Expected ProviderResponse in __end__ event" + assert len(response.changes) == count + + +@then('the Anthropic provider creation should fail with error "{message}"') +def step_assert_anthropic_provider_error(context, message): + error = getattr(context, "anthropic_provider_error", None) + assert error is not None, "Expected the provider to raise an error" + assert isinstance(error, ValueError) + assert str(error) == message + + +@then("the Anthropic provider should support streaming responses") +def step_assert_anthropic_supports_streaming(context): + provider = getattr(context, "provider", None) + assert provider is not None, "Provider should exist" + assert provider._supports_streaming is True, ( + "Anthropic provider should support streaming" + ) diff --git a/features/steps/consolidated_ai_models_providers_steps.py b/features/steps/consolidated_ai_models_providers_steps.py index e69de29bb..1e1bffac7 100644 --- a/features/steps/consolidated_ai_models_providers_steps.py +++ b/features/steps/consolidated_ai_models_providers_steps.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from behave import given, then, when + +from cleveragents.providers.llm.anthropic_provider import AnthropicChatProvider + +# Shared step definitions (I have sample provider domain inputs, plan generation graph steps) +# are registered by features/steps/provider_shared_steps.py which Behave loads automatically. +# Helper functions are duplicated here to avoid cross-module import issues with Behave's exec_file. + + +def _register_cleanup(context, cleanup): + if hasattr(context, "add_cleanup"): + context.add_cleanup(cleanup) + else: + cleanup_handlers = getattr(context, "_cleanup_handlers", []) + cleanup_handlers.append(cleanup) + context._cleanup_handlers = cleanup_handlers + + +@then('the Anthropic provider creation should fail with "{message}"') +def step_assert_anthropic_provider_error_no_error_keyword(context, message): + """Variant of the error assertion step without the word 'error' in the step text.""" + error = getattr(context, "anthropic_provider_error", None) + assert error is not None, "Expected the provider to raise an error" + assert isinstance(error, ValueError) + assert str(error) == message + + +@when( + 'I create an Anthropic chat provider with API key "{api_key}" and model "{model_id}" with custom overrides' +) +def step_create_anthropic_provider_with_custom_overrides(context, api_key, model_id): + patcher = patch("cleveragents.providers.llm.anthropic_provider.ChatAnthropic") + context.chat_anthropic_patcher = patcher + mock_chat_anthropic_class = patcher.start() + _register_cleanup(context, patcher.stop) + mock_chat_anthropic_instance = MagicMock(name="ChatAnthropicInstance") + mock_chat_anthropic_instance.get_num_tokens = MagicMock(return_value=0) + mock_chat_anthropic_class.return_value = mock_chat_anthropic_instance + + # Store custom overrides for later assertion + context.anthropic_custom_overrides = {"temperature": 0.5, "max_tokens": 1024} + + context.provider = AnthropicChatProvider( + api_key=api_key, + model=model_id, + **context.anthropic_custom_overrides, + ) + context.chat_anthropic_class = mock_chat_anthropic_class + context.chat_anthropic_instance = mock_chat_anthropic_instance + + +@then("the Anthropic provider should apply the custom overrides to ChatAnthropic") +def step_assert_anthropic_custom_overrides(context): + assert context.chat_anthropic_call is not None, ( + "ChatAnthropic should have been called" + ) + _, call_kwargs = context.chat_anthropic_call + overrides = getattr(context, "anthropic_custom_overrides", {}) + for key, value in overrides.items(): + assert key in call_kwargs, f"Expected {key} in ChatAnthropic kwargs" + assert call_kwargs[key] == value, ( + f"Expected ChatAnthropic kwargs[{key!r}] to equal {value!r}" + ) + + +@given("the Google provider token estimator returns {token_count:d} tokens") +def step_google_token_estimator(context, token_count): + context.google_token_count = token_count + + +@then("the Google provider response token count should equal {expected:d}") +def step_assert_google_token_count(context, expected): + assert context.response is not None + assert context.response.token_count == expected diff --git a/features/steps/google_provider_steps.py b/features/steps/google_provider_steps.py index 431103c8b..a799329b2 100644 --- a/features/steps/google_provider_steps.py +++ b/features/steps/google_provider_steps.py @@ -8,6 +8,10 @@ from behave import given, then, when from cleveragents.providers.llm.google_provider import GoogleChatProvider +# Shared step definitions (I have sample provider domain inputs, plan generation graph steps) +# are registered by features/steps/provider_shared_steps.py which Behave loads automatically. +# Helper functions are duplicated here to avoid cross-module import issues with Behave's exec_file. + def _register_cleanup(context, cleanup): if hasattr(context, "add_cleanup"): @@ -21,13 +25,13 @@ def _register_cleanup(context, cleanup): def _parse_kwargs_string(kwargs_string: str) -> dict[str, Any]: if not kwargs_string: return {} - parsed: dict[str, Any] = {} + result: dict[str, Any] = {} for entry in kwargs_string.split(","): entry = entry.strip() if not entry: continue if "=" not in entry: - parsed[entry] = True + result[entry] = True continue key, value = entry.split("=", 1) key = key.strip() @@ -36,8 +40,8 @@ def _parse_kwargs_string(kwargs_string: str) -> dict[str, Any]: parsed_value = ast.literal_eval(value) except Exception: parsed_value = value - parsed[key] = parsed_value - return parsed + result[key] = parsed_value + return result def _setup_plan_generation_graph(context) -> MagicMock: @@ -83,23 +87,21 @@ def step_set_google_provider_kwargs(context, kwargs_string): context.google_provider_kwargs = _parse_kwargs_string(kwargs_string) -@given("the Google provider token estimator returns {token_count:d} tokens") -def step_set_google_token_estimator(context, token_count): - context.google_token_count = token_count - - +@given( + 'I create a Google chat provider with API key "{api_key}" and model "{model_id}"' +) @when('I create a Google chat provider with API key "{api_key}" and model "{model_id}"') def step_create_google_provider(context, api_key, model_id): patcher = patch("cleveragents.providers.llm.google_provider.ChatGoogleGenerativeAI") context.chat_google_patcher = patcher - mock_chat_class = patcher.start() + mock_chat_google_class = patcher.start() _register_cleanup(context, patcher.stop) - mock_chat_instance = MagicMock(name="ChatGoogleGenerativeAIInstance") - mock_chat_instance.get_num_tokens = MagicMock(return_value=0) + mock_chat_google_instance = MagicMock(name="ChatGoogleGenerativeAIInstance") + mock_chat_google_instance.get_num_tokens = MagicMock(return_value=0) token_override = getattr(context, "google_token_count", None) if token_override is not None: - mock_chat_instance.get_num_tokens.return_value = token_override - mock_chat_class.return_value = mock_chat_instance + mock_chat_google_instance.get_num_tokens.return_value = token_override + mock_chat_google_class.return_value = mock_chat_google_instance extra_kwargs = dict(getattr(context, "google_provider_kwargs", {})) @@ -108,8 +110,8 @@ def step_create_google_provider(context, api_key, model_id): model=model_id, **extra_kwargs, ) - context.chat_google_class = mock_chat_class - context.chat_google_instance = mock_chat_instance + context.chat_google_class = mock_chat_google_class + context.chat_google_instance = mock_chat_google_instance @when("I attempt to create a Google chat provider without an API key") @@ -117,13 +119,14 @@ def step_google_provider_without_api_key(context): context.google_provider_error = None try: GoogleChatProvider(api_key="", model="gemini-2.0-flash") - except Exception as exc: # pragma: no cover - defensive + except Exception as exc: # pragma: no cover - defensive logging only context.google_provider_error = exc @when("I request plan generation from the Google provider") -def step_request_google_plan_generation(context): +def step_request_plan_generation(context): _setup_plan_generation_graph(context) + context.response = context.provider.generate_changes( context.project, context.plan, @@ -133,32 +136,51 @@ def step_request_google_plan_generation(context): context.plan_generation_graph_call = context.plan_generation_graph_class.call_args +@when("I stream plan generation from the Google provider") +def step_stream_plan_generation(context): + _setup_plan_generation_graph(context) + context.streamed_events = list( + context.provider.stream_changes( + context.project, + context.plan, + context.contexts, + ) + ) + context.chat_google_call = context.chat_google_class.call_args + context.plan_generation_graph_call = context.plan_generation_graph_class.call_args + + @then( 'the Google provider should construct ChatGoogleGenerativeAI with api key "{api_key}" and model "{model_id}"' ) -def step_assert_google_constructor(context, api_key, model_id): - call = getattr(context, "chat_google_call", None) - assert call is not None, "ChatGoogleGenerativeAI should have been called" - call_args, call_kwargs = call - assert call_args == (), "ChatGoogleGenerativeAI should use keyword arguments" +def step_assert_chat_google_constructor(context, api_key, model_id): + assert context.chat_google_call is not None, ( + "ChatGoogleGenerativeAI should have been called" + ) + call_args, call_kwargs = context.chat_google_call + assert call_args == (), ( + "ChatGoogleGenerativeAI should be called with keyword arguments" + ) assert call_kwargs["api_key"] == api_key assert call_kwargs["model"] == model_id - graph_call = getattr(context, "plan_generation_graph_call", None) - assert graph_call is not None, "PlanGenerationGraph should receive the LLM" - _, graph_kwargs = graph_call + assert context.plan_generation_graph_call is not None, ( + "PlanGenerationGraph should receive the ChatGoogleGenerativeAI instance" + ) + _, graph_kwargs = context.plan_generation_graph_call assert graph_kwargs["llm"] is context.chat_google_instance @then( 'the Google provider should include kwargs "{kwargs_string}" in the ChatGoogleGenerativeAI call' ) -def step_assert_google_kwargs(context, kwargs_string): - call = getattr(context, "chat_google_call", None) - assert call is not None, "ChatGoogleGenerativeAI should have been called" - _, call_kwargs = call - expected = _parse_kwargs_string(kwargs_string) - for key, value in expected.items(): +def step_assert_chat_google_kwargs(context, kwargs_string): + assert context.chat_google_call is not None, ( + "ChatGoogleGenerativeAI should have been called" + ) + _, call_kwargs = context.chat_google_call + expected_kwargs = _parse_kwargs_string(kwargs_string) + for key, value in expected_kwargs.items(): assert key in call_kwargs, f"Expected {key} in ChatGoogleGenerativeAI kwargs" assert call_kwargs[key] == value, ( f"Expected ChatGoogleGenerativeAI kwargs[{key!r}] to equal {value!r}" @@ -168,9 +190,9 @@ def step_assert_google_kwargs(context, kwargs_string): @then( 'the Google provider metadata should report name "{expected_name}" and model "{expected_model}"' ) -def step_assert_google_metadata(context, expected_name, expected_model): +def step_assert_provider_metadata(context, expected_name, expected_model): provider = getattr(context, "provider", None) - assert provider is not None, "Google provider should exist" + assert provider is not None, "Provider should exist" assert provider.name == expected_name assert provider.model_id == expected_model response = getattr(context, "response", None) @@ -178,30 +200,75 @@ def step_assert_google_metadata(context, expected_name, expected_model): assert response.model_used == expected_model +@then("the Google provider response should report the requested model without errors") +def step_assert_placeholder_metadata(context): + assert context.response is not None + assert context.response.model_used == context.provider.model_id + assert context.response.error_message in (None, "") + + +@then("the Google provider response should contain no generated changes") +def step_assert_no_changes(context): + assert context.response is not None, "Provider response should exist" + assert len(context.response.changes) == 0, "Expected no generated changes" + + @then( 'the Google provider response should include {count:d} generated change for "{file_path}"' ) -def step_assert_google_generated_change(context, count, file_path): - response = getattr(context, "response", None) - assert response is not None, "Provider response should exist" - assert len(response.changes) == count, ( - f"Expected {count} changes, got {len(response.changes)}" +def step_assert_generated_change(context, count, file_path): + assert context.response is not None, "Provider response should exist" + assert len(context.response.changes) == count, ( + f"Expected {count} changes, got {len(context.response.changes)}" ) assert any( - getattr(change, "file_path", None) == file_path for change in response.changes + getattr(change, "file_path", None) == file_path + for change in context.response.changes ), f"Expected change for {file_path}" -@then("the Google provider response token count should equal {expected:d}") -def step_assert_google_token_count(context, expected): - response = getattr(context, "response", None) - assert response is not None - assert response.token_count == expected +@then('the Google provider response should report error "{message}"') +def step_assert_response_error(context, message): + assert context.response is not None + assert context.response.error_message == message + + +@then('the Google provider streaming events should include nodes "{node_list}"') +def step_assert_stream_events(context, node_list): + expected = [node.strip() for node in node_list.split(",") if node.strip()] + actual = [ + next(iter(event.keys())) + for event in getattr(context, "streamed_events", []) + if "__end__" not in event + ] + assert actual == expected + + +@then( + "the Google provider streaming result should finish with a response containing {count:d} generated change" +) +def step_assert_stream_final_response(context, count): + events = getattr(context, "streamed_events", []) + assert events, "Expected streamed events" + final_event = events[-1] + assert "__end__" in final_event, "Expected final __end__ event" + response = final_event["__end__"].get("response") + assert response is not None, "Expected ProviderResponse in __end__ event" + assert len(response.changes) == count @then('the Google provider creation should fail with error "{message}"') -def step_assert_google_creation_error(context, message): +def step_assert_google_provider_error(context, message): error = getattr(context, "google_provider_error", None) assert error is not None, "Expected the provider to raise an error" assert isinstance(error, ValueError) assert str(error) == message + + +@then("the Google provider should support streaming responses") +def step_assert_google_supports_streaming(context): + provider = getattr(context, "provider", None) + assert provider is not None, "Provider should exist" + assert provider._supports_streaming is True, ( + "Google provider should support streaming" + ) diff --git a/features/steps/openai_provider_steps.py b/features/steps/openai_provider_steps.py index d96e37dba..a2a5a4273 100644 --- a/features/steps/openai_provider_steps.py +++ b/features/steps/openai_provider_steps.py @@ -6,14 +6,12 @@ from unittest.mock import MagicMock, patch from behave import given, then, when -from cleveragents.domain.models.core import ( - Context, - OperationType, - Plan, - Project, -) from cleveragents.providers.llm.openai_provider import OpenAIChatProvider +# Shared step definitions (I have sample provider domain inputs, plan generation graph steps) +# are registered by features/steps/provider_shared_steps.py which Behave loads automatically. +# Helper functions are duplicated here to avoid cross-module import issues with Behave's exec_file. + def _register_cleanup(context, cleanup): if hasattr(context, "add_cleanup"): @@ -84,66 +82,11 @@ def _setup_plan_generation_graph(context) -> MagicMock: return mock_graph_instance -@given("I have sample provider domain inputs") -def step_sample_provider_inputs(context): - context.project = MagicMock(spec=Project) - context.plan = MagicMock(spec=Plan) - context.plan.prompt = "Add placeholder coverage" - context.contexts = [MagicMock(spec=Context)] - context.contexts[0].content = "Sample context entry" - - -@given('the plan generation graph returns a generated change for "{file_path}"') -def step_plan_generation_returns_change(context, file_path): - change = { - "plan_id": 1, - "file_path": file_path, - "operation": OperationType.MODIFY.value, - "new_content": "# updated content", - } - context.plan_generation_state_override = { - "generated_changes": [change], - "validation_result": {"status": "PASS"}, - "error": None, - } - - @given("the OpenAI provider token estimator returns {token_count:d} tokens") def step_openai_token_estimator(context, token_count): context.openai_token_count = token_count -@given('the plan generation graph emits streaming nodes "{node_list}"') -def step_plan_generation_stream_nodes(context, node_list): - nodes = [node.strip() for node in node_list.split(",") if node.strip()] - state = getattr(context, "plan_generation_state_override", {}) or {} - events: list[dict[str, Any]] = [] - for node in nodes: - payload: dict[str, Any] = {"status": "completed"} - if node == "generate_plan" and isinstance(state.get("generated_changes"), list): - payload = { - "generated_changes": state["generated_changes"], - "status": "completed", - } - elif node == "validate" and isinstance(state.get("validation_result"), dict): - payload = { - "validation_result": state["validation_result"], - "status": "completed", - } - events.append({node: payload}) - context.plan_generation_stream_events = events - - -@given('the plan generation graph raises ValueError "{message}"') -def step_plan_generation_raises_value_error(context, message): - context.plan_generation_invoke_side_effect = ValueError(message) - - -@given('the plan generation graph raises RuntimeError "{message}"') -def step_plan_generation_runtime_error(context, message): - context.plan_generation_invoke_side_effect = RuntimeError(message) - - @given('I set the OpenAI provider organization "{organization}"') def step_set_openai_provider_org(context, organization): if organization.lower() == "none": diff --git a/features/steps/provider_shared_steps.py b/features/steps/provider_shared_steps.py new file mode 100644 index 000000000..ca5c4d24d --- /dev/null +++ b/features/steps/provider_shared_steps.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import ast +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given + +from cleveragents.domain.models.core import ( + Context, + OperationType, + Plan, + Project, +) + + +def _register_cleanup(context, cleanup): + if hasattr(context, "add_cleanup"): + context.add_cleanup(cleanup) + else: + cleanup_handlers = getattr(context, "_cleanup_handlers", []) + cleanup_handlers.append(cleanup) + context._cleanup_handlers = cleanup_handlers + + +def _parse_kwargs_string(kwargs_string: str) -> dict[str, Any]: + if not kwargs_string: + return {} + result: dict[str, Any] = {} + for entry in kwargs_string.split(","): + entry = entry.strip() + if not entry: + continue + if "=" not in entry: + result[entry] = True + continue + key, value = entry.split("=", 1) + key = key.strip() + value = value.strip() + try: + parsed_value = ast.literal_eval(value) + except Exception: + parsed_value = value + result[key] = parsed_value + return result + + +def _setup_plan_generation_graph(context) -> MagicMock: + patcher = patch( + "cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph" + ) + context.plan_generation_patcher = patcher + mock_graph_class = patcher.start() + _register_cleanup(context, patcher.stop) + mock_graph_instance = MagicMock(name="PlanGenerationGraphInstance") + + default_state = { + "generated_changes": [], + "validation_result": {"status": "PASS"}, + "error": None, + } + state_override = getattr(context, "plan_generation_state_override", None) + mock_graph_instance.invoke.return_value = state_override or default_state + + invoke_side_effect = getattr(context, "plan_generation_invoke_side_effect", None) + if invoke_side_effect is not None: + mock_graph_instance.invoke.side_effect = invoke_side_effect + + stream_events = getattr(context, "plan_generation_stream_events", None) + if stream_events is None: + mock_graph_instance.stream.return_value = iter(()) + else: + events_copy = list(stream_events) + + def _stream(*_args, **_kwargs): + yield from events_copy + + mock_graph_instance.stream.side_effect = _stream + + mock_graph_class.return_value = mock_graph_instance + context.plan_generation_graph = mock_graph_instance + context.plan_generation_graph_class = mock_graph_class + return mock_graph_instance + + +@given("I have sample provider domain inputs") +def step_sample_provider_inputs(context): + context.project = MagicMock(spec=Project) + context.plan = MagicMock(spec=Plan) + context.plan.prompt = "Add placeholder coverage" + context.contexts = [MagicMock(spec=Context)] + context.contexts[0].content = "Sample context entry" + + +@given('the plan generation graph returns a generated change for "{file_path}"') +def step_plan_generation_returns_change(context, file_path): + change = { + "plan_id": 1, + "file_path": file_path, + "operation": OperationType.MODIFY.value, + "new_content": "# updated content", + } + context.plan_generation_state_override = { + "generated_changes": [change], + "validation_result": {"status": "PASS"}, + "error": None, + } + + +@given('the plan generation graph emits streaming nodes "{node_list}"') +def step_plan_generation_stream_nodes(context, node_list): + nodes = [node.strip() for node in node_list.split(",") if node.strip()] + state = getattr(context, "plan_generation_state_override", {}) or {} + events: list[dict[str, Any]] = [] + for node in nodes: + payload: dict[str, Any] = {"status": "completed"} + if node == "generate_plan" and isinstance(state.get("generated_changes"), list): + payload = { + "generated_changes": state["generated_changes"], + "status": "completed", + } + elif node == "validate" and isinstance(state.get("validation_result"), dict): + payload = { + "validation_result": state["validation_result"], + "status": "completed", + } + events.append({node: payload}) + context.plan_generation_stream_events = events + + +@given('the plan generation graph raises ValueError "{message}"') +def step_plan_generation_raises_value_error(context, message): + context.plan_generation_invoke_side_effect = ValueError(message) + + +@given('the plan generation graph raises RuntimeError "{message}"') +def step_plan_generation_runtime_error(context, message): + context.plan_generation_invoke_side_effect = RuntimeError(message)