From a8af98c3728fa7b996e38ecd6d4c240839e788a5 Mon Sep 17 00:00:00 2001 From: drew Date: Tue, 16 Jun 2026 22:28:32 -0400 Subject: [PATCH 1/2] test(auto_debug): type BDD mutation fixtures --- ...auto_debug_analyze_error_mutation_steps.py | 3 +- .../tdd_auto_debug_state_mutation_steps.py | 58 +++++++------------ 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py b/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py index d7dd42012..e61d3c129 100644 --- a/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py +++ b/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py @@ -47,7 +47,7 @@ def step_create_agent_with_fake_llm(context: Any) -> None: @given("a state with an empty messages list for the mutation test") def step_create_state_with_empty_messages(context: Any) -> None: """Create a minimal AutoDebugState with an empty messages list.""" - context.mutation_test_state: AutoDebugState = { + state: AutoDebugState = { "error_message": "NullPointerException at line 42", "code_context": "x = obj.method()", "messages": [], @@ -59,6 +59,7 @@ def step_create_state_with_empty_messages(context: Any) -> None: "current_fix": {}, "fix_validated": False, } + context.mutation_test_state = state # Capture a reference to the original messages list before calling the node context.mutation_test_original_messages = context.mutation_test_state["messages"] diff --git a/features/steps/tdd_auto_debug_state_mutation_steps.py b/features/steps/tdd_auto_debug_state_mutation_steps.py index 8ed38b440..0ef7b8ea5 100644 --- a/features/steps/tdd_auto_debug_state_mutation_steps.py +++ b/features/steps/tdd_auto_debug_state_mutation_steps.py @@ -12,42 +12,11 @@ from copy import deepcopy from typing import Any from behave import given, then, when +from langchain_community.llms import FakeListLLM from cleveragents.agents.graphs.auto_debug import AutoDebugAgent, AutoDebugState -class _MockResponse: - """Minimal response object with a content attribute.""" - - def __init__(self, content: str) -> None: - self.content = content - - -class _MockLLM: - """Stub LLM that returns a fixed content string.""" - - def __init__(self, content: str = "Mock LLM response") -> None: - self._content = content - - def invoke(self, messages: Any) -> _MockResponse: - return _MockResponse(self._content) - - -class _InvalidValidationLLM: - """Stub LLM that returns an invalid validation response.""" - - def invoke(self, messages: Any) -> _MockResponse: - return _MockResponse( - json.dumps( - { - "is_valid": False, - "reasoning": "Fix failed", - "issues": ["error persists"], - } - ) - ) - - def _base_state(**overrides: Any) -> AutoDebugState: """Create a minimal valid AutoDebugState with optional overrides.""" state: AutoDebugState = { @@ -76,7 +45,10 @@ def step_module_imported(context: Any) -> None: @given("an auto debug agent for mutation testing") def step_create_agent(context: Any) -> None: """Create an AutoDebugAgent with a mock LLM.""" - context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3) + context.mutation_agent = AutoDebugAgent( + llm=FakeListLLM(responses=["Mock LLM response"]), + max_fix_attempts=3, + ) @given("an initial auto debug state for mutation testing") @@ -109,7 +81,10 @@ def step_state_with_analysis(context: Any) -> None: @given("a state with a current fix for mutation testing") def step_state_with_current_fix(context: Any) -> None: """Create a state with a current fix.""" - context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3) + context.mutation_agent = AutoDebugAgent( + llm=FakeListLLM(responses=["Mock LLM response"]), + max_fix_attempts=3, + ) context.mutation_input_state = _base_state( current_fix={ "description": "Proposed fix", @@ -126,8 +101,16 @@ def step_state_with_current_fix(context: Any) -> None: @given("a state with a current fix and invalid validation for mutation testing") def step_state_with_invalid_validation(context: Any) -> None: """Create a state with a current fix that will fail validation.""" + invalid_response = json.dumps( + { + "is_valid": False, + "reasoning": "Fix failed", + "issues": ["error persists"], + } + ) context.mutation_agent = AutoDebugAgent( - llm=_InvalidValidationLLM(), max_fix_attempts=3 + llm=FakeListLLM(responses=[invalid_response]), + max_fix_attempts=3, ) context.mutation_input_state = _base_state( current_fix={ @@ -146,7 +129,10 @@ def step_state_with_invalid_validation(context: Any) -> None: @given("a state ready for finalization for mutation testing") def step_state_for_finalization(context: Any) -> None: """Create a state ready for finalization.""" - context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3) + context.mutation_agent = AutoDebugAgent( + llm=FakeListLLM(responses=["Mock LLM response"]), + max_fix_attempts=3, + ) context.mutation_input_state = _base_state( fix_validated=True, current_fix={"description": "Final fix", "code": "# fixed"}, -- 2.52.0 From 7feab6d29def86d8f963076c63654a7eb6036d98 Mon Sep 17 00:00:00 2001 From: CleverAgents Bot Date: Tue, 16 Jun 2026 22:45:30 -0400 Subject: [PATCH 2/2] test(auto-debug): exercise message content fixtures --- ...auto_debug_analyze_error_mutation_steps.py | 20 ++++- .../tdd_auto_debug_state_mutation_steps.py | 75 +++++++++++++++++-- ..._auto_debug_analyze_error_mutation.feature | 3 +- .../tdd_auto_debug_state_mutation.feature | 10 +++ 4 files changed, 98 insertions(+), 10 deletions(-) diff --git a/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py b/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py index e61d3c129..1806bd33f 100644 --- a/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py +++ b/features/steps/tdd_auto_debug_analyze_error_mutation_steps.py @@ -30,15 +30,15 @@ from __future__ import annotations from typing import Any from behave import given, then, when -from langchain_community.llms import FakeListLLM +from langchain_core.language_models.fake_chat_models import FakeListChatModel from cleveragents.agents.graphs.auto_debug import AutoDebugAgent, AutoDebugState -@given("an AutoDebugAgent with a FakeListLLM for the mutation test") +@given("an AutoDebugAgent with a chat fake LLM for the mutation test") def step_create_agent_with_fake_llm(context: Any) -> None: - """Create an AutoDebugAgent backed by a FakeListLLM for deterministic testing.""" - context.mutation_test_llm = FakeListLLM( + """Create an AutoDebugAgent backed by a chat fake for deterministic testing.""" + context.mutation_test_llm = FakeListChatModel( responses=["Error analysis: null pointer dereference"] ) context.mutation_test_agent = AutoDebugAgent(llm=context.mutation_test_llm) @@ -136,3 +136,15 @@ def step_assert_result_is_plain_dict(context: Any) -> None: "_analyze_error returned the full state dict instead of an update dict. " f"Expected only changed keys (e.g. {{'messages'}}), got all keys: {result_keys}" ) + + +@then("the analyze_error result should contain the LLM analysis content") +def step_assert_result_contains_llm_content(context: Any) -> None: + """Assert that _analyze_error consumed the message object's content.""" + result = context.mutation_test_result + messages = result.get("messages", []) + assert messages, "_analyze_error returned no messages" + assert messages[-1]["content"] == "Error analysis: null pointer dereference", ( + "_analyze_error did not consume the fake chat model's non-empty " + f".content value: {messages[-1]['content']!r}" + ) diff --git a/features/steps/tdd_auto_debug_state_mutation_steps.py b/features/steps/tdd_auto_debug_state_mutation_steps.py index 0ef7b8ea5..a7db6c1d5 100644 --- a/features/steps/tdd_auto_debug_state_mutation_steps.py +++ b/features/steps/tdd_auto_debug_state_mutation_steps.py @@ -12,7 +12,7 @@ from copy import deepcopy from typing import Any from behave import given, then, when -from langchain_community.llms import FakeListLLM +from langchain_core.language_models.fake_chat_models import FakeListChatModel from cleveragents.agents.graphs.auto_debug import AutoDebugAgent, AutoDebugState @@ -46,7 +46,7 @@ def step_module_imported(context: Any) -> None: def step_create_agent(context: Any) -> None: """Create an AutoDebugAgent with a mock LLM.""" context.mutation_agent = AutoDebugAgent( - llm=FakeListLLM(responses=["Mock LLM response"]), + llm=FakeListChatModel(responses=["Mock LLM response"]), max_fix_attempts=3, ) @@ -63,6 +63,20 @@ def step_initial_state(context: Any) -> None: @given("a state with an error analysis for mutation testing") def step_state_with_analysis(context: Any) -> None: """Create a state with an error analysis message.""" + context.mutation_agent = AutoDebugAgent( + llm=FakeListChatModel( + responses=[ + json.dumps( + { + "description": "Define x before printing", + "code": "x = 42\nprint(x)", + "files_to_modify": ["main.py"], + } + ) + ] + ), + max_fix_attempts=3, + ) context.mutation_input_state = _base_state( messages=[ { @@ -82,7 +96,17 @@ def step_state_with_analysis(context: Any) -> None: def step_state_with_current_fix(context: Any) -> None: """Create a state with a current fix.""" context.mutation_agent = AutoDebugAgent( - llm=FakeListLLM(responses=["Mock LLM response"]), + llm=FakeListChatModel( + responses=[ + json.dumps( + { + "is_valid": True, + "reasoning": "The variable is defined before use.", + "issues": [], + } + ) + ] + ), max_fix_attempts=3, ) context.mutation_input_state = _base_state( @@ -109,7 +133,7 @@ def step_state_with_invalid_validation(context: Any) -> None: } ) context.mutation_agent = AutoDebugAgent( - llm=FakeListLLM(responses=[invalid_response]), + llm=FakeListChatModel(responses=[invalid_response]), max_fix_attempts=3, ) context.mutation_input_state = _base_state( @@ -130,7 +154,7 @@ def step_state_with_invalid_validation(context: Any) -> None: def step_state_for_finalization(context: Any) -> None: """Create a state ready for finalization.""" context.mutation_agent = AutoDebugAgent( - llm=FakeListLLM(responses=["Mock LLM response"]), + llm=FakeListChatModel(responses=["Mock LLM response"]), max_fix_attempts=3, ) context.mutation_input_state = _base_state( @@ -244,3 +268,44 @@ def step_returned_state_has_key(context: Any, key_str: str) -> None: f"Returned state missing expected key '{key}'. " f"Keys present: {list(context.mutation_result_state.keys())}" ) + + +@then("the analyze_error update should contain non-empty LLM content") +def step_analyze_update_contains_llm_content(context: Any) -> None: + """Verify _analyze_error consumed response.content from the chat fake.""" + messages = context.mutation_result_state.get("messages", []) + assert messages, "_analyze_error returned no messages" + content = messages[-1].get("content") + assert content == "Error analysis completed", ( + "_analyze_error should consume the chat fake's .content and map the " + f"configured mock response, got {content!r}" + ) + + +@then("the generated fix should contain the LLM JSON content") +def step_generated_fix_contains_llm_json_content(context: Any) -> None: + """Verify _generate_fix parsed the non-empty response.content JSON.""" + current_fix = context.mutation_result_state.get("current_fix") + assert current_fix == { + "description": "Define x before printing", + "code": "x = 42\nprint(x)", + "files_to_modify": ["main.py"], + }, f"_generate_fix did not parse the fake chat model content: {current_fix!r}" + + +@then("the validation result should come from the LLM JSON content") +def step_validation_result_comes_from_llm_json(context: Any) -> None: + """Verify _validate_fix parsed the non-empty response.content JSON.""" + assert context.mutation_result_state.get("fix_validated") is True, ( + "_validate_fix did not parse the fake chat model validation content: " + f"{context.mutation_result_state!r}" + ) + + +@then("the invalid validation result should come from the LLM JSON content") +def step_invalid_validation_result_comes_from_llm_json(context: Any) -> None: + """Verify _validate_fix used the JSON false response, not an empty fallback.""" + assert context.mutation_result_state.get("fix_validated") is False, ( + "_validate_fix did not parse the fake chat model invalid content: " + f"{context.mutation_result_state!r}" + ) diff --git a/features/tdd_auto_debug_analyze_error_mutation.feature b/features/tdd_auto_debug_analyze_error_mutation.feature index 44377ba3b..6e95f3ac3 100644 --- a/features/tdd_auto_debug_analyze_error_mutation.feature +++ b/features/tdd_auto_debug_analyze_error_mutation.feature @@ -16,9 +16,10 @@ Feature: TDD Issue #10494 — _analyze_error node mutates state in-place instead See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags. Scenario: Bug #10494 — _analyze_error returns the same state object instead of an update dict - Given an AutoDebugAgent with a FakeListLLM for the mutation test + Given an AutoDebugAgent with a chat fake LLM for the mutation test And a state with an empty messages list for the mutation test When _analyze_error is called with the state for the mutation test Then the analyze_error result should not be the same object as the input state And the original messages list should not have been mutated by analyze_error And the analyze_error result should be a plain dict of updates + And the analyze_error result should contain the LLM analysis content diff --git a/features/tdd_auto_debug_state_mutation.feature b/features/tdd_auto_debug_state_mutation.feature index 12d09a11c..84f02e3a2 100644 --- a/features/tdd_auto_debug_state_mutation.feature +++ b/features/tdd_auto_debug_state_mutation.feature @@ -19,6 +19,7 @@ Feature: AutoDebug node functions must not mutate state in-place And an initial auto debug state for mutation testing When I call _analyze_error and capture the result Then the returned state must contain key "messages" + And the analyze_error update should contain non-empty LLM content Scenario: _generate_fix does not mutate the original state object Given an auto debug agent for mutation testing @@ -32,6 +33,7 @@ Feature: AutoDebug node functions must not mutate state in-place And a state with an error analysis for mutation testing When I call _generate_fix and capture the result Then the returned state must contain key "current_fix" + And the generated fix should contain the LLM JSON content Scenario: _validate_fix does not mutate the original state object Given an auto debug agent for mutation testing @@ -46,6 +48,14 @@ Feature: AutoDebug node functions must not mutate state in-place When I call _validate_fix and capture the result Then the returned state must contain key "fix_validated" And the returned state must contain key "attempted_fixes" + And the invalid validation result should come from the LLM JSON content + + Scenario: _validate_fix with valid fix consumes LLM JSON content + Given an auto debug agent for mutation testing + And a state with a current fix for mutation testing + When I call _validate_fix and capture the result + Then the returned state must contain key "fix_validated" + And the validation result should come from the LLM JSON content Scenario: _validate_fix with invalid fix does not mutate attempted_fixes in-place Given an auto debug agent for mutation testing -- 2.52.0