Merge pull request 'fix(agents/graphs/auto_debug): return update dicts from node functions instead of mutating state in-place' (#11155) from fix/issue-10496-auto-debug-state-mutation into master

features/steps/tdd_auto_debug_analyze_error_mutation_steps.py

@@ -30,15 +30,15 @@ from __future__ import annotations
 from typing import Any
 
 from behave import given, then, when
-from langchain_community.llms import FakeListLLM
+from langchain_core.language_models.fake_chat_models import FakeListChatModel
 
 from cleveragents.agents.graphs.auto_debug import AutoDebugAgent, AutoDebugState
 
 
-@given("an AutoDebugAgent with a FakeListLLM for the mutation test")
+@given("an AutoDebugAgent with a chat fake LLM for the mutation test")
 def step_create_agent_with_fake_llm(context: Any) -> None:
-    """Create an AutoDebugAgent backed by a FakeListLLM for deterministic testing."""
-    context.mutation_test_llm = FakeListLLM(
+    """Create an AutoDebugAgent backed by a chat fake for deterministic testing."""
+    context.mutation_test_llm = FakeListChatModel(
         responses=["Error analysis: null pointer dereference"]
     )
     context.mutation_test_agent = AutoDebugAgent(llm=context.mutation_test_llm)
@@ -47,7 +47,7 @@ def step_create_agent_with_fake_llm(context: Any) -> None:
 @given("a state with an empty messages list for the mutation test")
 def step_create_state_with_empty_messages(context: Any) -> None:
     """Create a minimal AutoDebugState with an empty messages list."""
-    context.mutation_test_state: AutoDebugState = {
+    state: AutoDebugState = {
         "error_message": "NullPointerException at line 42",
         "code_context": "x = obj.method()",
         "messages": [],
@@ -59,6 +59,7 @@ def step_create_state_with_empty_messages(context: Any) -> None:
         "current_fix": {},
         "fix_validated": False,
     }
+    context.mutation_test_state = state
     # Capture a reference to the original messages list before calling the node
     context.mutation_test_original_messages = context.mutation_test_state["messages"]
 
@@ -135,3 +136,15 @@ def step_assert_result_is_plain_dict(context: Any) -> None:
         "_analyze_error returned the full state dict instead of an update dict. "
         f"Expected only changed keys (e.g. {{'messages'}}), got all keys: {result_keys}"
     )
+
+
+@then("the analyze_error result should contain the LLM analysis content")
+def step_assert_result_contains_llm_content(context: Any) -> None:
+    """Assert that _analyze_error consumed the message object's content."""
+    result = context.mutation_test_result
+    messages = result.get("messages", [])
+    assert messages, "_analyze_error returned no messages"
+    assert messages[-1]["content"] == "Error analysis: null pointer dereference", (
+        "_analyze_error did not consume the fake chat model's non-empty "
+        f".content value: {messages[-1]['content']!r}"
+    )

features/steps/tdd_auto_debug_state_mutation_steps.py

@@ -12,42 +12,11 @@ from copy import deepcopy
 from typing import Any
 
 from behave import given, then, when
+from langchain_core.language_models.fake_chat_models import FakeListChatModel
 
 from cleveragents.agents.graphs.auto_debug import AutoDebugAgent, AutoDebugState
 
 
-class _MockResponse:
-    """Minimal response object with a content attribute."""
-
-    def __init__(self, content: str) -> None:
-        self.content = content
-
-
-class _MockLLM:
-    """Stub LLM that returns a fixed content string."""
-
-    def __init__(self, content: str = "Mock LLM response") -> None:
-        self._content = content
-
-    def invoke(self, messages: Any) -> _MockResponse:
-        return _MockResponse(self._content)
-
-
-class _InvalidValidationLLM:
-    """Stub LLM that returns an invalid validation response."""
-
-    def invoke(self, messages: Any) -> _MockResponse:
-        return _MockResponse(
-            json.dumps(
-                {
-                    "is_valid": False,
-                    "reasoning": "Fix failed",
-                    "issues": ["error persists"],
-                }
-            )
-        )
-
-
 def _base_state(**overrides: Any) -> AutoDebugState:
     """Create a minimal valid AutoDebugState with optional overrides."""
     state: AutoDebugState = {
@@ -76,7 +45,10 @@ def step_module_imported(context: Any) -> None:
 @given("an auto debug agent for mutation testing")
 def step_create_agent(context: Any) -> None:
     """Create an AutoDebugAgent with a mock LLM."""
-    context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3)
+    context.mutation_agent = AutoDebugAgent(
+        llm=FakeListChatModel(responses=["Mock LLM response"]),
+        max_fix_attempts=3,
+    )
 
 
 @given("an initial auto debug state for mutation testing")
@@ -91,6 +63,20 @@ def step_initial_state(context: Any) -> None:
 @given("a state with an error analysis for mutation testing")
 def step_state_with_analysis(context: Any) -> None:
     """Create a state with an error analysis message."""
+    context.mutation_agent = AutoDebugAgent(
+        llm=FakeListChatModel(
+            responses=[
+                json.dumps(
+                    {
+                        "description": "Define x before printing",
+                        "code": "x = 42\nprint(x)",
+                        "files_to_modify": ["main.py"],
+                    }
+                )
+            ]
+        ),
+        max_fix_attempts=3,
+    )
     context.mutation_input_state = _base_state(
         messages=[
             {
@@ -109,7 +95,20 @@ def step_state_with_analysis(context: Any) -> None:
 @given("a state with a current fix for mutation testing")
 def step_state_with_current_fix(context: Any) -> None:
     """Create a state with a current fix."""
-    context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3)
+    context.mutation_agent = AutoDebugAgent(
+        llm=FakeListChatModel(
+            responses=[
+                json.dumps(
+                    {
+                        "is_valid": True,
+                        "reasoning": "The variable is defined before use.",
+                        "issues": [],
+                    }
+                )
+            ]
+        ),
+        max_fix_attempts=3,
+    )
     context.mutation_input_state = _base_state(
         current_fix={
             "description": "Proposed fix",
@@ -126,8 +125,16 @@ def step_state_with_current_fix(context: Any) -> None:
 @given("a state with a current fix and invalid validation for mutation testing")
 def step_state_with_invalid_validation(context: Any) -> None:
     """Create a state with a current fix that will fail validation."""
+    invalid_response = json.dumps(
+        {
+            "is_valid": False,
+            "reasoning": "Fix failed",
+            "issues": ["error persists"],
+        }
+    )
     context.mutation_agent = AutoDebugAgent(
-        llm=_InvalidValidationLLM(), max_fix_attempts=3
+        llm=FakeListChatModel(responses=[invalid_response]),
+        max_fix_attempts=3,
     )
     context.mutation_input_state = _base_state(
         current_fix={
@@ -146,7 +153,10 @@ def step_state_with_invalid_validation(context: Any) -> None:
 @given("a state ready for finalization for mutation testing")
 def step_state_for_finalization(context: Any) -> None:
     """Create a state ready for finalization."""
-    context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3)
+    context.mutation_agent = AutoDebugAgent(
+        llm=FakeListChatModel(responses=["Mock LLM response"]),
+        max_fix_attempts=3,
+    )
     context.mutation_input_state = _base_state(
         fix_validated=True,
         current_fix={"description": "Final fix", "code": "# fixed"},
@@ -258,3 +268,44 @@ def step_returned_state_has_key(context: Any, key_str: str) -> None:
         f"Returned state missing expected key '{key}'. "
         f"Keys present: {list(context.mutation_result_state.keys())}"
     )
+
+
+@then("the analyze_error update should contain non-empty LLM content")
+def step_analyze_update_contains_llm_content(context: Any) -> None:
+    """Verify _analyze_error consumed response.content from the chat fake."""
+    messages = context.mutation_result_state.get("messages", [])
+    assert messages, "_analyze_error returned no messages"
+    content = messages[-1].get("content")
+    assert content == "Error analysis completed", (
+        "_analyze_error should consume the chat fake's .content and map the "
+        f"configured mock response, got {content!r}"
+    )
+
+
+@then("the generated fix should contain the LLM JSON content")
+def step_generated_fix_contains_llm_json_content(context: Any) -> None:
+    """Verify _generate_fix parsed the non-empty response.content JSON."""
+    current_fix = context.mutation_result_state.get("current_fix")
+    assert current_fix == {
+        "description": "Define x before printing",
+        "code": "x = 42\nprint(x)",
+        "files_to_modify": ["main.py"],
+    }, f"_generate_fix did not parse the fake chat model content: {current_fix!r}"
+
+
+@then("the validation result should come from the LLM JSON content")
+def step_validation_result_comes_from_llm_json(context: Any) -> None:
+    """Verify _validate_fix parsed the non-empty response.content JSON."""
+    assert context.mutation_result_state.get("fix_validated") is True, (
+        "_validate_fix did not parse the fake chat model validation content: "
+        f"{context.mutation_result_state!r}"
+    )
+
+
+@then("the invalid validation result should come from the LLM JSON content")
+def step_invalid_validation_result_comes_from_llm_json(context: Any) -> None:
+    """Verify _validate_fix used the JSON false response, not an empty fallback."""
+    assert context.mutation_result_state.get("fix_validated") is False, (
+        "_validate_fix did not parse the fake chat model invalid content: "
+        f"{context.mutation_result_state!r}"
+    )

features/tdd_auto_debug_analyze_error_mutation.feature

@@ -16,9 +16,10 @@ Feature: TDD Issue #10494 — _analyze_error node mutates state in-place instead
   See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags.
 
   Scenario: Bug #10494 — _analyze_error returns the same state object instead of an update dict
-    Given an AutoDebugAgent with a FakeListLLM for the mutation test
+    Given an AutoDebugAgent with a chat fake LLM for the mutation test
     And a state with an empty messages list for the mutation test
     When _analyze_error is called with the state for the mutation test
     Then the analyze_error result should not be the same object as the input state
     And the original messages list should not have been mutated by analyze_error
     And the analyze_error result should be a plain dict of updates
+    And the analyze_error result should contain the LLM analysis content

features/tdd_auto_debug_state_mutation.feature

@@ -19,6 +19,7 @@ Feature: AutoDebug node functions must not mutate state in-place
     And an initial auto debug state for mutation testing
     When I call _analyze_error and capture the result
     Then the returned state must contain key "messages"
+    And the analyze_error update should contain non-empty LLM content
 
   Scenario: _generate_fix does not mutate the original state object
     Given an auto debug agent for mutation testing
@@ -32,6 +33,7 @@ Feature: AutoDebug node functions must not mutate state in-place
     And a state with an error analysis for mutation testing
     When I call _generate_fix and capture the result
     Then the returned state must contain key "current_fix"
+    And the generated fix should contain the LLM JSON content
 
   Scenario: _validate_fix does not mutate the original state object
     Given an auto debug agent for mutation testing
@@ -46,6 +48,14 @@ Feature: AutoDebug node functions must not mutate state in-place
     When I call _validate_fix and capture the result
     Then the returned state must contain key "fix_validated"
     And the returned state must contain key "attempted_fixes"
+    And the invalid validation result should come from the LLM JSON content
+
+  Scenario: _validate_fix with valid fix consumes LLM JSON content
+    Given an auto debug agent for mutation testing
+    And a state with a current fix for mutation testing
+    When I call _validate_fix and capture the result
+    Then the returned state must contain key "fix_validated"
+    And the validation result should come from the LLM JSON content
 
   Scenario: _validate_fix with invalid fix does not mutate attempted_fixes in-place
     Given an auto debug agent for mutation testing