Compare commits

...

5 Commits

Author SHA1 Message Date
HAL9000 a2b5d3b371 fix(tests): use provider/model format in actor YAML model field
CI / helm (pull_request) Successful in 39s
CI / build (pull_request) Successful in 1m17s
CI / lint (pull_request) Failing after 1m33s
CI / typecheck (pull_request) Successful in 1m45s
CI / quality (pull_request) Successful in 1m47s
CI / security (pull_request) Successful in 2m15s
CI / integration_tests (pull_request) Successful in 4m14s
CI / push-validation (pull_request) Successful in 28s
CI / unit_tests (pull_request) Failing after 6m56s
CI / coverage (pull_request) Has been skipped
CI / docker (pull_request) Has been skipped
CI / status-check (pull_request) Failing after 3s
2026-05-15 07:00:16 +00:00
HAL9000 352e61f526 fix(11153): close fail-open security bug and add positive assertions
CI / lint (pull_request) Failing after 1m14s
CI / build (pull_request) Successful in 1m12s
CI / typecheck (pull_request) Successful in 1m39s
CI / quality (pull_request) Successful in 1m24s
CI / helm (pull_request) Successful in 30s
CI / security (pull_request) Successful in 1m59s
CI / push-validation (pull_request) Successful in 31s
CI / integration_tests (pull_request) Successful in 12m47s
CI / unit_tests (pull_request) Failing after 18m9s
CI / coverage (pull_request) Has been skipped
CI / docker (pull_request) Has been skipped
CI / status-check (pull_request) Failing after 4s
- _validate_fix exception handler defaults to False (not True),
  preventing crashed LLM validators from passing unvalidated fixes.
- Added positive test assertions verifying returned partial-state dicts
  contain expected keys (messages, current_fix, fix_validated,
  attempted_fixes, result). This closes a coverage blind spot where
  an empty return dict would silently pass immutability tests.

ISSUES CLOSED: #10496
2026-05-15 04:55:22 +00:00
HAL9000 55bea49585 fix(11153): address peer review findings for PR #11153
- Fix CHANGELOG entry: replace non-existent update_node()/set_active_node() with
  actual method names (_analyze_error, _generate_fix, _validate_fix, _finalize)
  and clarify LangGraph node contract reference (HAL9000 observation #1).

- Add inline comment explaining shallow-copy semantics in _analyze_error's
  immutability pattern (HAL9000 observation #2): list is new but inner dicts
  are shared refs — safe because messages are immutable-after-creation.

- Document return-asymmetry in _validate_fix docstring: both fix_validated and
  attempted_fixes keys when invalid, only fix_validated when valid. This is
  intentional LangGraph behavior (omitted keys are not reset) (HAL9000 obs #4).

Addresses review comments from peer review #8822 (HAL9000).
2026-05-15 04:55:22 +00:00
HAL9000 79a5bdea5a test(auto_debug): add missing @tdd_issue tags per CI quality gate (issue #10496)
The feature file for auto-debug state mutation tests was missing required
TDD tags (@tdd_issue, @tdd_issue_10496). This caused CI / tdd_quality_gate
to fail the tag validation check. Added minimal tagging to pass CI while
keeping the fix PR's scenarios passing (no @tdd_expected_fail needed since
the underlying bug is being fixed).
2026-05-15 04:55:22 +00:00
freemo 0bad313aa3 fix(agents/graphs/auto_debug): return update dicts from node functions instead of mutating state in-place
ISSUES CLOSED: #10494
#10496
2026-05-15 04:55:22 +00:00
5 changed files with 396 additions and 28 deletions
+2
View File
@@ -232,6 +232,8 @@ Changed `wf10_batch.robot` to be less likely to create files, and
`hot_context_hash`, `hot_context_ref`, `actor_state_ref`, and `relevant_resources`
are all populated for Strategize-phase decisions.
- **`auto_debug` node functions return partial state updates instead of mutating state** (#10494, #10496): The `_analyze_error()`, `_generate_fix()`, `_validate_fix()`, and `_finalize()` methods in `src/cleveragents/agents/graphs/auto_debug.py` now return new `dict[str, Any]` objects containing only the keys they update rather than mutating the input state in-place or returning full state objects. This respects the LangGraph node contract (all `StateGraph` node functions are passed the current state and must return a partial dict representing only their incremental updates). Callers rely on LangGraph's built-in state delta-merge logic to combine these partial updates across successive nodes.
### Changed
- Fixed stale `AUTO-BUG-POOL` tracking prefix references in automation-tracking.md documentation and agent-system-specification.md spec document, replaced with correct `AUTO-BUG-SUP` prefix used by the bug-hunt-pool-supervisor agent (#7875).
@@ -0,0 +1,260 @@
"""Step definitions for tdd_auto_debug_state_mutation.feature.
These steps verify that AutoDebug node functions return new state dicts
instead of mutating the input state in-place, respecting the LangGraph
node contract.
"""
from __future__ import annotations
import json
from copy import deepcopy
from typing import Any
from behave import given, then, when
from cleveragents.agents.graphs.auto_debug import AutoDebugAgent, AutoDebugState
class _MockResponse:
"""Minimal response object with a content attribute."""
def __init__(self, content: str) -> None:
self.content = content
class _MockLLM:
"""Stub LLM that returns a fixed content string."""
def __init__(self, content: str = "Mock LLM response") -> None:
self._content = content
def invoke(self, messages: Any) -> _MockResponse:
return _MockResponse(self._content)
class _InvalidValidationLLM:
"""Stub LLM that returns an invalid validation response."""
def invoke(self, messages: Any) -> _MockResponse:
return _MockResponse(
json.dumps(
{
"is_valid": False,
"reasoning": "Fix failed",
"issues": ["error persists"],
}
)
)
def _base_state(**overrides: Any) -> AutoDebugState:
"""Create a minimal valid AutoDebugState with optional overrides."""
state: AutoDebugState = {
"messages": [],
"context": {},
"result": None,
"error": None,
"metadata": {},
"error_message": "NameError: name 'x' is not defined",
"code_context": "print(x)",
"attempted_fixes": [],
"current_fix": {},
"fix_validated": False,
}
state.update(overrides) # type: ignore[typeddict-item]
return state
@given("the auto debug state mutation module is imported")
def step_module_imported(context: Any) -> None:
"""Verify the auto_debug module is importable."""
assert AutoDebugAgent is not None
assert AutoDebugState is not None
@given("an auto debug agent for mutation testing")
def step_create_agent(context: Any) -> None:
"""Create an AutoDebugAgent with a mock LLM."""
context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3)
@given("an initial auto debug state for mutation testing")
def step_initial_state(context: Any) -> None:
"""Create an initial state with empty messages list."""
context.mutation_input_state = _base_state(messages=[])
context.mutation_original_messages_len = len(
context.mutation_input_state["messages"]
)
@given("a state with an error analysis for mutation testing")
def step_state_with_analysis(context: Any) -> None:
"""Create a state with an error analysis message."""
context.mutation_input_state = _base_state(
messages=[
{
"role": "assistant",
"content": "Detected a NameError",
"type": "error_analysis",
}
],
current_fix={},
)
context.mutation_original_current_fix = deepcopy(
context.mutation_input_state["current_fix"]
)
@given("a state with a current fix for mutation testing")
def step_state_with_current_fix(context: Any) -> None:
"""Create a state with a current fix."""
context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3)
context.mutation_input_state = _base_state(
current_fix={
"description": "Proposed fix",
"code": "x = 42\nprint(x)",
"files_to_modify": ["main.py"],
},
fix_validated=False,
)
context.mutation_original_fix_validated = context.mutation_input_state[
"fix_validated"
]
@given("a state with a current fix and invalid validation for mutation testing")
def step_state_with_invalid_validation(context: Any) -> None:
"""Create a state with a current fix that will fail validation."""
context.mutation_agent = AutoDebugAgent(
llm=_InvalidValidationLLM(), max_fix_attempts=3
)
context.mutation_input_state = _base_state(
current_fix={
"description": "Proposed fix",
"code": "x = 42\nprint(x)",
"files_to_modify": ["main.py"],
},
attempted_fixes=[],
fix_validated=False,
)
context.mutation_original_attempted_fixes_len = len(
context.mutation_input_state["attempted_fixes"]
)
@given("a state ready for finalization for mutation testing")
def step_state_for_finalization(context: Any) -> None:
"""Create a state ready for finalization."""
context.mutation_agent = AutoDebugAgent(llm=_MockLLM(), max_fix_attempts=3)
context.mutation_input_state = _base_state(
fix_validated=True,
current_fix={"description": "Final fix", "code": "# fixed"},
result=None,
)
context.mutation_original_result = context.mutation_input_state["result"]
@when("I call _analyze_error and capture the result")
def step_call_analyze_error(context: Any) -> None:
"""Call _analyze_error and capture both input and output."""
context.mutation_result_state = context.mutation_agent._analyze_error(
context.mutation_input_state
)
@when("I call _generate_fix and capture the result")
def step_call_generate_fix(context: Any) -> None:
"""Call _generate_fix and capture both input and output."""
context.mutation_result_state = context.mutation_agent._generate_fix(
context.mutation_input_state
)
@when("I call _validate_fix and capture the result")
def step_call_validate_fix(context: Any) -> None:
"""Call _validate_fix and capture both input and output."""
context.mutation_result_state = context.mutation_agent._validate_fix(
context.mutation_input_state
)
@when("I call _finalize and capture the result")
def step_call_finalize(context: Any) -> None:
"""Call _finalize and capture both input and output."""
context.mutation_result_state = context.mutation_agent._finalize(
context.mutation_input_state
)
@then("the returned state should be a different object from the input state")
def step_different_object(context: Any) -> None:
"""Verify the returned state is a new dict, not the same object."""
assert context.mutation_result_state is not context.mutation_input_state, (
"Node function returned the same state object (mutated in-place). "
"LangGraph node functions must return a new dict."
)
@then("the original state messages list should be unchanged")
def step_messages_unchanged(context: Any) -> None:
"""Verify the original state's messages list was not mutated."""
original_len = context.mutation_original_messages_len
actual_len = len(context.mutation_input_state["messages"])
assert actual_len == original_len, (
f"Original state messages list was mutated: "
f"expected {original_len} messages, got {actual_len}"
)
@then("the original state current_fix should be unchanged")
def step_current_fix_unchanged(context: Any) -> None:
"""Verify the original state's current_fix was not mutated."""
original = context.mutation_original_current_fix
actual = context.mutation_input_state["current_fix"]
assert actual == original, (
f"Original state current_fix was mutated: expected {original!r}, got {actual!r}"
)
@then("the original state fix_validated should be unchanged")
def step_fix_validated_unchanged(context: Any) -> None:
"""Verify the original state's fix_validated was not mutated."""
original = context.mutation_original_fix_validated
actual = context.mutation_input_state["fix_validated"]
assert actual == original, (
f"Original state fix_validated was mutated: "
f"expected {original!r}, got {actual!r}"
)
@then("the original state attempted_fixes list should be unchanged")
def step_attempted_fixes_unchanged(context: Any) -> None:
"""Verify the original state's attempted_fixes list was not mutated."""
original_len = context.mutation_original_attempted_fixes_len
actual_len = len(context.mutation_input_state["attempted_fixes"])
assert actual_len == original_len, (
f"Original state attempted_fixes list was mutated: "
f"expected {original_len} items, got {actual_len}"
)
@then("the original state result should be unchanged")
def step_result_unchanged(context: Any) -> None:
"""Verify the original state's result was not mutated."""
original = context.mutation_original_result
actual = context.mutation_input_state["result"]
assert actual == original, (
f"Original state result was mutated: expected {original!r}, got {actual!r}"
)
@then("the returned state must contain key {key_str}")
def step_returned_state_has_key(context: Any, key_str: str) -> None:
"""Verify the returned partial-state dict contains the expected key."""
# Strip quotes if present (e.g. '"messages"' or "messages")
key = key_str.strip("\"'")
assert key in context.mutation_result_state, (
f"Returned state missing expected key '{key}'. "
f"Keys present: {list(context.mutation_result_state.keys())}"
)
@@ -0,0 +1,68 @@
@tdd_issue @tdd_issue_10496
Feature: AutoDebug node functions must not mutate state in-place
As a LangGraph developer
I want AutoDebug node functions to return new state dicts
So that the LangGraph node contract is respected and state isolation is maintained
Background:
Given the auto debug state mutation module is imported
Scenario: _analyze_error does not mutate the original state object
Given an auto debug agent for mutation testing
And an initial auto debug state for mutation testing
When I call _analyze_error and capture the result
Then the returned state should be a different object from the input state
And the original state messages list should be unchanged
Scenario: _analyze_error returns expected keys in partial state dict
Given an auto debug agent for mutation testing
And an initial auto debug state for mutation testing
When I call _analyze_error and capture the result
Then the returned state must contain key "messages"
Scenario: _generate_fix does not mutate the original state object
Given an auto debug agent for mutation testing
And a state with an error analysis for mutation testing
When I call _generate_fix and capture the result
Then the returned state should be a different object from the input state
And the original state current_fix should be unchanged
Scenario: _generate_fix returns expected keys in partial state dict
Given an auto debug agent for mutation testing
And a state with an error analysis for mutation testing
When I call _generate_fix and capture the result
Then the returned state must contain key "current_fix"
Scenario: _validate_fix does not mutate the original state object
Given an auto debug agent for mutation testing
And a state with a current fix for mutation testing
When I call _validate_fix and capture the result
Then the returned state should be a different object from the input state
And the original state fix_validated should be unchanged
Scenario: _validate_fix (invalid) returns expected keys in partial state dict
Given an auto debug agent for mutation testing
And a state with a current fix and invalid validation for mutation testing
When I call _validate_fix and capture the result
Then the returned state must contain key "fix_validated"
And the returned state must contain key "attempted_fixes"
Scenario: _validate_fix with invalid fix does not mutate attempted_fixes in-place
Given an auto debug agent for mutation testing
And a state with a current fix and invalid validation for mutation testing
When I call _validate_fix and capture the result
Then the returned state should be a different object from the input state
And the original state attempted_fixes list should be unchanged
Scenario: _finalize does not mutate the original state object
Given an auto debug agent for mutation testing
And a state ready for finalization for mutation testing
When I call _finalize and capture the result
Then the returned state should be a different object from the input state
And the original state result should be unchanged
Scenario: _finalize returns expected keys in partial state dict
Given an auto debug agent for mutation testing
And a state ready for finalization for mutation testing
When I call _finalize and capture the result
Then the returned state must contain key "result"
+1 -1
View File
@@ -129,7 +129,7 @@ class ActorRegistry:
yaml_dict: dict[str, Any] = {
"name": self._actor_name(provider, model),
"type": "llm",
"model": model,
"model": f"{provider}/{model}",
"description": (
f"Built-in actor from provider registry ({provider}/{model})"
),
+65 -27
View File
@@ -89,7 +89,12 @@ class AutoDebugAgent:
return workflow
def _analyze_error(self, state: AutoDebugState) -> AutoDebugState:
def _analyze_error(self, state: AutoDebugState) -> dict[str, Any]:
"""Analyze the error message and return updated messages.
Returns a new partial state dict with the updated messages list,
without mutating the input state.
"""
logger.info("Analyzing error message")
error_msg = state.get("error_message", "")
@@ -129,17 +134,28 @@ Analyze this error and provide insights."""
logger.warning("LLM analysis failed, using fallback: %s", exc)
analysis = "Error analysis completed"
state.setdefault("messages", []).append(
{
"role": "assistant",
"content": analysis,
"type": "error_analysis",
}
)
new_message: dict[str, Any] = {
"role": "assistant",
"content": analysis,
"type": "error_analysis",
}
# Unpack operator creates a new list (not in-place mutation), but inner
# message dicts are shared by reference. This is safe because individual
# message dicts are immutable-after-creation; LangGraph's state merging
# replaces them rather than mutating dict contents.
updated_messages: list[dict[str, Any]] = [
*state.get("messages", []),
new_message,
]
return state
return {"messages": updated_messages}
def _generate_fix(self, state: AutoDebugState) -> AutoDebugState:
def _generate_fix(self, state: AutoDebugState) -> dict[str, Any]:
"""Generate a fix suggestion and return updated current_fix.
Returns a new partial state dict with the updated current_fix,
without mutating the input state.
"""
logger.info("Generating fix suggestion")
error_analysis = next(
@@ -220,10 +236,21 @@ Generate fix attempt #{attempt_num}."""
"files_to_modify": [],
}
state["current_fix"] = fix_data
return state
return {"current_fix": fix_data}
def _validate_fix(self, state: AutoDebugState) -> AutoDebugState:
def _validate_fix(self, state: AutoDebugState) -> dict[str, Any]:
"""Validate the current fix and return updated validation state.
Returns a new partial state dict with the updated ``fix_validated``
and (when invalid) ``attempted_fixes`` fields, without mutating the
input state.
The returned dict has **asymmetric key coverage**: when the fix is
invalid, both ``"fix_validated"`` and ``"attempted_fixes"`` are
included; when valid, only ``"fix_validated"`` is returned. This is
intentional LangGraph merges partial dicts into state, so omitted
keys are left unmodified rather than reset to defaults.
"""
logger.info("Validating fix")
current_fix = state.get("current_fix", {})
@@ -275,17 +302,22 @@ Validate this fix."""
for word in ["valid", "correct", "resolves", "fixes"]
)
except Exception as exc: # pragma: no cover - defensive
logger.warning("LLM validation failed, using fallback: %s", exc)
is_valid = True
state["fix_validated"] = is_valid
logger.warning(
"LLM validation failed, treating fix as invalid: %s", exc
)
is_valid = False
if not is_valid:
attempted_fixes = state.get("attempted_fixes", [])
attempted_fixes.append(current_fix)
state["attempted_fixes"] = attempted_fixes
updated_attempted_fixes: list[dict[str, Any]] = [
*state.get("attempted_fixes", []),
current_fix,
]
return {
"fix_validated": is_valid,
"attempted_fixes": updated_attempted_fixes,
}
return state
return {"fix_validated": is_valid}
def _should_retry_fix(self, state: AutoDebugState) -> str:
if not state.get("fix_validated", False):
@@ -294,15 +326,21 @@ Validate this fix."""
return "retry"
return "done"
def _finalize(self, state: AutoDebugState) -> AutoDebugState:
def _finalize(self, state: AutoDebugState) -> dict[str, Any]:
"""Finalize the auto-debug results and return updated result.
Returns a new partial state dict with the result field populated,
without mutating the input state.
"""
logger.info("Finalizing auto-debug results")
state["result"] = {
"success": state.get("fix_validated", False),
"fix": state.get("current_fix"),
"attempts": len(state.get("attempted_fixes", [])),
return {
"result": {
"success": state.get("fix_validated", False),
"fix": state.get("current_fix"),
"attempts": len(state.get("attempted_fixes", [])),
}
}
return state
def invoke(
self, input_state: AutoDebugState, config: dict[str, Any] | None = None