fix(11153): close fail-open security bug and add positive assertions
- _validate_fix exception handler defaults to False (not True), preventing crashed LLM validators from passing unvalidated fixes. - Added positive test assertions verifying returned partial-state dicts contain expected keys (messages, current_fix, fix_validated, attempted_fixes, result). This closes a coverage blind spot where an empty return dict would silently pass immutability tests. ISSUES CLOSED: #10496
This commit is contained in:
@@ -247,3 +247,14 @@ def step_result_unchanged(context: Any) -> None:
|
||||
assert actual == original, (
|
||||
f"Original state result was mutated: expected {original!r}, got {actual!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the returned state must contain key {key_str}")
|
||||
def step_returned_state_has_key(context: Any, key_str: str) -> None:
|
||||
"""Verify the returned partial-state dict contains the expected key."""
|
||||
# Strip quotes if present (e.g. '"messages"' or "messages")
|
||||
key = key_str.strip("\"'")
|
||||
assert key in context.mutation_result_state, (
|
||||
f"Returned state missing expected key '{key}'. "
|
||||
f"Keys present: {list(context.mutation_result_state.keys())}"
|
||||
)
|
||||
|
||||
@@ -14,6 +14,12 @@ Feature: AutoDebug node functions must not mutate state in-place
|
||||
Then the returned state should be a different object from the input state
|
||||
And the original state messages list should be unchanged
|
||||
|
||||
Scenario: _analyze_error returns expected keys in partial state dict
|
||||
Given an auto debug agent for mutation testing
|
||||
And an initial auto debug state for mutation testing
|
||||
When I call _analyze_error and capture the result
|
||||
Then the returned state must contain key "messages"
|
||||
|
||||
Scenario: _generate_fix does not mutate the original state object
|
||||
Given an auto debug agent for mutation testing
|
||||
And a state with an error analysis for mutation testing
|
||||
@@ -21,6 +27,12 @@ Feature: AutoDebug node functions must not mutate state in-place
|
||||
Then the returned state should be a different object from the input state
|
||||
And the original state current_fix should be unchanged
|
||||
|
||||
Scenario: _generate_fix returns expected keys in partial state dict
|
||||
Given an auto debug agent for mutation testing
|
||||
And a state with an error analysis for mutation testing
|
||||
When I call _generate_fix and capture the result
|
||||
Then the returned state must contain key "current_fix"
|
||||
|
||||
Scenario: _validate_fix does not mutate the original state object
|
||||
Given an auto debug agent for mutation testing
|
||||
And a state with a current fix for mutation testing
|
||||
@@ -28,6 +40,13 @@ Feature: AutoDebug node functions must not mutate state in-place
|
||||
Then the returned state should be a different object from the input state
|
||||
And the original state fix_validated should be unchanged
|
||||
|
||||
Scenario: _validate_fix (invalid) returns expected keys in partial state dict
|
||||
Given an auto debug agent for mutation testing
|
||||
And a state with a current fix and invalid validation for mutation testing
|
||||
When I call _validate_fix and capture the result
|
||||
Then the returned state must contain key "fix_validated"
|
||||
And the returned state must contain key "attempted_fixes"
|
||||
|
||||
Scenario: _validate_fix with invalid fix does not mutate attempted_fixes in-place
|
||||
Given an auto debug agent for mutation testing
|
||||
And a state with a current fix and invalid validation for mutation testing
|
||||
@@ -41,3 +60,9 @@ Feature: AutoDebug node functions must not mutate state in-place
|
||||
When I call _finalize and capture the result
|
||||
Then the returned state should be a different object from the input state
|
||||
And the original state result should be unchanged
|
||||
|
||||
Scenario: _finalize returns expected keys in partial state dict
|
||||
Given an auto debug agent for mutation testing
|
||||
And a state ready for finalization for mutation testing
|
||||
When I call _finalize and capture the result
|
||||
Then the returned state must contain key "result"
|
||||
|
||||
@@ -302,8 +302,10 @@ Validate this fix."""
|
||||
for word in ["valid", "correct", "resolves", "fixes"]
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning("LLM validation failed, using fallback: %s", exc)
|
||||
is_valid = True
|
||||
logger.warning(
|
||||
"LLM validation failed, treating fix as invalid: %s", exc
|
||||
)
|
||||
is_valid = False
|
||||
|
||||
if not is_valid:
|
||||
updated_attempted_fixes: list[dict[str, Any]] = [
|
||||
|
||||
Reference in New Issue
Block a user