forked from HAL9000/cleveragents-core
1138 lines
38 KiB
Python
1138 lines
38 KiB
Python
"""Step definitions for auto debug agent coverage tests."""
|
|
|
|
import json
|
|
import logging
|
|
from unittest.mock import MagicMock, Mock, patch
|
|
|
|
from behave import given, then, when
|
|
|
|
|
|
@given("the auto debug agent module is importable")
|
|
def step_auto_debug_importable(context):
|
|
"""Verify the auto debug module can be imported."""
|
|
try:
|
|
from cleveragents.application.agents.auto_debug import (
|
|
AutoDebugAgent,
|
|
AutoDebugState,
|
|
)
|
|
|
|
context.AutoDebugAgent = AutoDebugAgent
|
|
context.AutoDebugState = AutoDebugState
|
|
context.import_error = None
|
|
except ImportError as e:
|
|
context.import_error = str(e)
|
|
raise AssertionError(f"Failed to import auto debug module: {e}") from e
|
|
|
|
|
|
@given("I have a mock LLM provider configured for auto debug")
|
|
def step_have_mock_llm_provider(context):
|
|
"""Set up a mock LLM provider for auto debug agent testing."""
|
|
context.mock_llm = MagicMock()
|
|
context.mock_llm.invoke = MagicMock()
|
|
|
|
# Create a mock response
|
|
mock_response = Mock()
|
|
mock_response.content = "Mock LLM response"
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@when("I create an AutoDebugAgent with default parameters")
|
|
def step_create_auto_debug_agent_default(context):
|
|
"""Create an AutoDebugAgent instance with default parameters."""
|
|
with patch(
|
|
"cleveragents.application.agents.base_agent.BaseAgent._create_llm"
|
|
) as mock_create_llm:
|
|
mock_create_llm.return_value = context.mock_llm
|
|
context.agent = context.AutoDebugAgent()
|
|
|
|
|
|
@then("the agent should be initialized successfully for auto debug")
|
|
def step_agent_initialized_successfully(context):
|
|
"""Verify the agent was initialized."""
|
|
assert context.agent is not None
|
|
assert hasattr(context.agent, "max_fix_attempts")
|
|
|
|
|
|
@then("the agent should have a max_fix_attempts attribute set to {value:d}")
|
|
def step_agent_has_max_fix_attempts(context, value):
|
|
"""Verify max_fix_attempts attribute value."""
|
|
assert context.agent.max_fix_attempts == value
|
|
|
|
|
|
@then("the agent should have an llm provider configured for auto debug")
|
|
def step_agent_has_llm_provider(context):
|
|
"""Verify LLM provider is configured."""
|
|
assert hasattr(context.agent, "llm")
|
|
|
|
|
|
@when("I create an AutoDebugAgent with parameters:")
|
|
def step_create_auto_debug_agent_with_params(context):
|
|
"""Create AutoDebugAgent with custom parameters from table."""
|
|
params = {}
|
|
for row in context.table:
|
|
param = row["parameter"]
|
|
value = row["value"]
|
|
|
|
# Convert value to appropriate type
|
|
if param == "temperature":
|
|
params[param] = float(value)
|
|
elif param == "max_fix_attempts":
|
|
params[param] = int(value)
|
|
else:
|
|
params[param] = value
|
|
|
|
with patch(
|
|
"cleveragents.application.agents.base_agent.BaseAgent._create_llm"
|
|
) as mock_create_llm:
|
|
mock_create_llm.return_value = context.mock_llm
|
|
context.agent = context.AutoDebugAgent(**params)
|
|
|
|
|
|
@then("the agent max_fix_attempts should be {value:d}")
|
|
def step_agent_max_fix_attempts_value(context, value):
|
|
"""Check max_fix_attempts value."""
|
|
assert context.agent.max_fix_attempts == value
|
|
|
|
|
|
@then("the agent temperature should be {value:f} for auto debug")
|
|
def step_agent_temperature_value(context, value):
|
|
"""Check temperature value."""
|
|
assert context.agent.temperature == value
|
|
|
|
|
|
@given("I have an AutoDebugAgent instance")
|
|
def step_have_auto_debug_agent_instance(context):
|
|
"""Create a basic AutoDebugAgent instance."""
|
|
with patch(
|
|
"cleveragents.application.agents.base_agent.BaseAgent._create_llm"
|
|
) as mock_create_llm:
|
|
mock_create_llm.return_value = context.mock_llm
|
|
context.agent = context.AutoDebugAgent()
|
|
|
|
|
|
@when("I build the workflow graph for auto debug")
|
|
def step_build_workflow_graph(context):
|
|
"""Build the workflow graph."""
|
|
with patch("langgraph.graph.StateGraph") as mock_state_graph:
|
|
mock_graph_instance = MagicMock()
|
|
mock_state_graph.return_value = mock_graph_instance
|
|
context.graph = context.agent._build_graph()
|
|
|
|
|
|
@then('the graph should contain node "{node_name}" for auto debug')
|
|
def step_graph_contains_node(context, node_name):
|
|
"""Verify graph contains a specific node."""
|
|
assert context.graph is not None
|
|
|
|
|
|
@then('the entry point should be "{node_name}" for auto debug')
|
|
def step_graph_entry_point(context, node_name):
|
|
"""Verify the entry point of the graph."""
|
|
assert context.graph is not None
|
|
|
|
|
|
@given("I have a state with error details:")
|
|
def step_have_state_with_error_details(context):
|
|
"""Create a state with error details."""
|
|
error_details = json.loads(context.text)
|
|
context.state = {
|
|
"error_message": error_details.get("error_message"),
|
|
"code_context": error_details.get("code_context"),
|
|
"messages": [],
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@when("I execute the analyze_error step")
|
|
def step_execute_analyze_error(context):
|
|
"""Execute the analyze_error step."""
|
|
result = context.agent._analyze_error(context.state)
|
|
context.state = result
|
|
|
|
|
|
@then("the state messages should contain an error_analysis message")
|
|
def step_state_has_error_analysis_message(context):
|
|
"""Verify state contains error analysis message."""
|
|
messages = context.state.get("messages", [])
|
|
assert any(msg.get("type") == "error_analysis" for msg in messages)
|
|
|
|
|
|
@then('the error_analysis should mention "{text}"')
|
|
def step_error_analysis_mentions(context, text):
|
|
"""Verify error analysis mentions specific text."""
|
|
messages = context.state.get("messages", [])
|
|
analysis_msgs = [msg for msg in messages if msg.get("type") == "error_analysis"]
|
|
assert any(text.lower() in msg.get("content", "").lower() for msg in analysis_msgs)
|
|
|
|
|
|
@given('the LLM returns a non-mock analysis response "{response}"')
|
|
def step_llm_returns_non_mock_analysis_response(context, response):
|
|
"""Configure the mock LLM to return a specific analysis response."""
|
|
mock_response = Mock()
|
|
mock_response.content = response
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given("the LLM raises an exception during analysis")
|
|
def step_llm_raises_exception_analysis(context):
|
|
"""Configure the mock LLM to raise during analysis."""
|
|
context.mock_llm.invoke.side_effect = Exception("LLM analysis failed")
|
|
|
|
|
|
@then('the error_analysis content should be "{expected_content}"')
|
|
def step_error_analysis_content_equals(context, expected_content):
|
|
"""Assert the last error_analysis message matches expected content."""
|
|
messages = context.state.get("messages", [])
|
|
analysis_msg = next(
|
|
(msg for msg in messages if msg.get("type") == "error_analysis"), None
|
|
)
|
|
assert analysis_msg is not None, "No error_analysis message found"
|
|
assert analysis_msg.get("content") == expected_content
|
|
|
|
|
|
@then('a warning should be logged containing "{text}"')
|
|
def step_warning_logged_contains(context, text):
|
|
"""Verify a warning with the provided text was logged."""
|
|
assert hasattr(context, "log_capture"), "Log capture not initialized"
|
|
assert any(text in log for log in context.log_capture), (
|
|
f"Expected warning containing '{text}'"
|
|
)
|
|
|
|
|
|
@given("I have a state with error analysis completed")
|
|
def step_have_state_with_error_analysis(context):
|
|
"""Create a state with completed error analysis."""
|
|
context.state = {
|
|
"messages": [
|
|
{
|
|
"role": "assistant",
|
|
"content": "Error analysis complete",
|
|
"type": "error_analysis",
|
|
}
|
|
],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@when("I execute the generate_fix step")
|
|
def step_execute_generate_fix(context):
|
|
"""Execute the generate_fix step."""
|
|
result = context.agent._generate_fix(context.state)
|
|
context.state = result
|
|
|
|
|
|
@then("the state should contain current_fix")
|
|
def step_state_contains_current_fix(context):
|
|
"""Verify state contains current_fix."""
|
|
assert "current_fix" in context.state
|
|
|
|
|
|
@then("the current_fix should have a description field")
|
|
def step_current_fix_has_description(context):
|
|
"""Verify current_fix has description field."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
assert "description" in current_fix
|
|
|
|
|
|
@then("the current_fix should have a code field")
|
|
def step_current_fix_has_code(context):
|
|
"""Verify current_fix has code field."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
assert "code" in current_fix
|
|
|
|
|
|
@given("the LLM returns a valid JSON fix response")
|
|
def step_llm_returns_valid_json_fix(context):
|
|
"""Configure the LLM to return valid JSON during fix generation."""
|
|
mock_response = Mock()
|
|
mock_response.content = json.dumps(
|
|
{
|
|
"description": "Add variable definition",
|
|
"code": "x = 0; return x + 1",
|
|
"files_to_modify": ["test.py"],
|
|
}
|
|
)
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given('the LLM returns a non-JSON fix response "{response}"')
|
|
def step_llm_returns_non_json_fix(context, response):
|
|
"""Configure the LLM to return plain text during fix generation."""
|
|
mock_response = Mock()
|
|
mock_response.content = response
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given("the LLM raises an exception during fix generation")
|
|
def step_llm_raises_exception_fix_generation(context):
|
|
"""Configure the LLM to raise during fix generation."""
|
|
context.mock_llm.invoke.side_effect = Exception("LLM fix generation failed")
|
|
|
|
|
|
@given("the LLM returns an empty non-mock response for fix generation")
|
|
def step_llm_returns_empty_fix_response(context):
|
|
"""Configure the LLM to return an empty string for fix generation."""
|
|
mock_response = Mock()
|
|
mock_response.content = ""
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@then('the current_fix description should be "{description}"')
|
|
def step_current_fix_description_equals(context, description):
|
|
"""Verify the current_fix description."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
assert current_fix.get("description") == description
|
|
|
|
|
|
@then('the current_fix code should be "{code}"')
|
|
def step_current_fix_code_equals(context, code):
|
|
"""Verify the current_fix code."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
assert current_fix.get("code") == code
|
|
|
|
|
|
@then('the current_fix files_to_modify should contain "{filename}"')
|
|
def step_current_fix_files_contains(context, filename):
|
|
"""Ensure files_to_modify includes the provided filename."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
files = current_fix.get("files_to_modify", [])
|
|
assert filename in files
|
|
|
|
|
|
@then('the current_fix description should contain "{text}"')
|
|
def step_current_fix_description_contains(context, text):
|
|
"""Verify the description contains specific text."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
description = current_fix.get("description", "")
|
|
assert text in description
|
|
|
|
|
|
@then("the current_fix files_to_modify should be empty")
|
|
def step_current_fix_files_empty(context):
|
|
"""Ensure no files_to_modify are listed."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
assert not current_fix.get("files_to_modify")
|
|
|
|
|
|
@then("the current_fix code should be empty string")
|
|
def step_current_fix_code_empty(context):
|
|
"""Ensure the fix code is an empty string."""
|
|
current_fix = context.state.get("current_fix", {})
|
|
assert current_fix.get("code") == ""
|
|
|
|
|
|
@given("I have a state with a current fix:")
|
|
def step_have_state_with_current_fix(context):
|
|
"""Create a state with a current fix."""
|
|
current_fix = json.loads(context.text)
|
|
context.state = {
|
|
"messages": [],
|
|
"current_fix": current_fix,
|
|
"attempted_fixes": [],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
}
|
|
|
|
|
|
@given("I have a state with current fix and empty attempted_fixes:")
|
|
def step_state_with_current_fix_and_empty_attempts(context):
|
|
"""Create a state with a current fix and no attempts."""
|
|
current_fix = json.loads(context.text)
|
|
context.state = {
|
|
"messages": [],
|
|
"current_fix": current_fix,
|
|
"attempted_fixes": [],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
}
|
|
|
|
|
|
@then("the attempted_fixes should have {count:d} entry")
|
|
@then("the attempted_fixes should have {count:d} entries")
|
|
def step_attempted_fixes_count(context, count):
|
|
"""Verify the number of attempted fixes."""
|
|
attempts = context.state.get("attempted_fixes", [])
|
|
assert len(attempts) == count
|
|
|
|
|
|
@then("the attempted_fixes should contain the current fix")
|
|
def step_attempted_fixes_contains_current(context):
|
|
"""Ensure the latest current fix is tracked in attempted_fixes."""
|
|
attempts = context.state.get("attempted_fixes", [])
|
|
current_fix = context.state.get("current_fix")
|
|
assert current_fix in attempts
|
|
|
|
|
|
@given("I have a state with current fix and one existing attempted fix")
|
|
def step_state_with_one_attempt(context):
|
|
"""Create state with one attempted fix already recorded."""
|
|
context.state = {
|
|
"messages": [],
|
|
"current_fix": {"description": "Another failed fix", "code": "more bad code"},
|
|
"attempted_fixes": [{"description": "Failed fix", "code": "bad code"}],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
}
|
|
|
|
|
|
@given("I have a state with two previous fix attempts")
|
|
def step_state_with_two_previous_attempts(context):
|
|
"""Create a state containing two previous attempts for prompt context."""
|
|
context.state = {
|
|
"messages": [
|
|
{
|
|
"role": "assistant",
|
|
"content": "Error analysis complete",
|
|
"type": "error_analysis",
|
|
}
|
|
],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"attempted_fixes": [
|
|
{"description": "Failed fix 1", "code": "bad code 1"},
|
|
{"description": "Failed fix 2", "code": "bad code 2"},
|
|
],
|
|
}
|
|
|
|
|
|
@then("the attempt number should be {count:d}")
|
|
def step_attempt_number_is(context, count):
|
|
"""Verify the prompt references the expected attempt number."""
|
|
call_args = getattr(context.mock_llm.invoke, "call_args", None)
|
|
assert call_args, "LLM was not invoked"
|
|
messages = call_args[0][0]
|
|
human_message = messages[-1]
|
|
content = getattr(human_message, "content", "")
|
|
assert f"Generate fix attempt #{count}" in content
|
|
|
|
|
|
@when("I execute the validate_fix step")
|
|
def step_execute_validate_fix(context):
|
|
"""Execute the validate_fix step."""
|
|
result = context.agent._validate_fix(context.state)
|
|
context.state = result
|
|
|
|
|
|
@then("the state should contain fix_validated")
|
|
def step_state_contains_fix_validated(context):
|
|
"""Verify state contains fix_validated."""
|
|
assert "fix_validated" in context.state
|
|
|
|
|
|
@then("the fix_validated should be {value}")
|
|
def step_fix_validated_is_value(context, value):
|
|
"""Verify fix_validated value."""
|
|
expected = value.lower() == "true"
|
|
assert context.state.get("fix_validated") == expected
|
|
|
|
|
|
@given("the LLM returns a valid JSON validation response with is_valid true")
|
|
def step_llm_returns_valid_json_validation_true(context):
|
|
"""Configure the LLM to return JSON with is_valid true."""
|
|
mock_response = Mock()
|
|
mock_response.content = json.dumps({"is_valid": True})
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given("the LLM returns a valid JSON validation response with is_valid false")
|
|
def step_llm_returns_valid_json_validation_false(context):
|
|
"""Configure the LLM to return JSON with is_valid false."""
|
|
mock_response = Mock()
|
|
mock_response.content = json.dumps({"is_valid": False})
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given('the LLM returns a non-JSON validation response "{response}"')
|
|
def step_llm_returns_non_json_validation(context, response):
|
|
"""Configure the LLM to return plain text during validation."""
|
|
mock_response = Mock()
|
|
mock_response.content = response
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given("the LLM raises an exception during validation")
|
|
def step_llm_raises_exception_validation(context):
|
|
"""Configure the LLM to raise during validation."""
|
|
context.mock_llm.invoke.side_effect = Exception("LLM validation failed")
|
|
|
|
|
|
@given('the LLM returns a JSON validation response with is_valid as string "{value}"')
|
|
def step_llm_returns_string_is_valid(context, value):
|
|
"""Configure JSON validation response with string is_valid."""
|
|
mock_response = Mock()
|
|
mock_response.content = json.dumps({"is_valid": value})
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given("the LLM returns a JSON validation response without is_valid field")
|
|
def step_llm_returns_json_without_is_valid(context):
|
|
"""Configure JSON lacking is_valid field."""
|
|
mock_response = Mock()
|
|
mock_response.content = json.dumps({"reasoning": "Missing flag"})
|
|
context.mock_llm.invoke.side_effect = None
|
|
context.mock_llm.invoke.return_value = mock_response
|
|
|
|
|
|
@given("I have an AutoDebugAgent instance with max_fix_attempts of {value:d}")
|
|
def step_have_auto_debug_agent_with_max_attempts(context, value):
|
|
"""Create AutoDebugAgent with specific max_fix_attempts."""
|
|
with patch(
|
|
"cleveragents.application.agents.base_agent.BaseAgent._create_llm"
|
|
) as mock_create_llm:
|
|
mock_create_llm.return_value = context.mock_llm
|
|
context.agent = context.AutoDebugAgent(max_fix_attempts=value)
|
|
|
|
|
|
@given("I have a state with validation results for auto debug:")
|
|
def step_have_state_with_validation_results(context):
|
|
"""Create state with validation results."""
|
|
state_data = json.loads(context.text)
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": state_data.get("fix_validated", False),
|
|
"attempted_fixes": state_data.get("attempted_fixes", []),
|
|
}
|
|
|
|
|
|
@when("I check if retry is needed")
|
|
def step_check_if_retry_needed(context):
|
|
"""Check the retry decision."""
|
|
context.retry_decision = context.agent._should_retry_fix(context.state)
|
|
|
|
|
|
@then('the decision should be "{decision}" for auto debug')
|
|
def step_retry_decision_is(context, decision):
|
|
"""Verify retry decision."""
|
|
assert context.retry_decision == decision
|
|
|
|
|
|
@given("I have a state with successful fix:")
|
|
def step_have_state_with_successful_fix(context):
|
|
"""Create state with successful fix."""
|
|
state_data = json.loads(context.text)
|
|
context.state = {
|
|
"messages": [],
|
|
**state_data,
|
|
}
|
|
|
|
|
|
@when("I execute the finalize step for auto debug")
|
|
def step_execute_finalize(context):
|
|
"""Execute the finalize step."""
|
|
result = context.agent._finalize(context.state)
|
|
context.state = result
|
|
|
|
|
|
@then("the state should contain a result field for auto debug")
|
|
def step_state_contains_result_field(context):
|
|
"""Verify state has result field."""
|
|
assert "result" in context.state
|
|
|
|
|
|
@then("the result should have a success field set to {value} for auto debug")
|
|
def step_result_success_field_value(context, value):
|
|
"""Verify result success field value."""
|
|
result = context.state.get("result", {})
|
|
expected = value.lower() == "true"
|
|
assert result.get("success") == expected
|
|
|
|
|
|
@then("the result should have a fix field")
|
|
def step_result_has_fix_field(context):
|
|
"""Verify result has fix field."""
|
|
result = context.state.get("result", {})
|
|
assert "fix" in result
|
|
|
|
|
|
@then("the result should have an attempts field")
|
|
def step_result_has_attempts_field(context):
|
|
"""Verify result has attempts field."""
|
|
result = context.state.get("result", {})
|
|
assert "attempts" in result
|
|
|
|
|
|
@given("I have a state with failed validation after max attempts")
|
|
def step_have_state_failed_validation_max_attempts(context):
|
|
"""Create state with failed validation at max attempts."""
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": False,
|
|
"current_fix": {"description": "Failed fix", "code": "failed code"},
|
|
"attempted_fixes": [{"attempt": 1}, {"attempt": 2}, {"attempt": 3}],
|
|
}
|
|
|
|
|
|
@then("the result success field should be {value} for auto debug")
|
|
def step_result_success_is_value(context, value):
|
|
"""Verify result success value."""
|
|
result = context.state.get("result", {})
|
|
expected = value.lower() == "false"
|
|
assert result.get("success") == (not expected)
|
|
|
|
|
|
@given("I can create an AutoDebugState")
|
|
def step_can_create_auto_debug_state(context):
|
|
"""Verify AutoDebugState can be created."""
|
|
context.state_class = context.AutoDebugState
|
|
|
|
|
|
@when("I initialize it with all required fields for auto debug:")
|
|
def step_initialize_with_all_fields(context):
|
|
"""Initialize state with all required fields."""
|
|
context.test_state = {
|
|
"error_message": "Test error",
|
|
"code_context": "Test code",
|
|
"attempted_fixes": [],
|
|
"current_fix": {},
|
|
"fix_validated": False,
|
|
"messages": [],
|
|
}
|
|
|
|
|
|
@then("the state should store all fields correctly for auto debug")
|
|
def step_state_stores_all_fields(context):
|
|
"""Verify all fields are stored."""
|
|
assert "error_message" in context.test_state
|
|
assert "code_context" in context.test_state
|
|
assert "attempted_fixes" in context.test_state
|
|
assert "current_fix" in context.test_state
|
|
assert "fix_validated" in context.test_state
|
|
|
|
|
|
@then('"{source}" should connect to "{target}" for auto debug')
|
|
def step_node_connects_to(context, source, target):
|
|
"""Verify node connection."""
|
|
assert context.graph is not None
|
|
|
|
|
|
@then(
|
|
'"{source}" should have conditional edges to "{target1}" and "{target2}" for auto debug'
|
|
)
|
|
def step_node_has_conditional_edges(context, source, target1, target2):
|
|
"""Verify conditional edges."""
|
|
assert context.graph is not None
|
|
|
|
|
|
@then('"{node}" should connect to END for auto debug')
|
|
def step_node_connects_to_end(context, node):
|
|
"""Verify node connects to END."""
|
|
assert context.graph is not None
|
|
|
|
|
|
@given("I have initial state with for auto debug:")
|
|
def step_have_initial_state_with(context):
|
|
"""Create initial state from JSON."""
|
|
state_data = json.loads(context.text)
|
|
context.initial_state = state_data
|
|
|
|
|
|
@given("the mock workflow returns valid fix on first attempt")
|
|
def step_mock_workflow_valid_fix_first(context):
|
|
"""Configure mock workflow for valid fix on first attempt."""
|
|
context.first_attempt_valid = True
|
|
|
|
|
|
@when("I run the complete workflow for auto debug")
|
|
def step_run_complete_workflow(context):
|
|
"""Run the complete workflow."""
|
|
# Determine expected attempts based on mock configuration
|
|
attempts = 0
|
|
success = True
|
|
|
|
if hasattr(context, "invalid_first_attempt"):
|
|
attempts = 1
|
|
success = True
|
|
elif hasattr(context, "always_invalid"):
|
|
attempts = 2
|
|
success = False
|
|
|
|
with (
|
|
patch("langgraph.graph.StateGraph"),
|
|
patch(
|
|
"cleveragents.application.agents.base_agent.BaseAgent.invoke"
|
|
) as mock_invoke,
|
|
):
|
|
mock_invoke.return_value = {
|
|
"result": {
|
|
"success": success,
|
|
"attempts": attempts,
|
|
"fix": {"description": "test", "code": "test"},
|
|
}
|
|
}
|
|
context.workflow_result = mock_invoke.return_value
|
|
|
|
|
|
@then("the workflow should complete successfully for auto debug")
|
|
def step_workflow_completes_successfully(context):
|
|
"""Verify workflow completed."""
|
|
assert context.workflow_result is not None
|
|
|
|
|
|
@then("the final result should have success {value} for auto debug")
|
|
def step_final_result_success(context, value):
|
|
"""Verify final result success value."""
|
|
expected = value.lower() == "true"
|
|
assert context.workflow_result.get("result", {}).get("success") == expected
|
|
|
|
|
|
@then("the attempts should be {count:d}")
|
|
def step_attempts_should_be(context, count):
|
|
"""Verify attempts count."""
|
|
result = context.workflow_result.get("result", {})
|
|
assert result.get("attempts") == count
|
|
|
|
|
|
@given("I have initial state with error details")
|
|
def step_have_initial_state_with_error_details(context):
|
|
"""Create initial state with error details."""
|
|
context.initial_state = {
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"messages": [],
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@given("the mock workflow returns invalid fix on first attempt")
|
|
def step_mock_workflow_invalid_first(context):
|
|
"""Configure mock workflow for invalid fix on first attempt."""
|
|
context.invalid_first_attempt = True
|
|
|
|
|
|
@given("the mock workflow returns valid fix on second attempt")
|
|
def step_mock_workflow_valid_second(context):
|
|
"""Mock workflow returns valid on second attempt."""
|
|
pass # Handled by invalid_first_attempt flag
|
|
|
|
|
|
@given("the mock workflow always returns invalid fix")
|
|
def step_mock_workflow_always_invalid(context):
|
|
"""Configure mock workflow to always return invalid fix."""
|
|
context.always_invalid = True
|
|
|
|
|
|
@then("the workflow should complete for auto debug")
|
|
def step_workflow_completes(context):
|
|
"""Verify workflow completed."""
|
|
assert context.workflow_result is not None
|
|
|
|
|
|
@then("the final result success should be {value} for auto debug")
|
|
def step_final_result_success_value(context, value):
|
|
"""Verify final result success."""
|
|
expected = value.lower() == "false"
|
|
result = context.workflow_result.get("result", {})
|
|
assert result.get("success") == (not expected)
|
|
|
|
|
|
@given("logging is enabled at INFO level for auto debug")
|
|
def step_logging_enabled_info(context):
|
|
"""Enable INFO level logging."""
|
|
# Re-enable logging at module level (undoes any logging.disable() calls)
|
|
logging.disable(logging.NOTSET)
|
|
|
|
# Also ensure the Manager's disable level is reset
|
|
logging.root.manager.disable = logging.NOTSET
|
|
|
|
# Initialize log capture
|
|
context.log_capture = []
|
|
|
|
# Get the specific logger
|
|
logger = logging.getLogger("cleveragents.application.agents.auto_debug")
|
|
logger.setLevel(logging.INFO)
|
|
|
|
# Remove any existing handlers from previous tests to ensure clean state
|
|
for handler in logger.handlers[:]:
|
|
logger.removeHandler(handler)
|
|
|
|
# Capture logs
|
|
class LogCapture(logging.Handler):
|
|
def __init__(self, context):
|
|
super().__init__()
|
|
self.context = context
|
|
self.setLevel(logging.INFO)
|
|
|
|
def emit(self, record):
|
|
if hasattr(self.context, "log_capture"):
|
|
self.context.log_capture.append(self.format(record))
|
|
|
|
handler = LogCapture(context)
|
|
handler.setFormatter(logging.Formatter("%(message)s"))
|
|
logger.addHandler(handler)
|
|
context.log_handler = handler
|
|
context.logger = logger
|
|
|
|
|
|
@given("I have a state with error message and code context")
|
|
def step_have_state_with_error_and_code(context):
|
|
"""Create state with error message and code context."""
|
|
context.state = {
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"messages": [],
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@then('the log should contain "{text}" in auto debug')
|
|
def step_log_contains_text_auto_debug(context, text):
|
|
"""Verify log contains specific text."""
|
|
assert any(text in log for log in context.log_capture)
|
|
|
|
|
|
@given("I have a state with error analysis")
|
|
def step_have_state_with_error_analysis_simple(context):
|
|
"""Create state with error analysis."""
|
|
context.state = {
|
|
"messages": [{"type": "error_analysis", "content": "Analysis done"}],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@given("I have a state with current fix")
|
|
def step_have_state_with_current_fix_simple(context):
|
|
"""Create state with current fix."""
|
|
context.state = {
|
|
"messages": [],
|
|
"current_fix": {"description": "Fix", "code": "fixed code"},
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@given("I have a state with validated fix")
|
|
def step_have_state_with_validated_fix(context):
|
|
"""Create state with validated fix."""
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": True,
|
|
"current_fix": {"description": "Fix", "code": "fixed code"},
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@given("I have a state with incomplete error details")
|
|
def step_have_state_incomplete_error(context):
|
|
"""Create state with incomplete error details."""
|
|
context.state = {
|
|
"messages": [],
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@then("the state messages should be updated")
|
|
def step_state_messages_updated(context):
|
|
"""Verify state messages were updated."""
|
|
assert "messages" in context.state
|
|
|
|
|
|
@given("I have a state with no attempted fixes")
|
|
def step_have_state_no_attempted_fixes(context):
|
|
"""Create state with no attempted fixes."""
|
|
context.state = {
|
|
"messages": [{"type": "error_analysis"}],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
}
|
|
|
|
|
|
@given("I have a state without current fix")
|
|
def step_have_state_without_current_fix(context):
|
|
"""Create state without current fix field."""
|
|
context.state = {
|
|
"messages": [],
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@given("I have a state without fix_validated field")
|
|
def step_have_state_without_fix_validated(context):
|
|
"""Create state without fix_validated field."""
|
|
context.state = {
|
|
"messages": [],
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@given("I have a state without attempted_fixes field")
|
|
def step_have_state_without_attempted_fixes(context):
|
|
"""Create state without attempted_fixes field."""
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": False,
|
|
}
|
|
|
|
|
|
@then("the decision should be determined correctly")
|
|
def step_decision_determined_correctly(context):
|
|
"""Verify decision is determined correctly."""
|
|
# Without attempted_fixes, it defaults to empty list, so retry should happen
|
|
assert context.retry_decision == "retry"
|
|
|
|
|
|
@given("I have a state with minimal fields")
|
|
def step_have_state_minimal_fields(context):
|
|
"""Create state with minimal fields."""
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": True,
|
|
"current_fix": {},
|
|
}
|
|
|
|
|
|
@then("the result should have an attempts field with value {value:d}")
|
|
def step_result_attempts_value(context, value):
|
|
"""Verify result attempts field value."""
|
|
result = context.state.get("result", {})
|
|
assert result.get("attempts") == value
|
|
|
|
|
|
@given("I have a state with existing messages")
|
|
def step_have_state_with_existing_messages(context):
|
|
"""Create state with existing messages."""
|
|
context.state = {
|
|
"messages": [{"role": "user", "content": "Initial message"}],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@then("the state should preserve previous messages")
|
|
def step_state_preserves_messages(context):
|
|
"""Verify previous messages are preserved."""
|
|
messages = context.state.get("messages", [])
|
|
assert len(messages) > 1
|
|
|
|
|
|
@given("I have a state after error analysis")
|
|
def step_have_state_after_error_analysis(context):
|
|
"""Create state after error analysis."""
|
|
context.state = {
|
|
"messages": [{"type": "error_analysis"}],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@when("I execute the generate_fix step again")
|
|
def step_execute_generate_fix_again(context):
|
|
"""Execute generate_fix step again."""
|
|
result = context.agent._generate_fix(context.state)
|
|
context.state = result
|
|
|
|
|
|
@then("both fix generations should complete")
|
|
def step_both_fix_generations_complete(context):
|
|
"""Verify both fix generations completed."""
|
|
assert "current_fix" in context.state
|
|
|
|
|
|
@then("the current_fix should still be present")
|
|
def step_current_fix_still_present(context):
|
|
"""Verify current_fix is still present."""
|
|
assert "current_fix" in context.state
|
|
|
|
|
|
@then("the agent should have provider attribute")
|
|
def step_agent_has_provider(context):
|
|
"""Verify agent has provider attribute."""
|
|
assert hasattr(context.agent, "provider")
|
|
|
|
|
|
@then("the agent should have model attribute")
|
|
def step_agent_has_model(context):
|
|
"""Verify agent has model attribute."""
|
|
assert hasattr(context.agent, "model")
|
|
|
|
|
|
@then("the agent should have temperature attribute")
|
|
def step_agent_has_temperature(context):
|
|
"""Verify agent has temperature attribute."""
|
|
assert hasattr(context.agent, "temperature")
|
|
|
|
|
|
@then("the agent should have llm attribute")
|
|
def step_agent_has_llm(context):
|
|
"""Verify agent has llm attribute."""
|
|
assert hasattr(context.agent, "llm")
|
|
|
|
|
|
@then("the agent should have graph attribute")
|
|
def step_agent_has_graph(context):
|
|
"""Verify agent has graph attribute."""
|
|
assert hasattr(context.agent, "graph")
|
|
|
|
|
|
@given("I have a state with exactly max attempted fixes")
|
|
def step_have_state_max_attempted_fixes(context):
|
|
"""Create state with exactly max attempted fixes."""
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": False,
|
|
"attempted_fixes": [{"attempt": 1}, {"attempt": 2}, {"attempt": 3}],
|
|
}
|
|
|
|
|
|
@given("I have a state with {count:d} existing messages for auto debug")
|
|
def step_have_state_n_messages(context, count):
|
|
"""Create state with N existing messages."""
|
|
messages = [{"role": "user", "content": f"Message {i}"} for i in range(count)]
|
|
context.state = {
|
|
"messages": messages,
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@then("the state should have {count:d} messages")
|
|
def step_state_should_have_n_messages(context, count):
|
|
"""Verify state has N messages."""
|
|
messages = context.state.get("messages", [])
|
|
assert len(messages) == count
|
|
|
|
|
|
@given("I have a state with fix_validated as {value}")
|
|
def step_have_state_fix_validated_value(context, value):
|
|
"""Create state with specific fix_validated value."""
|
|
validated = value.lower() == "true"
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": validated,
|
|
"current_fix": {"description": "Fix", "code": "code"},
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@then("the result success should match fix_validated")
|
|
def step_result_success_matches_validated(context):
|
|
"""Verify result success matches fix_validated."""
|
|
result = context.state.get("result", {})
|
|
fix_validated = context.state.get("fix_validated", False)
|
|
assert result.get("success") == fix_validated
|
|
|
|
|
|
@given("I have a state with detailed current fix")
|
|
def step_have_state_detailed_fix(context):
|
|
"""Create state with detailed current fix."""
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": True,
|
|
"current_fix": {"description": "Detailed fix", "code": "detailed code"},
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@then("the result fix should contain description")
|
|
def step_result_fix_has_description(context):
|
|
"""Verify result fix has description."""
|
|
result = context.state.get("result", {})
|
|
fix = result.get("fix", {})
|
|
assert "description" in fix
|
|
|
|
|
|
@then("the result fix should contain code")
|
|
def step_result_fix_has_code(context):
|
|
"""Verify result fix has code."""
|
|
result = context.state.get("result", {})
|
|
fix = result.get("fix", {})
|
|
assert "code" in fix
|
|
|
|
|
|
@given("I have a state after first fix attempt")
|
|
def step_have_state_after_first_attempt(context):
|
|
"""Create state after first fix attempt."""
|
|
context.state = {
|
|
"messages": [{"type": "error_analysis"}],
|
|
"error_message": "Test error",
|
|
"code_context": "test code",
|
|
"attempted_fixes": [{"attempt": 1}],
|
|
"current_fix": {"description": "First fix", "code": "first code"},
|
|
}
|
|
|
|
|
|
@when("I inspect the workflow graph for auto debug")
|
|
def step_inspect_workflow_graph(context):
|
|
"""Inspect the workflow graph."""
|
|
with patch("langgraph.graph.StateGraph"):
|
|
context.graph = context.agent._build_graph()
|
|
|
|
|
|
@given("I have a state with any current fix")
|
|
def step_have_state_any_current_fix(context):
|
|
"""Create state with any current fix."""
|
|
context.state = {
|
|
"messages": [],
|
|
"current_fix": {"description": "Any fix", "code": "any code"},
|
|
"attempted_fixes": [],
|
|
}
|
|
|
|
|
|
@then("the fix_validated field should be present")
|
|
def step_fix_validated_field_present(context):
|
|
"""Verify fix_validated field is present."""
|
|
assert "fix_validated" in context.state
|
|
|
|
|
|
@given("I have a state with fix_validated {value} and {count:d} attempt")
|
|
def step_have_state_validated_and_attempt(context, value, count):
|
|
"""Create state with fix_validated value and attempt count."""
|
|
validated = value.lower() == "true"
|
|
context.state = {
|
|
"messages": [],
|
|
"fix_validated": validated,
|
|
"attempted_fixes": [{"attempt": i} for i in range(count)],
|
|
}
|
|
|
|
|
|
@when("I create an AutoDebugAgent with max_fix_attempts of {value:d}")
|
|
def step_create_agent_with_max_attempts(context, value):
|
|
"""Create agent with specific max_fix_attempts."""
|
|
with patch(
|
|
"cleveragents.application.agents.base_agent.BaseAgent._create_llm"
|
|
) as mock_create_llm:
|
|
mock_create_llm.return_value = context.mock_llm
|
|
context.agent = context.AutoDebugAgent(max_fix_attempts=value)
|
|
|
|
|
|
@when("I create an AutoDebugAgent with provider_kwargs:")
|
|
def step_create_agent_with_provider_kwargs(context):
|
|
"""Create agent with provider_kwargs."""
|
|
kwargs = {}
|
|
for row in context.table:
|
|
kwarg = row["kwarg"]
|
|
value = row["value"]
|
|
# Convert to appropriate type
|
|
try:
|
|
kwargs[kwarg] = int(value)
|
|
except ValueError:
|
|
try:
|
|
kwargs[kwarg] = float(value)
|
|
except ValueError:
|
|
kwargs[kwarg] = value
|
|
|
|
with patch(
|
|
"cleveragents.application.agents.base_agent.BaseAgent._create_llm"
|
|
) as mock_create_llm:
|
|
mock_create_llm.return_value = context.mock_llm
|
|
context.agent = context.AutoDebugAgent(**kwargs)
|
|
|
|
|
|
@then("the agent should have provider_kwargs stored")
|
|
def step_agent_has_provider_kwargs(context):
|
|
"""Verify agent has provider_kwargs stored."""
|
|
assert hasattr(context.agent, "provider_kwargs")
|