"""Step definitions for error recovery pattern tests. Uses mock objects for PlanLifecycleService to test error recording, classification, recovery hints, and retry policy without needing a DB. """ from __future__ import annotations from unittest.mock import MagicMock from behave import given, then, when from behave.runner import Context from cleveragents.application.services.error_recovery_service import ( ErrorRecoveryService, ) from cleveragents.domain.models.core.error_recovery import ( ErrorCategory, ErrorRecord, classify_error, get_recovery_hints, ) from cleveragents.domain.models.core.plan import ( PlanPhase, PlanTimestamps, ProcessingState, ) __all__: list[str] = [] # --------------------------------------------------------------------------- # Mock helpers # --------------------------------------------------------------------------- _PLAN_ID = "01ERRTEST000000000000001" def _make_mock_plan( *, plan_id: str = _PLAN_ID, phase: PlanPhase = PlanPhase.EXECUTE, state: ProcessingState = ProcessingState.ERRORED, is_terminal: bool = False, ) -> MagicMock: """Create a mock Plan object in errored state.""" plan = MagicMock() plan.identity.plan_id = plan_id plan.phase = phase plan.processing_state = state plan.is_terminal = is_terminal plan.error_message = None plan.error_details = None plan.timestamps = PlanTimestamps() return plan def _make_service( plan: MagicMock, auto_retry_threshold: float = 0.0, max_retries: int = 3, ) -> ErrorRecoveryService: """Create an ErrorRecoveryService with mocked lifecycle.""" lifecycle = MagicMock() lifecycle.get_plan.return_value = plan lifecycle.commit_plan = MagicMock() return ErrorRecoveryService( lifecycle_service=lifecycle, auto_retry_threshold=auto_retry_threshold, max_retries=max_retries, ) # --------------------------------------------------------------------------- # Background # --------------------------------------------------------------------------- @given("an error recovery test environment") def step_given_err_env(context: Context) -> None: context.er_plan_id = _PLAN_ID context.er_plan = None context.er_service = None context.er_classified_category = None # ErrorCategory | None context.er_hints = None # list[RecoveryHint] | None context.er_recorded_error = None # ErrorRecord | None context.er_formatted_output = None # str | None context.er_error_record = None # ErrorRecord | None context.er_auto_retry_threshold = 0.0 context.er_max_retries = 3 # --------------------------------------------------------------------------- # Given steps # --------------------------------------------------------------------------- @given("a plan in errored state for error recovery") def step_given_errored_plan(context: Context) -> None: plan = _make_mock_plan() context.er_plan = plan context.er_service = _make_service( plan, auto_retry_threshold=context.er_auto_retry_threshold, max_retries=context.er_max_retries, ) @given("the auto retry threshold is {threshold}") def step_given_auto_retry_threshold(context: Context, threshold: str) -> None: context.er_auto_retry_threshold = float(threshold) # Recreate service if plan already exists if context.er_plan is not None: context.er_service = _make_service( context.er_plan, auto_retry_threshold=float(threshold), max_retries=context.er_max_retries, ) @given("the max retries is {n}") def step_given_max_retries(context: Context, n: str) -> None: context.er_max_retries = int(n) # Recreate service if plan already exists if context.er_plan is not None: context.er_service = _make_service( context.er_plan, auto_retry_threshold=context.er_auto_retry_threshold, max_retries=int(n), ) @given( 'an error record with category "{cat}" and retry count {rc} and max retries {mr}' ) def step_given_error_record(context: Context, cat: str, rc: str, mr: str) -> None: context.er_error_record = ErrorRecord( error_id="ERR-TEST-001", phase="execute", category=ErrorCategory(cat), message="test error", retry_count=int(rc), max_retries=int(mr), ) # --------------------------------------------------------------------------- # When steps # --------------------------------------------------------------------------- @when('I classify the error message "{message}"') def step_when_classify_message(context: Context, message: str) -> None: context.er_classified_category = classify_error(message) @when('I classify error text "{message}" using exception type "{exc_type}"') def step_when_classify_with_exc(context: Context, message: str, exc_type: str) -> None: context.er_classified_category = classify_error(message, exception_type=exc_type) @when('I get recovery hints for category "{cat}" and plan "{plan_id}"') def step_when_get_hints(context: Context, cat: str, plan_id: str) -> None: context.er_hints = get_recovery_hints(ErrorCategory(cat), plan_id) @when('I record an error with phase "{phase}" and message "{message}"') def step_when_record_error(context: Context, phase: str, message: str) -> None: context.er_recorded_error = context.er_service.record_error( plan_id=context.er_plan_id, phase=phase, message=message, ) @when( 'I record a categorized error with phase "{phase}" ' 'and message "{message}" and category "{cat}"' ) def step_when_record_error_with_category( context: Context, phase: str, message: str, cat: str ) -> None: context.er_recorded_error = context.er_service.record_error( plan_id=context.er_plan_id, phase=phase, message=message, category=ErrorCategory(cat), ) @when('I format the error output as "{fmt}"') def step_when_format_output(context: Context, fmt: str) -> None: context.er_formatted_output = context.er_service.format_error_output( plan_id=context.er_plan_id, fmt=fmt, ) @when('I format error output as "{fmt}" for separate plan "{plan_id}"') def step_when_format_output_separate_plan( context: Context, fmt: str, plan_id: str ) -> None: # Create a minimal service for the empty plan if context.er_service is None: plan = _make_mock_plan(plan_id=plan_id) context.er_service = _make_service(plan) context.er_formatted_output = context.er_service.format_error_output( plan_id=plan_id, fmt=fmt, ) # --------------------------------------------------------------------------- # Then steps # --------------------------------------------------------------------------- @then('the classified error category should be "{expected}"') def step_then_category(context: Context, expected: str) -> None: assert context.er_classified_category is not None, "No classification result" assert context.er_classified_category == ErrorCategory(expected), ( f"Expected category '{expected}', got '{context.er_classified_category.value}'" ) @then('the first recovery hint action should be "{action}"') def step_then_first_hint_action(context: Context, action: str) -> None: assert context.er_hints is not None and len(context.er_hints) > 0, ( "No recovery hints" ) first = sorted(context.er_hints, key=lambda h: h.priority)[0] assert first.action.value == action, ( f"Expected action '{action}', got '{first.action.value}'" ) @then('the first recovery hint should have CLI command containing "{text}"') def step_then_first_hint_cli(context: Context, text: str) -> None: assert context.er_hints is not None and len(context.er_hints) > 0 first = sorted(context.er_hints, key=lambda h: h.priority)[0] assert first.cli_command is not None, "First hint has no CLI command" assert text in first.cli_command, ( f"Expected CLI command to contain '{text}', got: {first.cli_command}" ) @then('the recorded error should have category "{cat}"') def step_then_recorded_category(context: Context, cat: str) -> None: assert context.er_recorded_error is not None assert context.er_recorded_error.category == ErrorCategory(cat), ( f"Expected '{cat}', got '{context.er_recorded_error.category.value}'" ) @then('the recorded error should have phase "{phase}"') def step_then_recorded_phase(context: Context, phase: str) -> None: assert context.er_recorded_error is not None assert context.er_recorded_error.phase == phase @then("the recorded error should have recovery hints") def step_then_recorded_has_hints(context: Context) -> None: assert context.er_recorded_error is not None assert len(context.er_recorded_error.recovery_hints) > 0, ( "Expected recovery hints, got none" ) @then("the error history should have {n} record") def step_then_history_count_singular(context: Context, n: str) -> None: history = context.er_service.get_error_history(context.er_plan_id) assert history.total_errors == int(n), ( f"Expected {n} records, got {history.total_errors}" ) @then("the error history should have {n} records") def step_then_history_count(context: Context, n: str) -> None: history = context.er_service.get_error_history(context.er_plan_id) assert history.total_errors == int(n), ( f"Expected {n} records, got {history.total_errors}" ) @then("the latest error retry count should be {n}") def step_then_latest_retry_count(context: Context, n: str) -> None: latest = context.er_service.get_latest_error(context.er_plan_id) assert latest is not None assert latest.retry_count == int(n), ( f"Expected retry_count={n}, got {latest.retry_count}" ) @then("the service should recommend retry") def step_then_should_retry(context: Context) -> None: assert context.er_service.should_retry(context.er_plan_id), ( "Expected should_retry=True" ) @then("the service should recommend escalation") def step_then_should_escalate(context: Context) -> None: assert context.er_service.should_escalate(context.er_plan_id), ( "Expected should_escalate=True" ) @then("the latest error should have retries exhausted") def step_then_retries_exhausted(context: Context) -> None: latest = context.er_service.get_latest_error(context.er_plan_id) assert latest is not None assert latest.retries_exhausted, ( f"Expected retries exhausted, " f"got retry_count={latest.retry_count}, max={latest.max_retries}" ) @then('the error history category breakdown should include "{cat}"') def step_then_category_breakdown(context: Context, cat: str) -> None: history = context.er_service.get_error_history(context.er_plan_id) breakdown = history.errors_by_category() assert cat in breakdown, f"Expected category '{cat}' in breakdown, got: {breakdown}" @then('the error recovery output should contain "{text}"') def step_then_output_contains(context: Context, text: str) -> None: assert context.er_formatted_output is not None assert text in context.er_formatted_output, ( f"Expected output to contain '{text}', got: {context.er_formatted_output[:200]}" ) @then("the error record should be retriable") def step_then_record_retriable(context: Context) -> None: assert context.er_error_record is not None assert context.er_error_record.is_retriable, "Expected is_retriable=True" @then("the error record should not be retriable") def step_then_record_not_retriable(context: Context) -> None: assert context.er_error_record is not None assert not context.er_error_record.is_retriable, "Expected is_retriable=False" @then("the error record should have retries exhausted") def step_then_record_exhausted(context: Context) -> None: assert context.er_error_record is not None assert context.er_error_record.retries_exhausted, "Expected retries_exhausted=True" @then('the error record CLI dict should have key "{key}"') def step_then_record_cli_dict_key(context: Context, key: str) -> None: assert context.er_error_record is not None cli_dict = context.er_error_record.to_cli_dict() assert key in cli_dict, ( f"Expected key '{key}' in CLI dict, got keys: {list(cli_dict.keys())}" ) # --------------------------------------------------------------------------- # Additional coverage steps # --------------------------------------------------------------------------- @when('I record an error with exception for phase "{phase}" and message "{message}"') def step_when_record_with_exception(context: Context, phase: str, message: str) -> None: try: raise ValueError("simulated error for stack trace") except ValueError as exc: context.er_recorded_error = context.er_service.record_error( plan_id=context.er_plan_id, phase=phase, message=message, exception=exc, ) @when( 'I record an actor-error in phase "{phase}" with message "{message}" ' 'actor "{actor}" tool "{tool}"' ) def step_when_record_with_actor_tool( context: Context, phase: str, message: str, actor: str, tool: str ) -> None: context.er_recorded_error = context.er_service.record_error( plan_id=context.er_plan_id, phase=phase, message=message, actor=actor, tool_call=tool, ) @given('an error record with actor "{actor}" and tool "{tool}"') def step_given_record_with_actor_tool(context: Context, actor: str, tool: str) -> None: context.er_error_record = ErrorRecord( error_id="ERR-ACTORTOOL-001", phase="execute", category=ErrorCategory.TRANSIENT, message="test error with actor and tool", actor=actor, tool_call=tool, retry_count=0, max_retries=3, ) @then("the recorded error should have a stack summary") def step_then_has_stack_summary(context: Context) -> None: assert context.er_recorded_error is not None assert context.er_recorded_error.stack_summary is not None, ( "Expected stack_summary to be set" ) assert len(context.er_recorded_error.stack_summary) > 0 @then("the service should not recommend retry for a clean plan") def step_then_no_retry_clean(context: Context) -> None: assert not context.er_service.should_retry(context.er_plan_id) @then("the service should not recommend escalation for a clean plan") def step_then_no_escalate_clean(context: Context) -> None: assert not context.er_service.should_escalate(context.er_plan_id) @then('the CLI recovery hints should contain "{text}"') def step_then_cli_hints_contain(context: Context, text: str) -> None: hints = context.er_service.format_recovery_hints_for_cli(context.er_plan_id) assert text in hints, f"Expected CLI hints to contain '{text}', got: {hints[:200]}" @then("the error history should not have retriable errors") def step_then_no_retriable(context: Context) -> None: history = context.er_service.get_error_history(context.er_plan_id) assert not history.has_retriable, "Expected no retriable errors" @then("the CLI recovery hints for a clean plan should be empty") def step_then_clean_plan_no_hints(context: Context) -> None: hints = context.er_service.format_recovery_hints_for_cli(context.er_plan_id) assert hints == "", f"Expected empty hints, got: {hints}" @given('an error record with stack summary "{summary}"') def step_given_record_with_stack(context: Context, summary: str) -> None: context.er_error_record = ErrorRecord( error_id="ERR-STACK-001", phase="execute", category=ErrorCategory.TRANSIENT, message="test error with stack", stack_summary=summary, retry_count=0, max_retries=3, ) @given("a policy with threshold {threshold} and max retries {mr}") def step_given_policy(context: Context, threshold: str, mr: str) -> None: from cleveragents.domain.models.core.error_recovery import ( ErrorRecoveryPolicy, ) context.er_policy = ErrorRecoveryPolicy( auto_retry_threshold=float(threshold), max_retries=int(mr), ) @then("the policy should not recommend retry") def step_then_policy_no_retry(context: Context) -> None: assert context.er_error_record is not None assert not context.er_policy.should_retry(context.er_error_record), ( "Expected should_retry=False" ) # --------------------------------------------------------------------------- # T1: Persistence failure path # --------------------------------------------------------------------------- @given("the lifecycle service update_error_details raises an error") def step_given_lifecycle_update_raises(context: Context) -> None: context.er_service._lifecycle.update_error_details.side_effect = RuntimeError( "DB connection lost" ) @then("the error should still be recorded in memory") def step_then_error_in_memory(context: Context) -> None: history = context.er_service.get_error_history(context.er_plan_id) assert history is not None assert history.total_errors >= 1, "Expected at least 1 error in memory" # --------------------------------------------------------------------------- # T2: Branch coverage for policy and format_recovery_output # --------------------------------------------------------------------------- @then("the policy should recommend escalation") def step_then_policy_escalation(context: Context) -> None: assert context.er_error_record is not None assert context.er_policy.should_escalate(context.er_error_record), ( "Expected should_escalate=True" ) @given( 'a hinted error record with category "{cat}" and retry count {rc}' " and max retries {mr}" ) def step_given_error_record_with_hints( context: Context, cat: str, rc: str, mr: str, ) -> None: from cleveragents.domain.models.core.error_recovery import ( RecoveryAction, RecoveryHint, ) hints = [ RecoveryHint( action=RecoveryAction.RETRY, message="Retry the operation", cli_command="agents plan prompt TESTPLAN", priority=0, ), ] context.er_error_record = ErrorRecord( error_id="ERR-HINT-001", phase="execute", category=ErrorCategory(cat), message="test error with hints", retry_count=int(rc), max_retries=int(mr), recovery_hints=hints, ) @then("the formatted recovery output should contain CLI command text") def step_then_format_contains_cli_cmd(context: Context) -> None: output = context.er_policy.format_recovery_output(context.er_error_record) assert "$ agents plan prompt" in output, ( f"Expected CLI command in output, got: {output}" ) @then('the formatted recovery output should contain "{text}"') def step_then_format_contains_text(context: Context, text: str) -> None: output = context.er_policy.format_recovery_output(context.er_error_record) assert text.lower() in output.lower(), f"Expected '{text}' in output, got: {output}" # --------------------------------------------------------------------------- # T4: Integration test with executor retry loop # --------------------------------------------------------------------------- @given("a plan executor with error recovery service") def step_given_executor_with_recovery(context: Context) -> None: from cleveragents.application.services.plan_executor import PlanExecutor # Build mock lifecycle service lifecycle = MagicMock() plan = _make_mock_plan(phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED) # Ensure real enum values (not MagicMock proxies) plan.phase = PlanPhase.EXECUTE plan.state = ProcessingState.QUEUED plan.decision_root_id = "01DECROOT00000000000001" plan.definition_of_done = "- Step one\n- Step two" lifecycle.get_plan.return_value = plan lifecycle.update_error_details = MagicMock() lifecycle.commit_plan = MagicMock() # Build error recovery service wrapping the same lifecycle er_service = ErrorRecoveryService( lifecycle_service=lifecycle, auto_retry_threshold=0.0, max_retries=3, ) executor = PlanExecutor( lifecycle_service=lifecycle, error_recovery_service=er_service, ) context.er_executor = executor context.er_executor_lifecycle = lifecycle context.er_executor_recovery = er_service context.er_executor_plan_id = plan.identity.plan_id @given("the execute actor fails twice then succeeds") def step_given_actor_fails_twice(context: Context) -> None: call_count = {"n": 0} def _side_effect(**kwargs): call_count["n"] += 1 if call_count["n"] <= 2: raise RuntimeError("Connection timed out — transient failure") # Return a mock result on 3rd call result = MagicMock() result.changeset_id = "CS-001" result.sandbox_refs = [] result.tool_calls_count = 1 return result context.er_executor._execute_actor.execute = MagicMock(side_effect=_side_effect) @when("I run the executor for the plan") def step_when_run_executor(context: Context) -> None: result = context.er_executor.run_execute(context.er_executor_plan_id) context.er_executor_result = result @then("the execute should complete after retries") def step_then_execute_completed(context: Context) -> None: assert context.er_executor_result is not None context.er_executor_lifecycle.complete_execute.assert_called_once() @then("the error recovery service should have recorded {n} errors") def step_then_recovery_recorded_n(context: Context, n: str) -> None: history = context.er_executor_recovery.get_error_history( context.er_executor_plan_id, ) assert history.total_errors == int(n), ( f"Expected {n} errors, got {history.total_errors}" ) @given("a plan executor with error recovery service max retries {n}") def step_given_executor_max_retries(context: Context, n: str) -> None: from cleveragents.application.services.plan_executor import PlanExecutor lifecycle = MagicMock() plan = _make_mock_plan(phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED) plan.phase = PlanPhase.EXECUTE plan.state = ProcessingState.QUEUED plan.decision_root_id = "01DECROOT00000000000001" plan.definition_of_done = "- Step one" lifecycle.get_plan.return_value = plan lifecycle.update_error_details = MagicMock() lifecycle.commit_plan = MagicMock() er_service = ErrorRecoveryService( lifecycle_service=lifecycle, auto_retry_threshold=0.0, max_retries=int(n), ) executor = PlanExecutor( lifecycle_service=lifecycle, error_recovery_service=er_service, ) context.er_executor = executor context.er_executor_lifecycle = lifecycle context.er_executor_recovery = er_service context.er_executor_plan_id = plan.identity.plan_id @given('the execute actor always fails with "{msg}"') def step_given_actor_always_fails(context: Context, msg: str) -> None: context.er_executor._execute_actor.execute = MagicMock( side_effect=RuntimeError(f"Connection timed out — {msg}"), ) @when("I run the executor expecting failure") def step_when_run_executor_failure(context: Context) -> None: try: context.er_executor.run_execute(context.er_executor_plan_id) context.er_executor_raised = False except RuntimeError: context.er_executor_raised = True @then("the lifecycle should record the plan as failed") def step_then_plan_failed(context: Context) -> None: assert context.er_executor_raised is True context.er_executor_lifecycle.fail_execute.assert_called_once() # --------------------------------------------------------------------------- # B5: Resource conflict classification # --------------------------------------------------------------------------- @then('the classified error category should not be "{category}"') def step_then_category_not_eq(context: Context, category: str) -> None: assert context.er_classified_category is not None assert context.er_classified_category.value != category, ( f"Expected category != '{category}', " f"got '{context.er_classified_category.value}'" ) # --------------------------------------------------------------------------- # S1: Safe format-string substitution # --------------------------------------------------------------------------- @when('I get recovery hints for category "{cat}" with plan_id "{pid}"') def step_when_hints_with_braces(context: Context, cat: str, pid: str) -> None: context.er_hints = get_recovery_hints(ErrorCategory(cat), pid) @then('the hints should contain "{text}" in CLI commands') def step_then_hints_contain_text(context: Context, text: str) -> None: cli_cmds = [h.cli_command for h in context.er_hints if h.cli_command] assert any(text in cmd for cmd in cli_cmds), ( f"Expected '{text}' in CLI commands, got: {cli_cmds}" )