cleveragents-core/robot/retry_patterns.robot

*** Settings ***
Documentation     Integration tests for retry patterns implementation
...               Tests the retry patterns module with various failure scenarios
...               and verifies proper retry behavior and recovery mechanisms.

Library           OperatingSystem
Library           Process
Library           String
Library           Collections
Library           BuiltIn
Resource          ${CURDIR}/common.resource

Test Setup        Setup Retry Test Environment
Test Teardown     Cleanup Retry Test Environment

*** Variables ***
${WORKSPACE_ROOT}     ${CURDIR}/..
${RETRY_TEST_FILE}    ${EMPTY}
${RETRY_OUTPUT}       ${EMPTY}
${MAX_RETRIES}        3
${TIMEOUT}            30s

*** Test Cases ***
Test Exponential Backoff Retry Pattern
    [Documentation]    Verify exponential backoff retry works correctly
    Create Retry Test Script    exponential_backoff
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Attempt 1 failed
    Should Contain    ${output}    Attempt 2 failed
    Should Contain    ${output}    Attempt 3 succeeded
    Verify Exponential Delays    ${output}

Test Async Exponential Backoff Pattern
    [Documentation]    Verify async exponential backoff retry works correctly
    Create Async Retry Test Script    async_exponential_backoff
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Async attempt 1 failed
    Should Contain    ${output}    Async attempt 2 failed
    Should Contain    ${output}    Async attempt 3 succeeded

Test Network Retry Pattern
    [Documentation]    Test network-specific retry with NetworkError
    Create Network Error Test Script
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    NetworkError retry attempt
    Should Contain    ${output}    Network operation succeeded after retries
    Should Match Regexp    ${output}    Retry count: [3-5]

Test Provider Retry Pattern With Rate Limiting
    [Documentation]    Test provider retry with RateLimitError and jitter
    Create Rate Limit Test Script
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    RateLimitError encountered
    Should Contain    ${output}    Provider operation succeeded
    Verify Jitter In Delays    ${output}

Test Circuit Breaker Opens After Threshold
    [Documentation]    Verify circuit breaker opens after failure threshold
    Create Circuit Breaker Test Script    open
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Circuit breaker opened after 3 failures
    Should Contain    ${output}    CircuitBreakerOpen exception raised

Test Circuit Breaker Recovery
    [Documentation]    Verify circuit breaker recovers to half-open then closed
    Create Circuit Breaker Test Script    recovery
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Circuit breaker state: open
    Should Contain    ${output}    Circuit breaker state: half-open
    Should Contain    ${output}    Circuit breaker state: closed

Test Retry Context Manager
    [Documentation]    Test RetryContext for tracking retry attempts
    Create Retry Context Test Script
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Retry context: test_operation
    Should Contain    ${output}    Total attempts: 3
    Should Contain    ${output}    Operation succeeded

Test Auto Debug Retry Pattern
    [Documentation]    Test auto-debug retry with debug callback
    Create Auto Debug Test Script
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Auto-debug attempt 1
    Should Contain    ${output}    Debug callback invoked
    Should Contain    ${output}    Issue fixed by debug callback
    Should Contain    ${output}    Operation succeeded after debugging

Test Retry With Timeout
    [Documentation]    Test retry with overall timeout limit
    Create Timeout Retry Test Script
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    # This should fail due to timeout
    Should Not Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Timeout exceeded

Test Category-Specific Retry Decorators
    [Documentation]    Test all category-specific retry patterns
    Create Category Test Script
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Should Contain    ${output}    Network retry: max_attempts=5
    Should Contain    ${output}    Provider retry: max_attempts=3
    Should Contain    ${output}    Database retry: max_attempts=3
    Should Contain    ${output}    File operation retry: max_attempts=3

Test Concurrent Retries With Jitter
    [Documentation]    Test multiple concurrent operations with jitter
    Create Concurrent Retry Test Script
    ${result} =    Run Process    ${PYTHON}    ${RETRY_TEST_FILE}
    ...    stdout=${RETRY_OUTPUT}    stderr=STDOUT    timeout=${TIMEOUT}    on_timeout=kill
    Should Be Equal As Integers    ${result.rc}    0
    ${output} =    Get File    ${RETRY_OUTPUT}
    Verify No Thundering Herd    ${output}

*** Keywords ***
Setup Retry Test Environment
    [Documentation]    Create temporary directory for test files
    ${temp_dir}=    Evaluate    tempfile.mkdtemp()    modules=tempfile
    Set Test Variable    ${TEMP_DIR}    ${temp_dir}
    Set Test Variable    ${RETRY_TEST_FILE}    ${temp_dir}/retry_test.py
    Set Test Variable    ${RETRY_OUTPUT}    ${temp_dir}/retry_output.txt

Cleanup Retry Test Environment
    [Documentation]    Clean up temporary test files
    Run Keyword And Ignore Error    Remove File    ${RETRY_TEST_FILE}
    Run Keyword And Ignore Error    Remove File    ${RETRY_OUTPUT}
    Run Keyword And Ignore Error    Remove Directory    ${TEMP_DIR}    recursive=True

Create Retry Test Script
    [Arguments]    ${pattern_type}
    [Documentation]    Create a Python script to test retry patterns
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    import os
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import time
    ...    import logging
    ...    import structlog
    ...    from cleveragents.core.retry_patterns import retry_with_exponential_backoff
    ...
    ...    # Suppress all debug logging
    ...    logging.getLogger().setLevel(logging.ERROR)
    ...    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.ERROR))
    ...    os.environ['PYTHONUNBUFFERED'] = '1'
    ...
    ...    attempt_count = 0
    ...    start_times = []
    ...
    ...    @retry_with_exponential_backoff(max_attempts=3, min_wait=0.1, max_wait=5.0, multiplier=1.5)
    ...    def test_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}global attempt_count
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}attempt_count += 1
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}start_times.append(time.time())
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if attempt_count < 3:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print(f"Attempt {attempt_count} failed", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise Exception(f"Failure {attempt_count}")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Attempt {attempt_count} succeeded", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return "success"
    ...
    ...    try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}result = test_function()
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Final result: {result}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}# Verify exponential delays
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if len(start_times) > 1:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}for i in range(1, len(start_times)):
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}delay = start_times[i] - start_times[i-1]
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print(f"Delay {i}: {delay:.2f}s", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(0)
    ...    except Exception as e:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Error: {e}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(1)
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Async Retry Test Script
    [Arguments]    ${pattern_type}
    [Documentation]    Create a Python script to test async retry patterns
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    import os
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import asyncio
    ...    import logging
    ...    import structlog
    ...    from cleveragents.core.retry_patterns import async_retry_with_exponential_backoff
    ...
    ...    # Suppress all debug logging
    ...    logging.getLogger().setLevel(logging.ERROR)
    ...    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.ERROR))
    ...    os.environ['PYTHONUNBUFFERED'] = '1'
    ...
    ...    attempt_count = 0
    ...
    ...    @async_retry_with_exponential_backoff(max_attempts=3, min_wait=0.1, max_wait=5.0, multiplier=1.5)
    ...    async def test_async_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}global attempt_count
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}attempt_count += 1
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if attempt_count < 3:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print(f"Async attempt {attempt_count} failed", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise Exception(f"Async failure {attempt_count}")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Async attempt {attempt_count} succeeded", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return "async success"
    ...
    ...    async def main():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}result = await test_async_function()
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Final result: {result}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return result
    ...
    ...    try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}result = asyncio.run(main())
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(0)
    ...    except Exception as e:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Error: {e}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(1)
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Network Error Test Script
    [Documentation]    Create script testing network retry pattern
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    import os
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import logging
    ...    import structlog
    ...    from cleveragents.core.retry_patterns import retry_network_operation
    ...    from cleveragents.core.exceptions import NetworkError
    ...
    ...    # Suppress all debug logging
    ...    logging.getLogger().setLevel(logging.ERROR)
    ...    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.ERROR))
    ...    os.environ['PYTHONUNBUFFERED'] = '1'
    ...
    ...    retry_count = 0
    ...
    ...    @retry_network_operation
    ...    def network_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}global retry_count
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}retry_count += 1
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"NetworkError retry attempt {retry_count}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if retry_count <= 2:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise NetworkError(f"Network failure {retry_count}")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return "Network operation succeeded after retries"
    ...
    ...    try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}result = network_function()
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(result, flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Retry count: {retry_count}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(0)
    ...    except Exception as e:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Error: {e}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(1)
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Rate Limit Test Script
    [Documentation]    Create script testing provider retry with rate limiting
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    import os
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import time
    ...    import logging
    ...    import structlog
    ...    from cleveragents.core.retry_patterns import retry_provider_operation
    ...    from cleveragents.core.exceptions import RateLimitError
    ...
    ...    # Suppress all debug logging
    ...    logging.getLogger().setLevel(logging.ERROR)
    ...    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.ERROR))
    ...    os.environ['PYTHONUNBUFFERED'] = '1'
    ...
    ...    retry_count = 0
    ...    retry_times = []
    ...
    ...    @retry_provider_operation
    ...    def provider_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}global retry_count
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}retry_count += 1
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}retry_times.append(time.time())
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"RateLimitError encountered, attempt {retry_count}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if retry_count <= 2:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise RateLimitError("Rate limit exceeded")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return "Provider operation succeeded"
    ...
    ...    try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}result = provider_function()
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(result, flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}# Check for jitter in delays
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if len(retry_times) > 1:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}delays = [retry_times[i] - retry_times[i-1] for i in range(1, len(retry_times))]
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print(f"Delays with jitter: {[f'{d:.3f}' for d in delays]}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(0)
    ...    except Exception as e:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Error: {e}", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(1)
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Circuit Breaker Test Script
    [Arguments]    ${test_type}
    [Documentation]    Create script testing circuit breaker pattern
    ${script} =    Run Keyword If    '${test_type}' == 'open'
    ...    Get Circuit Breaker Open Script
    ...    ELSE    Get Circuit Breaker Recovery Script
    Create File    ${RETRY_TEST_FILE}    ${script}

Get Circuit Breaker Open Script
    [Documentation]    Return script for testing circuit breaker opening
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    import os
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import logging
    ...    import structlog
    ...    from cleveragents.core.retry_patterns import CircuitBreaker, CircuitBreakerOpen
    ...
    ...    # Suppress all debug logging
    ...    logging.getLogger().setLevel(logging.ERROR)
    ...    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.ERROR))
    ...    os.environ['PYTHONUNBUFFERED'] = '1'
    ...
    ...    cb = CircuitBreaker(failure_threshold=3, recovery_timeout=1.0)
    ...
    ...    def failing_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}raise Exception("Always fails")
    ...
    ...    # Trigger circuit breaker to open
    ...    for i in range(3):
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}cb.call(failing_function)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}except Exception:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}pass
    ...
    ...    print(f"Circuit breaker opened after 3 failures", flush=True)
    ...
    ...    # Try to call when open
    ...    try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}cb.call(lambda: "should not execute")
    ...    except CircuitBreakerOpen:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print("CircuitBreakerOpen exception raised", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(0)
    ...
    ...    # If we get here, the circuit breaker didn't open correctly
    ...    print("ERROR: Circuit breaker did not raise CircuitBreakerOpen", flush=True)
    ...    sys.exit(1)
    RETURN    ${script}

Get Circuit Breaker Recovery Script
    [Documentation]    Return script for testing circuit breaker recovery
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    import os
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import time
    ...    import logging
    ...    import structlog
    ...    from cleveragents.core.retry_patterns import CircuitBreaker, CircuitBreakerOpen
    ...
    ...    # Suppress all debug logging
    ...    logging.getLogger().setLevel(logging.ERROR)
    ...    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.ERROR))
    ...    os.environ['PYTHONUNBUFFERED'] = '1'
    ...
    ...    cb = CircuitBreaker(failure_threshold=2, recovery_timeout=0.5)
    ...
    ...    # Open the circuit
    ...    for i in range(2):
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}cb.call(lambda: 1/0)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}except:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}pass
    ...    print(f"Circuit breaker state: {cb.state}", flush=True)
    ...
    ...    # Wait for recovery timeout
    ...    time.sleep(0.6)
    ...
    ...    # Try with success - should go to half-open
    ...    cb.call(lambda: "success")
    ...    print(f"Circuit breaker state: {cb.state}", flush=True)
    ...
    ...    # Another success should close it
    ...    cb.call(lambda: "success")
    ...    print(f"Circuit breaker state: {cb.state}", flush=True)
    ...    sys.exit(0)
    RETURN    ${script}

Create Retry Context Test Script
    [Documentation]    Create script testing RetryContext manager
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    from cleveragents.core.retry_patterns import RetryContext
    ...
    ...    attempt_count = 0
    ...
    ...    def failing_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}global attempt_count
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}attempt_count += 1
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if attempt_count < 3:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise Exception(f"Attempt {attempt_count} failed")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return "Operation succeeded"
    ...
    ...    with RetryContext("test_operation", max_attempts=3) as ctx:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Retry context: test_operation")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}result = ctx.execute(failing_function)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Total attempts: {ctx.attempt_count}")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(result)
    ...
    ...    sys.exit(0)
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Auto Debug Test Script
    [Documentation]    Create script testing auto-debug retry pattern
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import asyncio
    ...    from cleveragents.core.retry_patterns import retry_auto_debug
    ...
    ...    debug_fixed = False
    ...
    ...    async def debug_callback(error, attempt):
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}global debug_fixed
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Debug callback invoked for attempt {attempt + 1}")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if attempt >= 1:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print("Issue fixed by debug callback")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}debug_fixed = True
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {"fixed": True}
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return {"fixed": False}
    ...
    ...    attempt_count = 0
    ...
    ...    @retry_auto_debug(max_debug_attempts=3, debug_callback=debug_callback)
    ...    async def test_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}global attempt_count, debug_fixed
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}attempt_count += 1
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Auto-debug attempt {attempt_count}")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if not debug_fixed and attempt_count <= 3:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise ValueError(f"Error {attempt_count}")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return {"success": True, "message": "Operation succeeded after debugging"}
    ...
    ...    async def main():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}result = await test_function()
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(result["message"])
    ...
    ...    asyncio.run(main())
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Timeout Retry Test Script
    [Documentation]    Create script testing retry with timeout
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import time
    ...    from cleveragents.core.retry_patterns import retry_with_timeout
    ...
    ...    @retry_with_timeout(max_attempts=10, timeout_seconds=2.0)
    ...    def slow_function():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print("Attempting slow operation...", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}time.sleep(1.0)  # Each attempt takes 1 second
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}raise Exception("Always fails")
    ...
    ...    try:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}slow_function()
    ...    except Exception as e:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print("Timeout exceeded", flush=True)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(1)  # Exit with non-zero code to indicate failure
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Category Test Script
    [Documentation]    Create script testing category-specific retry patterns
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    import os
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import logging
    ...    import structlog
    ...    from cleveragents.core.retry_patterns import RETRY_CATEGORIES
    ...
    ...    # Suppress all debug logging
    ...    logging.getLogger().setLevel(logging.ERROR)
    ...    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.ERROR))
    ...    os.environ['PYTHONUNBUFFERED'] = '1'
    ...
    ...    for category, config in RETRY_CATEGORIES.items():
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}max_attempts = config.get("max_attempts", "unknown")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}# Replace underscores with spaces and capitalize properly
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}display_name = category.replace("_", " ").capitalize()
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"{display_name} retry: max_attempts={max_attempts}", flush=True)
    ...    sys.exit(0)
    Create File    ${RETRY_TEST_FILE}    ${script}

Create Concurrent Retry Test Script
    [Documentation]    Create script testing concurrent retries with jitter
    ${script} =    Catenate    SEPARATOR=\n
    ...    import sys
    ...    sys.path.insert(0, '${WORKSPACE_ROOT}/src')
    ...    import time
    ...    import threading
    ...    from cleveragents.core.retry_patterns import retry_with_jitter
    ...
    ...    operation_times = []
    ...    lock = threading.Lock()
    ...
    ...    @retry_with_jitter(max_attempts=2, base_delay=0.1, max_delay=1.0)
    ...    def operation(op_id):
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}with lock:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}operation_times.append((op_id, time.time()))
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if len([t for t in operation_times if t[0] == op_id]) == 1:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}raise Exception(f"Operation {op_id} failed")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return f"Operation {op_id} succeeded"
    ...
    ...    threads = []
    ...    for i in range(5):
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}t = threading.Thread(target=operation, args=(i,))
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}threads.append(t)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}t.start()
    ...
    ...    for t in threads:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}t.join()
    ...
    ...    # Analyze timing to verify jitter
    ...    retry_times = {}
    ...    for op_id, timestamp in operation_times:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if op_id not in retry_times:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}retry_times[op_id] = []
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}retry_times[op_id].append(timestamp)
    ...
    ...    # Check that retries don't all happen at the same time
    ...    all_second_attempts = [times[1] for times in retry_times.values() if len(times) > 1]
    ...    if all_second_attempts:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}min_time = min(all_second_attempts)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}max_time = max(all_second_attempts)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}spread = max_time - min_time
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Retry time spread: {spread:.3f}s")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}if spread > 0.001:  # At least 1ms spread (jitter is working)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print("Jitter preventing thundering herd: SUCCESS")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(0)
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}else:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print("Warning: Retries too clustered")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(1)
    ...    else:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print("ERROR: No retry attempts detected")
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(1)
    Create File    ${RETRY_TEST_FILE}    ${script}

Verify Exponential Delays
    [Arguments]    ${output}
    [Documentation]    Verify that delays increase exponentially
    ${delays} =    Get Regexp Matches    ${output}    Delay \\d+: ([\\d.]+)s    1
    ${prev_delay} =    Set Variable    0
    FOR    ${delay}    IN    @{delays}
        ${delay_float} =    Convert To Number    ${delay}
        Should Be True    ${delay_float} > ${prev_delay}
        ${prev_delay} =    Set Variable    ${delay_float}
    END

Verify Jitter In Delays
    [Arguments]    ${output}
    [Documentation]    Verify that retry delays have jitter (randomness)
    ${match} =    Get Regexp Matches    ${output}    Delays with jitter: \\[(.+)\\]    1
    Should Not Be Empty    ${match}    No delay information found
    ${delays_str} =    Get From List    ${match}    0
    ${delays} =    Evaluate    [float(d.strip(" '")) for d in """${delays_str}""".split(',')]
    # Check that delays are not all the same (jitter is applied)
    ${unique_delays} =    Remove Duplicates    ${delays}
    ${unique_count} =    Get Length    ${unique_delays}
    Should Be True    ${unique_count} > 1    Delays should vary due to jitter

Verify No Thundering Herd
    [Arguments]    ${output}
    [Documentation]    Verify that concurrent retries are spread out
    Should Contain    ${output}    Jitter preventing thundering herd: SUCCESS