# Retry Pattern

## Problem

Transient failures — brief network glitches, temporary service overloads, database connection resets — are common in distributed systems. A single failure doesn't mean the operation is permanently broken. Without retries, these temporary issues cause unnecessary errors for users.

## Solution

Automatically reattempt a failed operation with increasing delays between attempts. Use **exponential backoff** to avoid thundering-herd problems, and add **jitter** (randomness) to spread retries across time so multiple clients don't all retry at the same instant.

Formula: `delay = min(base * 2^attempt + random_jitter, max_delay)`

## When to Use

- Network calls that can experience transient failures
- Database connections that occasionally timeout
- Message queue publishing where the broker might briefly be unavailable
- Any idempotent operation against an unreliable dependency

## When to Avoid

- Non-idempotent operations (e.g., payment processing without idempotency keys)
- Errors that are clearly permanent (404, 401, validation errors)
- When the caller needs immediate feedback (UI interactions)
- When the downstream service has no capacity to recover (combine with circuit breaker instead)

## Pseudocode

```
function retry_with_backoff(operation, max_attempts, base_delay, max_delay):
    for attempt in 0..max_attempts:
        try:
            return operation()
        catch transient_error:
            if attempt == max_attempts - 1:
                raise
            delay = min(base_delay * 2^attempt, max_delay)
            jitter = random(0, delay * 0.5)
            sleep(delay + jitter)
```

## Python Implementation

```python
import time
import random


class RetryExhaustedError(Exception):
    def __init__(self, attempts: int, last_error: str):
        super().__init__(f"All {attempts} attempts failed. Last error: {last_error}")


def retry_with_backoff(
    operation,
    max_attempts: int = 5,
    base_delay: float = 0.1,
    max_delay: float = 2.0,
):
    last_error = "unknown"
    for attempt in range(max_attempts):
        try:
            return operation()
        except Exception as exc:
            last_error = str(exc)
            if attempt == max_attempts - 1:
                break
            delay = min(base_delay * (2 ** attempt), max_delay)
            jitter = random.uniform(0, delay * 0.5)
            total_delay = delay + jitter
            print(f"  Attempt {attempt + 1} failed: {last_error}. "
                  f"Retrying in {total_delay:.3f}s...")
            time.sleep(total_delay)

    raise RetryExhaustedError(max_attempts, last_error)


# --- Demo ---
call_count = 0

def flaky_service():
    global call_count
    call_count += 1
    if call_count < 4:
        raise ConnectionError(f"Timeout on attempt {call_count}")
    return f"Success on attempt {call_count}"


print("=== Retry with exponential backoff ===")
try:
    result = retry_with_backoff(flaky_service, max_attempts=5, base_delay=0.05)
    print(f"Result: {result}")
except RetryExhaustedError as e:
    print(f"Gave up: {e}")

print()

# Demonstrate exhaustion
call_count = 0

def always_fails():
    global call_count
    call_count += 1
    raise ConnectionError(f"Down (attempt {call_count})")

print("=== Retry exhaustion ===")
try:
    retry_with_backoff(always_fails, max_attempts=3, base_delay=0.05)
except RetryExhaustedError as e:
    print(f"Gave up: {e}")
```

**Output:**
```
=== Retry with exponential backoff ===
  Attempt 1 failed: Timeout on attempt 1. Retrying in 0.0XXs...
  Attempt 2 failed: Timeout on attempt 2. Retrying in 0.1XXs...
  Attempt 3 failed: Timeout on attempt 3. Retrying in 0.2XXs...
Result: Success on attempt 4

=== Retry exhaustion ===
  Attempt 1 failed: Down (attempt 1). Retrying in 0.0XXs...
  Attempt 2 failed: Down (attempt 2). Retrying in 0.1XXs...
Gave up: All 3 attempts failed. Last error: Down (attempt 3)
```

## Go Implementation

```go
package main

import (
	"errors"
	"fmt"
	"math"
	"math/rand"
	"time"
)

type RetryConfig struct {
	MaxAttempts int
	BaseDelay   time.Duration
	MaxDelay    time.Duration
}

func RetryWithBackoff(cfg RetryConfig, operation func() (string, error)) (string, error) {
	var lastErr error
	for attempt := 0; attempt < cfg.MaxAttempts; attempt++ {
		result, err := operation()
		if err == nil {
			return result, nil
		}
		lastErr = err
		if attempt == cfg.MaxAttempts-1 {
			break
		}
		delay := time.Duration(float64(cfg.BaseDelay) * math.Pow(2, float64(attempt)))
		if delay > cfg.MaxDelay {
			delay = cfg.MaxDelay
		}
		jitter := time.Duration(rand.Float64() * float64(delay) * 0.5)
		total := delay + jitter
		fmt.Printf("  Attempt %d failed: %v. Retrying in %v...\n", attempt+1, err, total.Round(time.Millisecond))
		time.Sleep(total)
	}
	return "", fmt.Errorf("all %d attempts failed. Last error: %w", cfg.MaxAttempts, lastErr)
}

func main() {
	callCount := 0

	flaky := func() (string, error) {
		callCount++
		if callCount < 4 {
			return "", fmt.Errorf("timeout on attempt %d", callCount)
		}
		return fmt.Sprintf("Success on attempt %d", callCount), nil
	}

	cfg := RetryConfig{MaxAttempts: 5, BaseDelay: 50 * time.Millisecond, MaxDelay: 2 * time.Second}

	fmt.Println("=== Retry with exponential backoff ===")
	result, err := RetryWithBackoff(cfg, flaky)
	if err != nil {
		fmt.Println("Gave up:", err)
	} else {
		fmt.Println("Result:", result)
	}

	fmt.Println()
	fmt.Println("=== Retry exhaustion ===")
	failCount := 0
	alwaysFails := func() (string, error) {
		failCount++
		return "", errors.New(fmt.Sprintf("down (attempt %d)", failCount))
	}
	cfg2 := RetryConfig{MaxAttempts: 3, BaseDelay: 50 * time.Millisecond, MaxDelay: 2 * time.Second}
	_, err = RetryWithBackoff(cfg2, alwaysFails)
	if err != nil {
		fmt.Println("Gave up:", err)
	}
}
```

## JavaScript Implementation

```javascript
class RetryExhaustedError extends Error {
  constructor(attempts, lastError) {
    super(`All ${attempts} attempts failed. Last error: ${lastError}`);
    this.name = "RetryExhaustedError";
  }
}

async function retryWithBackoff(operation, { maxAttempts = 5, baseDelay = 100, maxDelay = 2000 } = {}) {
  let lastError;
  for (let attempt = 0; attempt < maxAttempts; attempt++) {
    try {
      return await operation();
    } catch (err) {
      lastError = err;
      if (attempt === maxAttempts - 1) break;

      const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
      const jitter = Math.random() * delay * 0.5;
      const total = delay + jitter;
      console.log(`  Attempt ${attempt + 1} failed: ${err.message}. Retrying in ${total.toFixed(0)}ms...`);
      await new Promise((r) => setTimeout(r, total));
    }
  }
  throw new RetryExhaustedError(maxAttempts, lastError.message);
}

// --- Demo ---
let callCount = 0;
function flakyService() {
  callCount++;
  if (callCount < 4) throw new Error(`Timeout on attempt ${callCount}`);
  return `Success on attempt ${callCount}`;
}

(async () => {
  console.log("=== Retry with exponential backoff ===");
  try {
    const result = await retryWithBackoff(flakyService, { maxAttempts: 5, baseDelay: 50 });
    console.log("Result:", result);
  } catch (e) {
    console.log("Gave up:", e.message);
  }

  console.log();
  console.log("=== Retry exhaustion ===");
  let failCount = 0;
  try {
    await retryWithBackoff(
      () => { failCount++; throw new Error(`Down (attempt ${failCount})`); },
      { maxAttempts: 3, baseDelay: 50 }
    );
  } catch (e) {
    console.log("Gave up:", e.message);
  }
})();
```

## Related Patterns

- **Circuit Breaker** — use circuit breaker with retry: after too many retries trip the breaker to stop futile attempts.
- **Timeout** — always pair retry with a timeout so each individual attempt is bounded.
- **Dead Letter Queue** — messages that exhaust all retries go to the DLQ for inspection and manual reprocessing.
- **Fallback** — when all retries are exhausted, a fallback provides a degraded response instead of a hard failure.
