Files
cleveragents-core/features/steps/sandbox_checkpoints_steps.py
T
CoreRasurae 86e245c585
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 19s
CI / security (pull_request) Successful in 35s
CI / typecheck (pull_request) Successful in 37s
CI / unit_tests (pull_request) Successful in 2m22s
CI / integration_tests (pull_request) Successful in 2m51s
CI / docker (pull_request) Successful in 38s
CI / coverage (pull_request) Successful in 3m34s
CI / benchmark-regression (pull_request) Successful in 22m10s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 14s
CI / quality (push) Successful in 18s
CI / security (push) Successful in 32s
CI / typecheck (push) Successful in 34s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 1m55s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 2m52s
CI / coverage (push) Successful in 4m4s
CI / benchmark-publish (push) Successful in 13m13s
feat(checkpoint): add checkpointing and rollback
fix Alembic migration down_revision to chain after
  m4_002_skill_flattened_tools (was pointing to stale
  c3_001_actor_registry)

BDD coverage: 33 scenarios (7 new), 129 steps, all passing.

ISSUES CLOSED: #206
2026-03-02 10:18:14 +00:00

541 lines
19 KiB
Python

"""Step definitions for sandbox checkpoint and rollback BDD tests."""
from __future__ import annotations
import os
import tempfile
from typing import Any
from unittest.mock import MagicMock
from behave import given, then, when
from cleveragents.infrastructure.sandbox.checkpoint import (
CheckpointManager,
SandboxCheckpoint,
)
from cleveragents.infrastructure.sandbox.no_sandbox import NoSandbox
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _create_temp_sandbox(context: Any) -> None:
"""Create a temporary directory with sample files for sandbox testing."""
tmpdir = tempfile.mkdtemp(prefix="ca-cp-test-")
context._tmpdir = tmpdir
# Write sample files
os.makedirs(os.path.join(tmpdir, "src"), exist_ok=True)
with open(os.path.join(tmpdir, "src", "main.py"), "w") as f:
f.write("print('hello')\n")
with open(os.path.join(tmpdir, "README.md"), "w") as f:
f.write("# Test project\n")
sandbox = NoSandbox(resource_id="res-cp-test", original_path=tmpdir)
sandbox.create(plan_id="plan-cp-test")
context.sandbox = sandbox
context.sandbox_path = tmpdir
# ---------------------------------------------------------------------------
# Given steps
# ---------------------------------------------------------------------------
@given("a checkpoint manager")
def step_given_checkpoint_manager(context: Any) -> None:
context.checkpoint_manager = CheckpointManager()
context.checkpoint = None
context.checkpoints = []
context.rollback_result = None
context.delete_result = None
context.error = None
@given("a sandbox with files in a temporary directory")
def step_given_sandbox_with_files(context: Any) -> None:
_create_temp_sandbox(context)
@given('a checkpoint exists for the sandbox with phase "{phase}" and plan "{plan_id}"')
def step_given_checkpoint_exists(context: Any, phase: str, plan_id: str) -> None:
meta: dict[str, str] = {}
if hasattr(context, "sandbox_path") and context.sandbox_path:
meta["sandbox_path"] = context.sandbox_path
context.checkpoint = context.checkpoint_manager.create_checkpoint(
sandbox=context.sandbox,
plan_id=plan_id,
phase=phase,
metadata=meta,
)
@given("a checkpoint with an invalid snapshot path")
def step_given_invalid_checkpoint(context: Any) -> None:
context.invalid_checkpoint = SandboxCheckpoint(
checkpoint_id="invalid-cp-001",
sandbox_id="sb-invalid",
plan_id="plan-invalid",
phase="pre_execute",
created_at=__import__("datetime").datetime.now(),
metadata={},
snapshot_path="/nonexistent/path/snapshot",
)
@given("a plan lifecycle service")
def step_given_lifecycle_service(context: Any) -> None:
mock_settings = MagicMock()
mock_settings.cleveragents_home = "/tmp/test"
mock_settings.default_automation_profile = None
context.lifecycle_service = MagicMock()
context.lifecycle_service.settings = mock_settings
@given("a plan executor without checkpoint manager")
def step_given_executor_no_checkpoint(context: Any) -> None:
from cleveragents.application.services.plan_executor import PlanExecutor
context.executor = PlanExecutor(
lifecycle_service=context.lifecycle_service,
)
@given("a plan executor with a checkpoint manager")
def step_given_executor_with_checkpoint(context: Any) -> None:
from cleveragents.application.services.plan_executor import PlanExecutor
context.executor = PlanExecutor(
lifecycle_service=context.lifecycle_service,
checkpoint_manager=CheckpointManager(),
)
@given("a plan apply service without checkpoint manager")
def step_given_apply_no_checkpoint(context: Any) -> None:
from cleveragents.application.services.plan_apply_service import (
PlanApplyService,
)
context.apply_service = PlanApplyService(
lifecycle_service=context.lifecycle_service,
)
@given("a plan apply service with a checkpoint manager")
def step_given_apply_with_checkpoint(context: Any) -> None:
from cleveragents.application.services.plan_apply_service import (
PlanApplyService,
)
context.apply_service = PlanApplyService(
lifecycle_service=context.lifecycle_service,
checkpoint_manager=CheckpointManager(),
)
# ---------------------------------------------------------------------------
# When steps
# ---------------------------------------------------------------------------
@when('I create a checkpoint with phase "{phase}" and plan "{plan_id}"')
def step_when_create_checkpoint(context: Any, phase: str, plan_id: str) -> None:
meta: dict[str, str] = {}
if hasattr(context, "sandbox_path") and context.sandbox_path:
meta["sandbox_path"] = context.sandbox_path
context.checkpoint = context.checkpoint_manager.create_checkpoint(
sandbox=context.sandbox,
plan_id=plan_id,
phase=phase,
metadata=meta,
)
context.checkpoints.append(context.checkpoint)
@when('I attach metadata "{key}" as "{value}"')
def step_when_attach_metadata(context: Any, key: str, value: str) -> None:
# Metadata is attached during creation; re-create with metadata
meta = dict(context.checkpoint.metadata)
meta[key] = value
if hasattr(context, "sandbox_path") and context.sandbox_path:
meta["sandbox_path"] = context.sandbox_path
context.checkpoint = context.checkpoint_manager.create_checkpoint(
sandbox=context.sandbox,
plan_id=context.checkpoint.plan_id,
phase=context.checkpoint.phase,
metadata=meta,
)
@when("I modify the sandbox files")
def step_when_modify_sandbox(context: Any) -> None:
main_path = os.path.join(context.sandbox_path, "src", "main.py")
with open(main_path, "w") as f:
f.write("print('modified')\n")
new_file = os.path.join(context.sandbox_path, "new_file.txt")
with open(new_file, "w") as f:
f.write("new content\n")
@when("I rollback to the checkpoint")
def step_when_rollback(context: Any) -> None:
context.rollback_result = context.checkpoint_manager.rollback_to(context.checkpoint)
@when("I attempt to rollback to the invalid checkpoint")
def step_when_rollback_invalid(context: Any) -> None:
context.rollback_result = context.checkpoint_manager.rollback_to(
context.invalid_checkpoint
)
@when("I delete the checkpoint")
def step_when_delete_checkpoint(context: Any) -> None:
context.delete_result = context.checkpoint_manager.delete_checkpoint(
context.checkpoint.checkpoint_id
)
@when('I delete a checkpoint with id "{checkpoint_id}"')
def step_when_delete_nonexistent(context: Any, checkpoint_id: str) -> None:
context.delete_result = context.checkpoint_manager.delete_checkpoint(checkpoint_id)
# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------
@then("a checkpoint should be returned")
def step_then_checkpoint_returned(context: Any) -> None:
assert context.checkpoint is not None, "Expected a checkpoint"
assert isinstance(context.checkpoint, SandboxCheckpoint)
@then('the checkpoint phase should be "{phase}"')
def step_then_checkpoint_phase(context: Any, phase: str) -> None:
assert context.checkpoint.phase == phase, (
f"Expected phase '{phase}', got '{context.checkpoint.phase}'"
)
@then('the sandbox checkpoint plan_id should be "{plan_id}"')
def step_then_checkpoint_plan_id(context: Any, plan_id: str) -> None:
assert context.checkpoint.plan_id == plan_id, (
f"Expected plan_id '{plan_id}', got '{context.checkpoint.plan_id}'"
)
@then("the checkpoint sandbox_id should match the sandbox")
def step_then_checkpoint_sandbox_id(context: Any) -> None:
assert context.checkpoint.sandbox_id == context.sandbox.sandbox_id, (
f"Expected sandbox_id '{context.sandbox.sandbox_id}', "
f"got '{context.checkpoint.sandbox_id}'"
)
@then('the checkpoint metadata should contain key "{key}" with value "{value}"')
def step_then_checkpoint_metadata(context: Any, key: str, value: str) -> None:
assert key in context.checkpoint.metadata, (
f"Key '{key}' not in metadata: {context.checkpoint.metadata}"
)
assert context.checkpoint.metadata[key] == value, (
f"Expected metadata['{key}'] = '{value}', "
f"got '{context.checkpoint.metadata[key]}'"
)
@then("the sandbox rollback should succeed")
def step_then_rollback_success(context: Any) -> None:
assert context.rollback_result is True, "Expected rollback to succeed"
@then("the rollback should fail")
def step_then_rollback_fail(context: Any) -> None:
assert context.rollback_result is False, "Expected rollback to fail"
@then("the sandbox files should match the original state")
def step_then_sandbox_matches_original(context: Any) -> None:
main_path = os.path.join(context.sandbox_path, "src", "main.py")
with open(main_path) as f:
content = f.read()
assert content == "print('hello')\n", f"Expected original content, got: {content!r}"
new_file = os.path.join(context.sandbox_path, "new_file.txt")
assert not os.path.exists(new_file), "new_file.txt should not exist"
@then("listing checkpoints for the sandbox should return {count:d} checkpoints")
def step_then_checkpoint_count(context: Any, count: int) -> None:
result = context.checkpoint_manager.list_checkpoints(context.sandbox.sandbox_id)
assert len(result) == count, f"Expected {count} checkpoints, got {len(result)}"
@then("the checkpoints should be in creation order")
def step_then_checkpoints_ordered(context: Any) -> None:
result = context.checkpoint_manager.list_checkpoints(context.sandbox.sandbox_id)
for i in range(1, len(result)):
assert result[i].created_at >= result[i - 1].created_at, (
f"Checkpoints not in order at index {i}"
)
@then("the checkpoint should have a valid ULID checkpoint_id")
def step_then_valid_ulid(context: Any) -> None:
cp_id = context.checkpoint.checkpoint_id
assert len(cp_id) == 26, f"ULID should be 26 chars, got {len(cp_id)}"
@then("the checkpoint should have a created_at timestamp")
def step_then_has_timestamp(context: Any) -> None:
assert context.checkpoint.created_at is not None
@then("the checkpoint should have a non-empty snapshot_path")
def step_then_has_snapshot_path(context: Any) -> None:
assert context.checkpoint.snapshot_path, "snapshot_path should not be empty"
assert os.path.isdir(context.checkpoint.snapshot_path), (
f"snapshot_path should be a directory: {context.checkpoint.snapshot_path}"
)
@then("the deletion should succeed")
def step_then_deletion_success(context: Any) -> None:
assert context.delete_result is True, "Expected deletion to succeed"
@then("the deletion should fail")
def step_then_deletion_fail(context: Any) -> None:
assert context.delete_result is False, "Expected deletion to fail"
@then("the plan executor checkpoint_manager should be None")
def step_then_executor_no_checkpoint(context: Any) -> None:
assert context.executor.checkpoint_manager is None
@then("the plan executor checkpoint_manager should not be None")
def step_then_executor_has_checkpoint(context: Any) -> None:
assert context.executor.checkpoint_manager is not None
@then("the plan apply service checkpoint_manager should be None")
def step_then_apply_no_checkpoint(context: Any) -> None:
assert context.apply_service.checkpoint_manager is None
@then("the plan apply service checkpoint_manager should not be None")
def step_then_apply_has_checkpoint(context: Any) -> None:
assert context.apply_service.checkpoint_manager is not None
# ---------------------------------------------------------------------------
# Context-less sandbox (metadata-only checkpoint) steps
# ---------------------------------------------------------------------------
class _NoContextSandboxProxy:
"""Sandbox proxy with ``context = None`` for metadata-only checkpoints."""
def __init__(self, sandbox_id: str) -> None:
self._sandbox_id = sandbox_id
@property
def sandbox_id(self) -> str:
return self._sandbox_id
@property
def context(self) -> Any:
return None
@given("a sandbox proxy with no context")
def step_given_no_context_proxy(context: Any) -> None:
context.sandbox = _NoContextSandboxProxy(sandbox_id="sb-no-ctx-001")
@when('I create a checkpoint with phase "{phase}" and plan "{plan_id}" using the proxy')
def step_when_create_checkpoint_proxy(context: Any, phase: str, plan_id: str) -> None:
context.checkpoint = context.checkpoint_manager.create_checkpoint(
sandbox=context.sandbox,
plan_id=plan_id,
phase=phase,
)
@then("the checkpoint snapshot_path should be an empty directory")
def step_then_snapshot_empty_dir(context: Any) -> None:
path = context.checkpoint.snapshot_path
assert os.path.isdir(path), f"snapshot_path should be a directory: {path}"
contents = os.listdir(path)
assert len(contents) == 0, f"Expected empty dir, got: {contents}"
# ---------------------------------------------------------------------------
# Rollback edge case steps
# ---------------------------------------------------------------------------
@given(
'a checkpoint exists for the sandbox with phase "{phase}" and plan "{plan_id}"'
" without sandbox_path metadata"
)
def step_given_checkpoint_no_sandbox_path(
context: Any, phase: str, plan_id: str
) -> None:
"""Create a checkpoint without embedding sandbox_path in metadata."""
context.checkpoint = context.checkpoint_manager.create_checkpoint(
sandbox=context.sandbox,
plan_id=plan_id,
phase=phase,
metadata={},
)
@when("I attempt to rollback to the checkpoint without sandbox path")
def step_when_rollback_no_sandbox_path(context: Any) -> None:
context.rollback_result = context.checkpoint_manager.rollback_to(context.checkpoint)
@when("I make the sandbox path read-only and attempt rollback")
def step_when_rollback_readonly_sandbox(context: Any) -> None:
from unittest.mock import patch
# Create checkpoint with sandbox_path in metadata
meta = {"sandbox_path": context.sandbox_path}
cp = context.checkpoint_manager.create_checkpoint(
sandbox=context.sandbox,
plan_id="plan-001",
phase="pre_execute",
metadata=meta,
)
# Patch os.listdir to raise OSError, simulating a filesystem failure
# during rollback. This avoids chmod which is ignored when running as root.
with patch("os.listdir", side_effect=OSError("Simulated I/O error")):
context.rollback_result = context.checkpoint_manager.rollback_to(cp)
# ---------------------------------------------------------------------------
# Multi-sandbox delete steps
# ---------------------------------------------------------------------------
@given("two sandboxes with checkpoints")
def step_given_two_sandboxes(context: Any) -> None:
"""Create checkpoints for two different sandboxes."""
# First sandbox
tmpdir1 = tempfile.mkdtemp(prefix="ca-cp-multi1-")
os.makedirs(os.path.join(tmpdir1, "src"), exist_ok=True)
with open(os.path.join(tmpdir1, "src", "a.py"), "w") as f:
f.write("# sandbox1\n")
sb1 = NoSandbox(resource_id="res-multi-1", original_path=tmpdir1)
sb1.create(plan_id="plan-multi")
tmpdir2 = tempfile.mkdtemp(prefix="ca-cp-multi2-")
os.makedirs(os.path.join(tmpdir2, "src"), exist_ok=True)
with open(os.path.join(tmpdir2, "src", "b.py"), "w") as f:
f.write("# sandbox2\n")
sb2 = NoSandbox(resource_id="res-multi-2", original_path=tmpdir2)
sb2.create(plan_id="plan-multi")
context.sandbox_one = sb1
context.sandbox_two = sb2
context.cp_one = context.checkpoint_manager.create_checkpoint(
sandbox=sb1, plan_id="plan-multi", phase="pre_execute"
)
context.cp_two = context.checkpoint_manager.create_checkpoint(
sandbox=sb2, plan_id="plan-multi", phase="pre_execute"
)
@when("I delete the checkpoint from the second sandbox")
def step_when_delete_second_sandbox_cp(context: Any) -> None:
context.delete_result = context.checkpoint_manager.delete_checkpoint(
context.cp_two.checkpoint_id
)
@then(
"listing checkpoints for the first sandbox should return"
" 1 checkpoint for sandbox one"
)
def step_then_first_sandbox_still_has_one(context: Any) -> None:
result = context.checkpoint_manager.list_checkpoints(context.sandbox_one.sandbox_id)
assert len(result) == 1, f"Expected 1 checkpoint, got {len(result)}"
# ---------------------------------------------------------------------------
# Snapshot copy failure steps
# ---------------------------------------------------------------------------
@given("a sandbox with an unreadable directory")
def step_given_unreadable_sandbox(context: Any) -> None:
tmpdir = tempfile.mkdtemp(prefix="ca-cp-unreadable-")
os.makedirs(os.path.join(tmpdir, "src"), exist_ok=True)
with open(os.path.join(tmpdir, "src", "file.txt"), "w") as f:
f.write("data\n")
sandbox = NoSandbox(resource_id="res-cp-unreadable", original_path=tmpdir)
sandbox.create(plan_id="plan-unreadable")
context.sandbox = sandbox
context.sandbox_path = tmpdir
@when("I attempt to create a checkpoint that fails during snapshot")
def step_when_create_checkpoint_snapshot_fail(context: Any) -> None:
from unittest.mock import patch
from cleveragents.infrastructure.sandbox.protocol import SandboxError
# Patch shutil.copytree to raise OSError, simulating a snapshot copy
# failure. This avoids chmod which is ignored when running as root.
with patch("shutil.copytree", side_effect=OSError("Simulated copy error")):
try:
context.checkpoint_manager.create_checkpoint(
sandbox=context.sandbox,
plan_id="plan-unreadable",
phase="pre_execute",
)
context.error = None
except SandboxError as exc:
context.error = exc
@then("a sandbox error should be raised")
def step_then_sandbox_error(context: Any) -> None:
from cleveragents.infrastructure.sandbox.protocol import SandboxError
assert context.error is not None, "Expected a SandboxError"
assert isinstance(context.error, SandboxError)
# ---------------------------------------------------------------------------
# Cleanup snapshot edge case steps
# ---------------------------------------------------------------------------
@when("I cleanup a snapshot with an empty path")
def step_when_cleanup_empty_path(context: Any) -> None:
context.error = None
try:
CheckpointManager._cleanup_snapshot("")
except Exception as exc:
context.error = exc
@when("I cleanup a snapshot with a non-existent parent")
def step_when_cleanup_nonexistent_parent(context: Any) -> None:
context.error = None
try:
CheckpointManager._cleanup_snapshot("/nonexistent/path/that/does/not/exist")
except Exception as exc:
context.error = exc
@then("no error should occur")
def step_then_no_error(context: Any) -> None:
assert context.error is None, f"Expected no error, got: {context.error}"