86e245c585
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 19s
CI / security (pull_request) Successful in 35s
CI / typecheck (pull_request) Successful in 37s
CI / unit_tests (pull_request) Successful in 2m22s
CI / integration_tests (pull_request) Successful in 2m51s
CI / docker (pull_request) Successful in 38s
CI / coverage (pull_request) Successful in 3m34s
CI / benchmark-regression (pull_request) Successful in 22m10s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 14s
CI / quality (push) Successful in 18s
CI / security (push) Successful in 32s
CI / typecheck (push) Successful in 34s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 1m55s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 2m52s
CI / coverage (push) Successful in 4m4s
CI / benchmark-publish (push) Successful in 13m13s
fix Alembic migration down_revision to chain after m4_002_skill_flattened_tools (was pointing to stale c3_001_actor_registry) BDD coverage: 33 scenarios (7 new), 129 steps, all passing. ISSUES CLOSED: #206
541 lines
19 KiB
Python
541 lines
19 KiB
Python
"""Step definitions for sandbox checkpoint and rollback BDD tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
from behave import given, then, when
|
|
|
|
from cleveragents.infrastructure.sandbox.checkpoint import (
|
|
CheckpointManager,
|
|
SandboxCheckpoint,
|
|
)
|
|
from cleveragents.infrastructure.sandbox.no_sandbox import NoSandbox
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _create_temp_sandbox(context: Any) -> None:
|
|
"""Create a temporary directory with sample files for sandbox testing."""
|
|
tmpdir = tempfile.mkdtemp(prefix="ca-cp-test-")
|
|
context._tmpdir = tmpdir
|
|
|
|
# Write sample files
|
|
os.makedirs(os.path.join(tmpdir, "src"), exist_ok=True)
|
|
with open(os.path.join(tmpdir, "src", "main.py"), "w") as f:
|
|
f.write("print('hello')\n")
|
|
with open(os.path.join(tmpdir, "README.md"), "w") as f:
|
|
f.write("# Test project\n")
|
|
|
|
sandbox = NoSandbox(resource_id="res-cp-test", original_path=tmpdir)
|
|
sandbox.create(plan_id="plan-cp-test")
|
|
context.sandbox = sandbox
|
|
context.sandbox_path = tmpdir
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a checkpoint manager")
|
|
def step_given_checkpoint_manager(context: Any) -> None:
|
|
context.checkpoint_manager = CheckpointManager()
|
|
context.checkpoint = None
|
|
context.checkpoints = []
|
|
context.rollback_result = None
|
|
context.delete_result = None
|
|
context.error = None
|
|
|
|
|
|
@given("a sandbox with files in a temporary directory")
|
|
def step_given_sandbox_with_files(context: Any) -> None:
|
|
_create_temp_sandbox(context)
|
|
|
|
|
|
@given('a checkpoint exists for the sandbox with phase "{phase}" and plan "{plan_id}"')
|
|
def step_given_checkpoint_exists(context: Any, phase: str, plan_id: str) -> None:
|
|
meta: dict[str, str] = {}
|
|
if hasattr(context, "sandbox_path") and context.sandbox_path:
|
|
meta["sandbox_path"] = context.sandbox_path
|
|
context.checkpoint = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=context.sandbox,
|
|
plan_id=plan_id,
|
|
phase=phase,
|
|
metadata=meta,
|
|
)
|
|
|
|
|
|
@given("a checkpoint with an invalid snapshot path")
|
|
def step_given_invalid_checkpoint(context: Any) -> None:
|
|
context.invalid_checkpoint = SandboxCheckpoint(
|
|
checkpoint_id="invalid-cp-001",
|
|
sandbox_id="sb-invalid",
|
|
plan_id="plan-invalid",
|
|
phase="pre_execute",
|
|
created_at=__import__("datetime").datetime.now(),
|
|
metadata={},
|
|
snapshot_path="/nonexistent/path/snapshot",
|
|
)
|
|
|
|
|
|
@given("a plan lifecycle service")
|
|
def step_given_lifecycle_service(context: Any) -> None:
|
|
mock_settings = MagicMock()
|
|
mock_settings.cleveragents_home = "/tmp/test"
|
|
mock_settings.default_automation_profile = None
|
|
context.lifecycle_service = MagicMock()
|
|
context.lifecycle_service.settings = mock_settings
|
|
|
|
|
|
@given("a plan executor without checkpoint manager")
|
|
def step_given_executor_no_checkpoint(context: Any) -> None:
|
|
from cleveragents.application.services.plan_executor import PlanExecutor
|
|
|
|
context.executor = PlanExecutor(
|
|
lifecycle_service=context.lifecycle_service,
|
|
)
|
|
|
|
|
|
@given("a plan executor with a checkpoint manager")
|
|
def step_given_executor_with_checkpoint(context: Any) -> None:
|
|
from cleveragents.application.services.plan_executor import PlanExecutor
|
|
|
|
context.executor = PlanExecutor(
|
|
lifecycle_service=context.lifecycle_service,
|
|
checkpoint_manager=CheckpointManager(),
|
|
)
|
|
|
|
|
|
@given("a plan apply service without checkpoint manager")
|
|
def step_given_apply_no_checkpoint(context: Any) -> None:
|
|
from cleveragents.application.services.plan_apply_service import (
|
|
PlanApplyService,
|
|
)
|
|
|
|
context.apply_service = PlanApplyService(
|
|
lifecycle_service=context.lifecycle_service,
|
|
)
|
|
|
|
|
|
@given("a plan apply service with a checkpoint manager")
|
|
def step_given_apply_with_checkpoint(context: Any) -> None:
|
|
from cleveragents.application.services.plan_apply_service import (
|
|
PlanApplyService,
|
|
)
|
|
|
|
context.apply_service = PlanApplyService(
|
|
lifecycle_service=context.lifecycle_service,
|
|
checkpoint_manager=CheckpointManager(),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I create a checkpoint with phase "{phase}" and plan "{plan_id}"')
|
|
def step_when_create_checkpoint(context: Any, phase: str, plan_id: str) -> None:
|
|
meta: dict[str, str] = {}
|
|
if hasattr(context, "sandbox_path") and context.sandbox_path:
|
|
meta["sandbox_path"] = context.sandbox_path
|
|
context.checkpoint = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=context.sandbox,
|
|
plan_id=plan_id,
|
|
phase=phase,
|
|
metadata=meta,
|
|
)
|
|
context.checkpoints.append(context.checkpoint)
|
|
|
|
|
|
@when('I attach metadata "{key}" as "{value}"')
|
|
def step_when_attach_metadata(context: Any, key: str, value: str) -> None:
|
|
# Metadata is attached during creation; re-create with metadata
|
|
meta = dict(context.checkpoint.metadata)
|
|
meta[key] = value
|
|
if hasattr(context, "sandbox_path") and context.sandbox_path:
|
|
meta["sandbox_path"] = context.sandbox_path
|
|
context.checkpoint = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=context.sandbox,
|
|
plan_id=context.checkpoint.plan_id,
|
|
phase=context.checkpoint.phase,
|
|
metadata=meta,
|
|
)
|
|
|
|
|
|
@when("I modify the sandbox files")
|
|
def step_when_modify_sandbox(context: Any) -> None:
|
|
main_path = os.path.join(context.sandbox_path, "src", "main.py")
|
|
with open(main_path, "w") as f:
|
|
f.write("print('modified')\n")
|
|
new_file = os.path.join(context.sandbox_path, "new_file.txt")
|
|
with open(new_file, "w") as f:
|
|
f.write("new content\n")
|
|
|
|
|
|
@when("I rollback to the checkpoint")
|
|
def step_when_rollback(context: Any) -> None:
|
|
context.rollback_result = context.checkpoint_manager.rollback_to(context.checkpoint)
|
|
|
|
|
|
@when("I attempt to rollback to the invalid checkpoint")
|
|
def step_when_rollback_invalid(context: Any) -> None:
|
|
context.rollback_result = context.checkpoint_manager.rollback_to(
|
|
context.invalid_checkpoint
|
|
)
|
|
|
|
|
|
@when("I delete the checkpoint")
|
|
def step_when_delete_checkpoint(context: Any) -> None:
|
|
context.delete_result = context.checkpoint_manager.delete_checkpoint(
|
|
context.checkpoint.checkpoint_id
|
|
)
|
|
|
|
|
|
@when('I delete a checkpoint with id "{checkpoint_id}"')
|
|
def step_when_delete_nonexistent(context: Any, checkpoint_id: str) -> None:
|
|
context.delete_result = context.checkpoint_manager.delete_checkpoint(checkpoint_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("a checkpoint should be returned")
|
|
def step_then_checkpoint_returned(context: Any) -> None:
|
|
assert context.checkpoint is not None, "Expected a checkpoint"
|
|
assert isinstance(context.checkpoint, SandboxCheckpoint)
|
|
|
|
|
|
@then('the checkpoint phase should be "{phase}"')
|
|
def step_then_checkpoint_phase(context: Any, phase: str) -> None:
|
|
assert context.checkpoint.phase == phase, (
|
|
f"Expected phase '{phase}', got '{context.checkpoint.phase}'"
|
|
)
|
|
|
|
|
|
@then('the sandbox checkpoint plan_id should be "{plan_id}"')
|
|
def step_then_checkpoint_plan_id(context: Any, plan_id: str) -> None:
|
|
assert context.checkpoint.plan_id == plan_id, (
|
|
f"Expected plan_id '{plan_id}', got '{context.checkpoint.plan_id}'"
|
|
)
|
|
|
|
|
|
@then("the checkpoint sandbox_id should match the sandbox")
|
|
def step_then_checkpoint_sandbox_id(context: Any) -> None:
|
|
assert context.checkpoint.sandbox_id == context.sandbox.sandbox_id, (
|
|
f"Expected sandbox_id '{context.sandbox.sandbox_id}', "
|
|
f"got '{context.checkpoint.sandbox_id}'"
|
|
)
|
|
|
|
|
|
@then('the checkpoint metadata should contain key "{key}" with value "{value}"')
|
|
def step_then_checkpoint_metadata(context: Any, key: str, value: str) -> None:
|
|
assert key in context.checkpoint.metadata, (
|
|
f"Key '{key}' not in metadata: {context.checkpoint.metadata}"
|
|
)
|
|
assert context.checkpoint.metadata[key] == value, (
|
|
f"Expected metadata['{key}'] = '{value}', "
|
|
f"got '{context.checkpoint.metadata[key]}'"
|
|
)
|
|
|
|
|
|
@then("the sandbox rollback should succeed")
|
|
def step_then_rollback_success(context: Any) -> None:
|
|
assert context.rollback_result is True, "Expected rollback to succeed"
|
|
|
|
|
|
@then("the rollback should fail")
|
|
def step_then_rollback_fail(context: Any) -> None:
|
|
assert context.rollback_result is False, "Expected rollback to fail"
|
|
|
|
|
|
@then("the sandbox files should match the original state")
|
|
def step_then_sandbox_matches_original(context: Any) -> None:
|
|
main_path = os.path.join(context.sandbox_path, "src", "main.py")
|
|
with open(main_path) as f:
|
|
content = f.read()
|
|
assert content == "print('hello')\n", f"Expected original content, got: {content!r}"
|
|
new_file = os.path.join(context.sandbox_path, "new_file.txt")
|
|
assert not os.path.exists(new_file), "new_file.txt should not exist"
|
|
|
|
|
|
@then("listing checkpoints for the sandbox should return {count:d} checkpoints")
|
|
def step_then_checkpoint_count(context: Any, count: int) -> None:
|
|
result = context.checkpoint_manager.list_checkpoints(context.sandbox.sandbox_id)
|
|
assert len(result) == count, f"Expected {count} checkpoints, got {len(result)}"
|
|
|
|
|
|
@then("the checkpoints should be in creation order")
|
|
def step_then_checkpoints_ordered(context: Any) -> None:
|
|
result = context.checkpoint_manager.list_checkpoints(context.sandbox.sandbox_id)
|
|
for i in range(1, len(result)):
|
|
assert result[i].created_at >= result[i - 1].created_at, (
|
|
f"Checkpoints not in order at index {i}"
|
|
)
|
|
|
|
|
|
@then("the checkpoint should have a valid ULID checkpoint_id")
|
|
def step_then_valid_ulid(context: Any) -> None:
|
|
cp_id = context.checkpoint.checkpoint_id
|
|
assert len(cp_id) == 26, f"ULID should be 26 chars, got {len(cp_id)}"
|
|
|
|
|
|
@then("the checkpoint should have a created_at timestamp")
|
|
def step_then_has_timestamp(context: Any) -> None:
|
|
assert context.checkpoint.created_at is not None
|
|
|
|
|
|
@then("the checkpoint should have a non-empty snapshot_path")
|
|
def step_then_has_snapshot_path(context: Any) -> None:
|
|
assert context.checkpoint.snapshot_path, "snapshot_path should not be empty"
|
|
assert os.path.isdir(context.checkpoint.snapshot_path), (
|
|
f"snapshot_path should be a directory: {context.checkpoint.snapshot_path}"
|
|
)
|
|
|
|
|
|
@then("the deletion should succeed")
|
|
def step_then_deletion_success(context: Any) -> None:
|
|
assert context.delete_result is True, "Expected deletion to succeed"
|
|
|
|
|
|
@then("the deletion should fail")
|
|
def step_then_deletion_fail(context: Any) -> None:
|
|
assert context.delete_result is False, "Expected deletion to fail"
|
|
|
|
|
|
@then("the plan executor checkpoint_manager should be None")
|
|
def step_then_executor_no_checkpoint(context: Any) -> None:
|
|
assert context.executor.checkpoint_manager is None
|
|
|
|
|
|
@then("the plan executor checkpoint_manager should not be None")
|
|
def step_then_executor_has_checkpoint(context: Any) -> None:
|
|
assert context.executor.checkpoint_manager is not None
|
|
|
|
|
|
@then("the plan apply service checkpoint_manager should be None")
|
|
def step_then_apply_no_checkpoint(context: Any) -> None:
|
|
assert context.apply_service.checkpoint_manager is None
|
|
|
|
|
|
@then("the plan apply service checkpoint_manager should not be None")
|
|
def step_then_apply_has_checkpoint(context: Any) -> None:
|
|
assert context.apply_service.checkpoint_manager is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Context-less sandbox (metadata-only checkpoint) steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class _NoContextSandboxProxy:
|
|
"""Sandbox proxy with ``context = None`` for metadata-only checkpoints."""
|
|
|
|
def __init__(self, sandbox_id: str) -> None:
|
|
self._sandbox_id = sandbox_id
|
|
|
|
@property
|
|
def sandbox_id(self) -> str:
|
|
return self._sandbox_id
|
|
|
|
@property
|
|
def context(self) -> Any:
|
|
return None
|
|
|
|
|
|
@given("a sandbox proxy with no context")
|
|
def step_given_no_context_proxy(context: Any) -> None:
|
|
context.sandbox = _NoContextSandboxProxy(sandbox_id="sb-no-ctx-001")
|
|
|
|
|
|
@when('I create a checkpoint with phase "{phase}" and plan "{plan_id}" using the proxy')
|
|
def step_when_create_checkpoint_proxy(context: Any, phase: str, plan_id: str) -> None:
|
|
context.checkpoint = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=context.sandbox,
|
|
plan_id=plan_id,
|
|
phase=phase,
|
|
)
|
|
|
|
|
|
@then("the checkpoint snapshot_path should be an empty directory")
|
|
def step_then_snapshot_empty_dir(context: Any) -> None:
|
|
path = context.checkpoint.snapshot_path
|
|
assert os.path.isdir(path), f"snapshot_path should be a directory: {path}"
|
|
contents = os.listdir(path)
|
|
assert len(contents) == 0, f"Expected empty dir, got: {contents}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rollback edge case steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given(
|
|
'a checkpoint exists for the sandbox with phase "{phase}" and plan "{plan_id}"'
|
|
" without sandbox_path metadata"
|
|
)
|
|
def step_given_checkpoint_no_sandbox_path(
|
|
context: Any, phase: str, plan_id: str
|
|
) -> None:
|
|
"""Create a checkpoint without embedding sandbox_path in metadata."""
|
|
context.checkpoint = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=context.sandbox,
|
|
plan_id=plan_id,
|
|
phase=phase,
|
|
metadata={},
|
|
)
|
|
|
|
|
|
@when("I attempt to rollback to the checkpoint without sandbox path")
|
|
def step_when_rollback_no_sandbox_path(context: Any) -> None:
|
|
context.rollback_result = context.checkpoint_manager.rollback_to(context.checkpoint)
|
|
|
|
|
|
@when("I make the sandbox path read-only and attempt rollback")
|
|
def step_when_rollback_readonly_sandbox(context: Any) -> None:
|
|
from unittest.mock import patch
|
|
|
|
# Create checkpoint with sandbox_path in metadata
|
|
meta = {"sandbox_path": context.sandbox_path}
|
|
cp = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=context.sandbox,
|
|
plan_id="plan-001",
|
|
phase="pre_execute",
|
|
metadata=meta,
|
|
)
|
|
# Patch os.listdir to raise OSError, simulating a filesystem failure
|
|
# during rollback. This avoids chmod which is ignored when running as root.
|
|
with patch("os.listdir", side_effect=OSError("Simulated I/O error")):
|
|
context.rollback_result = context.checkpoint_manager.rollback_to(cp)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Multi-sandbox delete steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("two sandboxes with checkpoints")
|
|
def step_given_two_sandboxes(context: Any) -> None:
|
|
"""Create checkpoints for two different sandboxes."""
|
|
# First sandbox
|
|
tmpdir1 = tempfile.mkdtemp(prefix="ca-cp-multi1-")
|
|
os.makedirs(os.path.join(tmpdir1, "src"), exist_ok=True)
|
|
with open(os.path.join(tmpdir1, "src", "a.py"), "w") as f:
|
|
f.write("# sandbox1\n")
|
|
sb1 = NoSandbox(resource_id="res-multi-1", original_path=tmpdir1)
|
|
sb1.create(plan_id="plan-multi")
|
|
|
|
tmpdir2 = tempfile.mkdtemp(prefix="ca-cp-multi2-")
|
|
os.makedirs(os.path.join(tmpdir2, "src"), exist_ok=True)
|
|
with open(os.path.join(tmpdir2, "src", "b.py"), "w") as f:
|
|
f.write("# sandbox2\n")
|
|
sb2 = NoSandbox(resource_id="res-multi-2", original_path=tmpdir2)
|
|
sb2.create(plan_id="plan-multi")
|
|
|
|
context.sandbox_one = sb1
|
|
context.sandbox_two = sb2
|
|
context.cp_one = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=sb1, plan_id="plan-multi", phase="pre_execute"
|
|
)
|
|
context.cp_two = context.checkpoint_manager.create_checkpoint(
|
|
sandbox=sb2, plan_id="plan-multi", phase="pre_execute"
|
|
)
|
|
|
|
|
|
@when("I delete the checkpoint from the second sandbox")
|
|
def step_when_delete_second_sandbox_cp(context: Any) -> None:
|
|
context.delete_result = context.checkpoint_manager.delete_checkpoint(
|
|
context.cp_two.checkpoint_id
|
|
)
|
|
|
|
|
|
@then(
|
|
"listing checkpoints for the first sandbox should return"
|
|
" 1 checkpoint for sandbox one"
|
|
)
|
|
def step_then_first_sandbox_still_has_one(context: Any) -> None:
|
|
result = context.checkpoint_manager.list_checkpoints(context.sandbox_one.sandbox_id)
|
|
assert len(result) == 1, f"Expected 1 checkpoint, got {len(result)}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Snapshot copy failure steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a sandbox with an unreadable directory")
|
|
def step_given_unreadable_sandbox(context: Any) -> None:
|
|
tmpdir = tempfile.mkdtemp(prefix="ca-cp-unreadable-")
|
|
os.makedirs(os.path.join(tmpdir, "src"), exist_ok=True)
|
|
with open(os.path.join(tmpdir, "src", "file.txt"), "w") as f:
|
|
f.write("data\n")
|
|
|
|
sandbox = NoSandbox(resource_id="res-cp-unreadable", original_path=tmpdir)
|
|
sandbox.create(plan_id="plan-unreadable")
|
|
context.sandbox = sandbox
|
|
context.sandbox_path = tmpdir
|
|
|
|
|
|
@when("I attempt to create a checkpoint that fails during snapshot")
|
|
def step_when_create_checkpoint_snapshot_fail(context: Any) -> None:
|
|
from unittest.mock import patch
|
|
|
|
from cleveragents.infrastructure.sandbox.protocol import SandboxError
|
|
|
|
# Patch shutil.copytree to raise OSError, simulating a snapshot copy
|
|
# failure. This avoids chmod which is ignored when running as root.
|
|
with patch("shutil.copytree", side_effect=OSError("Simulated copy error")):
|
|
try:
|
|
context.checkpoint_manager.create_checkpoint(
|
|
sandbox=context.sandbox,
|
|
plan_id="plan-unreadable",
|
|
phase="pre_execute",
|
|
)
|
|
context.error = None
|
|
except SandboxError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@then("a sandbox error should be raised")
|
|
def step_then_sandbox_error(context: Any) -> None:
|
|
from cleveragents.infrastructure.sandbox.protocol import SandboxError
|
|
|
|
assert context.error is not None, "Expected a SandboxError"
|
|
assert isinstance(context.error, SandboxError)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup snapshot edge case steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I cleanup a snapshot with an empty path")
|
|
def step_when_cleanup_empty_path(context: Any) -> None:
|
|
context.error = None
|
|
try:
|
|
CheckpointManager._cleanup_snapshot("")
|
|
except Exception as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I cleanup a snapshot with a non-existent parent")
|
|
def step_when_cleanup_nonexistent_parent(context: Any) -> None:
|
|
context.error = None
|
|
try:
|
|
CheckpointManager._cleanup_snapshot("/nonexistent/path/that/does/not/exist")
|
|
except Exception as exc:
|
|
context.error = exc
|
|
|
|
|
|
@then("no error should occur")
|
|
def step_then_no_error(context: Any) -> None:
|
|
assert context.error is None, f"Expected no error, got: {context.error}"
|