freemo
0ca1303927
CI / lint (pull_request) Successful in 23s
CI / quality (pull_request) Successful in 29s
CI / security (pull_request) Successful in 54s
CI / typecheck (pull_request) Successful in 59s
CI / benchmark-publish (pull_request) Has been skipped
CI / build (pull_request) Successful in 25s
CI / integration_tests (pull_request) Successful in 4m33s
CI / benchmark-regression (pull_request) Successful in 22m32s
CI / unit_tests (pull_request) Successful in 30m28s
CI / docker (pull_request) Successful in 39s
CI / coverage (pull_request) Successful in 1h39m42s
CI / lint (push) Successful in 12s
CI / quality (push) Successful in 20s
CI / build (push) Successful in 24s
CI / security (push) Successful in 28s
CI / typecheck (push) Successful in 1m0s
CI / benchmark-regression (push) Has been skipped
CI / integration_tests (push) Successful in 4m35s
CI / benchmark-publish (push) Successful in 13m36s
CI / coverage (push) Has been cancelled
CI / unit_tests (push) Has been cancelled
CI / docker (push) Has been cancelled
Introduce a lightweight checkpoint/rollback system for sandbox state during plan execute and apply flows. CheckpointManager snapshots the sandbox working directory before each phase and can restore it on failure, giving the execution engine a reliable undo mechanism. Key changes: - SandboxCheckpoint model, Checkpointable protocol, and CheckpointManager in infrastructure/sandbox/checkpoint.py - PlanExecutor gains optional checkpoint_manager with pre/post execute hooks and automatic rollback on failure - PlanApplyService gains optional checkpoint_manager with pre-apply checkpoint and rollback helper - 12 BDD scenarios (features/sandbox_checkpoints.feature) - 5 Robot Framework smoke tests (robot/sandbox_checkpoint_smoke.robot) - ASV benchmarks for creation, rollback, and listing operations - Reference documentation in docs/reference/sandbox.md ISSUES CLOSED: #183
118 lines
3.8 KiB
Python
118 lines
3.8 KiB
Python
"""ASV benchmarks for sandbox checkpoint infrastructure.
|
|
|
|
Measures the performance of:
|
|
- CheckpointManager.create_checkpoint()
|
|
- CheckpointManager.rollback_to()
|
|
- CheckpointManager.list_checkpoints()
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
# Force-reload so ASV picks up the source tree version.
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.infrastructure.sandbox.checkpoint import ( # noqa: E402
|
|
CheckpointManager,
|
|
)
|
|
from cleveragents.infrastructure.sandbox.no_sandbox import NoSandbox # noqa: E402
|
|
|
|
|
|
class CheckpointCreationSuite:
|
|
"""Benchmark CheckpointManager.create_checkpoint() overhead."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
self.manager = CheckpointManager()
|
|
self.tmpdir = tempfile.mkdtemp(prefix="ca-bench-cp-")
|
|
# Create sample files
|
|
os.makedirs(os.path.join(self.tmpdir, "src"), exist_ok=True)
|
|
with open(os.path.join(self.tmpdir, "src", "main.py"), "w") as f:
|
|
f.write("print('benchmark')\n")
|
|
with open(os.path.join(self.tmpdir, "README.md"), "w") as f:
|
|
f.write("# Benchmark project\n")
|
|
|
|
self.sandbox = NoSandbox(resource_id="res-bench-01", original_path=self.tmpdir)
|
|
self.sandbox.create(plan_id="plan-bench-01")
|
|
|
|
def time_checkpoint_creation(self) -> None:
|
|
"""Benchmark a single checkpoint creation."""
|
|
self.manager.create_checkpoint(
|
|
sandbox=self.sandbox,
|
|
plan_id="plan-bench-01",
|
|
phase="pre_execute",
|
|
metadata={"sandbox_path": self.tmpdir},
|
|
)
|
|
|
|
|
|
class CheckpointRollbackSuite:
|
|
"""Benchmark CheckpointManager.rollback_to() overhead."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
self.manager = CheckpointManager()
|
|
self.tmpdir = tempfile.mkdtemp(prefix="ca-bench-rb-")
|
|
os.makedirs(os.path.join(self.tmpdir, "src"), exist_ok=True)
|
|
with open(os.path.join(self.tmpdir, "src", "main.py"), "w") as f:
|
|
f.write("print('benchmark')\n")
|
|
|
|
self.sandbox = NoSandbox(resource_id="res-bench-02", original_path=self.tmpdir)
|
|
self.sandbox.create(plan_id="plan-bench-02")
|
|
|
|
self.checkpoint = self.manager.create_checkpoint(
|
|
sandbox=self.sandbox,
|
|
plan_id="plan-bench-02",
|
|
phase="pre_execute",
|
|
metadata={"sandbox_path": self.tmpdir},
|
|
)
|
|
|
|
# Modify the file to make rollback meaningful
|
|
with open(os.path.join(self.tmpdir, "src", "main.py"), "w") as f:
|
|
f.write("print('modified for benchmark')\n")
|
|
|
|
def time_checkpoint_rollback(self) -> None:
|
|
"""Benchmark a single rollback operation."""
|
|
self.manager.rollback_to(self.checkpoint)
|
|
|
|
|
|
class CheckpointListingSuite:
|
|
"""Benchmark CheckpointManager.list_checkpoints() overhead."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
self.manager = CheckpointManager()
|
|
self.tmpdir = tempfile.mkdtemp(prefix="ca-bench-ls-")
|
|
|
|
self.sandbox = NoSandbox(resource_id="res-bench-03", original_path=self.tmpdir)
|
|
self.sandbox.create(plan_id="plan-bench-03")
|
|
|
|
# Create several checkpoints
|
|
for phase in ["pre_execute", "post_execute", "pre_apply"]:
|
|
self.manager.create_checkpoint(
|
|
sandbox=self.sandbox,
|
|
plan_id="plan-bench-03",
|
|
phase=phase,
|
|
)
|
|
|
|
self.sandbox_id = self.sandbox.sandbox_id
|
|
|
|
def time_checkpoint_listing(self) -> None:
|
|
"""Benchmark listing checkpoints for a sandbox."""
|
|
self.manager.list_checkpoints(self.sandbox_id)
|