freemo
583e6b7ea2
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 54s
CI / unit_tests (pull_request) Successful in 2m39s
CI / integration_tests (pull_request) Successful in 3m10s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m0s
CI / lint (push) Successful in 14s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 34s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m8s
CI / coverage (push) Successful in 5m10s
CI / benchmark-publish (push) Successful in 16m26s
CI / benchmark-regression (pull_request) Successful in 30m12s
Implement spec-mandated Layer 4 Predictive Error Prevention system: - ErrorPattern domain model with pattern text, historical failures, preventive checks, frequency tracking, and keyword-based matching. - ErrorPatternRepository with in-memory CRUD + context-matching query. - ErrorPatternService with record_failure(), match_patterns(), and get_statistics() methods. - Wire into plan execution via plan_lifecycle_service pre-execution hook. - Add error pattern statistics to CLI diagnostics output. Behave BDD: 11 scenarios covering recording, matching, formatting, stats. Robot Framework: 3 integration smoke tests. ASV benchmarks: pattern matching performance. ISSUES CLOSED: #571
131 lines
4.3 KiB
Python
131 lines
4.3 KiB
Python
"""Airspeed Velocity benchmarks for checkpoint and rollback operations.
|
|
|
|
Measures checkpoint creation, listing, rollback, pruning, and metadata
|
|
overhead across varying checkpoint counts.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import ClassVar
|
|
|
|
from cleveragents.application.services.checkpoint_service import CheckpointService
|
|
from cleveragents.domain.models.core.checkpoint import CheckpointRetentionPolicy
|
|
|
|
_PLAN_ID = "01ARZ3NDEKTSV4RRFFQ69G5FAV"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Checkpoint creation benchmarks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class CheckpointCreateSuite:
|
|
"""Benchmark checkpoint creation at varying counts."""
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare a fresh service instance."""
|
|
self.svc = CheckpointService()
|
|
|
|
def time_create_single(self) -> None:
|
|
"""Time creating a single checkpoint."""
|
|
self.svc.create_checkpoint(_PLAN_ID, "commit-abc")
|
|
|
|
def time_create_ten(self) -> None:
|
|
"""Time creating 10 checkpoints."""
|
|
svc = CheckpointService()
|
|
for i in range(10):
|
|
svc.create_checkpoint(_PLAN_ID, f"commit-{i}")
|
|
|
|
def time_create_hundred(self) -> None:
|
|
"""Time creating 100 checkpoints."""
|
|
svc = CheckpointService()
|
|
for i in range(100):
|
|
svc.create_checkpoint(_PLAN_ID, f"commit-{i}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Checkpoint listing benchmarks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class CheckpointListSuite:
|
|
"""Benchmark listing checkpoints at varying plan sizes."""
|
|
|
|
params: ClassVar[list[int]] = [10, 50, 100]
|
|
param_names: ClassVar[list[str]] = ["checkpoint_count"]
|
|
|
|
def setup(self, checkpoint_count: int) -> None:
|
|
"""Populate service with checkpoints."""
|
|
self.svc = CheckpointService()
|
|
for i in range(checkpoint_count):
|
|
self.svc.create_checkpoint(_PLAN_ID, f"commit-{i}")
|
|
|
|
def time_list(self, checkpoint_count: int) -> None:
|
|
"""Time listing all checkpoints for a plan."""
|
|
self.svc.list_checkpoints(_PLAN_ID)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rollback benchmarks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class RollbackSuite:
|
|
"""Benchmark rollback operations."""
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare a service with sandbox and checkpoints."""
|
|
self.svc = CheckpointService()
|
|
self.svc.register_sandbox(_PLAN_ID, "sandbox-path")
|
|
self.cp = self.svc.create_checkpoint(_PLAN_ID, "commit-abc")
|
|
|
|
def time_rollback(self) -> None:
|
|
"""Time a single rollback operation."""
|
|
self.svc.rollback_to_checkpoint(_PLAN_ID, self.cp.checkpoint_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Prune benchmarks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class PruneSuite:
|
|
"""Benchmark checkpoint pruning at varying counts."""
|
|
|
|
params: ClassVar[list[int]] = [20, 50, 100]
|
|
param_names: ClassVar[list[str]] = ["checkpoint_count"]
|
|
|
|
def setup(self, checkpoint_count: int) -> None:
|
|
"""Populate service with checkpoints."""
|
|
self.svc = CheckpointService()
|
|
for i in range(checkpoint_count):
|
|
self.svc.create_checkpoint(_PLAN_ID, f"commit-{i}")
|
|
|
|
def time_prune_to_five(self, checkpoint_count: int) -> None:
|
|
"""Time pruning down to 5 checkpoints."""
|
|
policy = CheckpointRetentionPolicy(max_checkpoints=5, auto_prune=True)
|
|
self.svc.prune_checkpoints(_PLAN_ID, policy)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Metadata benchmarks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class MetadataSuite:
|
|
"""Benchmark checkpoint creation with metadata."""
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare service."""
|
|
self.svc = CheckpointService()
|
|
|
|
def time_create_with_metadata(self) -> None:
|
|
"""Time creating a checkpoint with full metadata."""
|
|
self.svc.create_checkpoint(
|
|
_PLAN_ID,
|
|
"commit-xyz",
|
|
reason="pre-execution snapshot",
|
|
source_tool="lint-check",
|
|
phase="execute",
|
|
)
|