feat(correcting-plans): implement Predictive Error Prevention (Layer 4) with Error Pattern Database
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 54s
CI / unit_tests (pull_request) Successful in 2m39s
CI / integration_tests (pull_request) Successful in 3m10s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m0s
CI / lint (push) Successful in 14s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 34s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m8s
CI / coverage (push) Successful in 5m10s
CI / benchmark-publish (push) Successful in 16m26s
CI / benchmark-regression (pull_request) Successful in 30m12s

Implement spec-mandated Layer 4 Predictive Error Prevention system:

- ErrorPattern domain model with pattern text, historical failures,
  preventive checks, frequency tracking, and keyword-based matching.
- ErrorPatternRepository with in-memory CRUD + context-matching query.
- ErrorPatternService with record_failure(), match_patterns(), and
  get_statistics() methods.
- Wire into plan execution via plan_lifecycle_service pre-execution hook.
- Add error pattern statistics to CLI diagnostics output.

Behave BDD: 11 scenarios covering recording, matching, formatting, stats.
Robot Framework: 3 integration smoke tests.
ASV benchmarks: pattern matching performance.

ISSUES CLOSED: #571
This commit was merged in pull request #643.
This commit is contained in:
2026-03-08 20:26:19 +00:00
parent 61fc70d84b
commit 583e6b7ea2
30 changed files with 1292 additions and 66 deletions
+9 -8
View File
@@ -13,6 +13,7 @@ from __future__ import annotations
import importlib
import sys
from pathlib import Path
from typing import ClassVar
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
@@ -59,8 +60,8 @@ def _make_fragments(n: int, *, unique_nodes: bool = True) -> list[ContextFragmen
class ContentHashDeduplicatorSuite:
"""Benchmark ContentHashDeduplicator throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["fragment_count"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None:
self.dedup = ContentHashDeduplicator()
@@ -85,8 +86,8 @@ class ContentHashDeduplicatorSuite:
class MaxDepthResolverSuite:
"""Benchmark MaxDepthResolver throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["fragment_count"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None:
self.resolver = MaxDepthResolver()
@@ -112,8 +113,8 @@ class MaxDepthResolverSuite:
class WeightedCompositeScorerSuite:
"""Benchmark WeightedCompositeScorer throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["fragment_count"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None:
self.scorer = WeightedCompositeScorer()
@@ -129,8 +130,8 @@ class WeightedCompositeScorerSuite:
class GreedyKnapsackPackerSuite:
"""Benchmark GreedyKnapsackPacker throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["fragment_count"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None:
self.packer = GreedyKnapsackPacker()
+11 -10
View File
@@ -14,6 +14,7 @@ from __future__ import annotations
import importlib
import sys
from pathlib import Path
from typing import ClassVar
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
@@ -71,8 +72,8 @@ def _make_fragments(
class RelevanceCoherenceOrdererSuite:
"""Benchmark RelevanceCoherenceOrderer throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["num_fragments"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None:
self.orderer = RelevanceCoherenceOrderer()
@@ -85,8 +86,8 @@ class RelevanceCoherenceOrdererSuite:
class ProvenancePreambleGeneratorSuite:
"""Benchmark ProvenancePreambleGenerator throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["num_fragments"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None:
self.generator = ProvenancePreambleGenerator()
@@ -99,8 +100,8 @@ class ProvenancePreambleGeneratorSuite:
class ArceStrategySuite:
"""Benchmark ArceStrategy throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["num_fragments"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None:
self.strategy = ArceStrategy(max_iterations=3)
@@ -114,8 +115,8 @@ class ArceStrategySuite:
class TemporalArchaeologyStrategySuite:
"""Benchmark TemporalArchaeologyStrategy throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["num_fragments"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None:
self.strategy = TemporalArchaeologyStrategy()
@@ -129,8 +130,8 @@ class TemporalArchaeologyStrategySuite:
class PlanDecisionContextStrategySuite:
"""Benchmark PlanDecisionContextStrategy throughput."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["num_fragments"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None:
self.strategy = PlanDecisionContextStrategy()
+3 -3
View File
@@ -133,7 +133,7 @@ class AsyncJobTransitionSuite:
phase="execute",
status=AsyncJobStatus.QUEUED,
)
job.is_terminal
_ = job.is_terminal
# ---------------------------------------------------------------------------
@@ -270,12 +270,12 @@ class CancellationTokenSuite:
def time_check_not_cancelled(self) -> None:
token = CancellationToken()
token.is_cancelled
_ = token.is_cancelled
def time_cancel_and_check(self) -> None:
token = CancellationToken()
token.cancel()
token.is_cancelled
_ = token.is_cancelled
def time_reset_token(self) -> None:
token = CancellationToken()
+6 -4
View File
@@ -6,6 +6,8 @@ overhead across varying checkpoint counts.
from __future__ import annotations
from typing import ClassVar
from cleveragents.application.services.checkpoint_service import CheckpointService
from cleveragents.domain.models.core.checkpoint import CheckpointRetentionPolicy
@@ -49,8 +51,8 @@ class CheckpointCreateSuite:
class CheckpointListSuite:
"""Benchmark listing checkpoints at varying plan sizes."""
params: list[int] = [10, 50, 100]
param_names: list[str] = ["checkpoint_count"]
params: ClassVar[list[int]] = [10, 50, 100]
param_names: ClassVar[list[str]] = ["checkpoint_count"]
def setup(self, checkpoint_count: int) -> None:
"""Populate service with checkpoints."""
@@ -90,8 +92,8 @@ class RollbackSuite:
class PruneSuite:
"""Benchmark checkpoint pruning at varying counts."""
params: list[int] = [20, 50, 100]
param_names: list[str] = ["checkpoint_count"]
params: ClassVar[list[int]] = [20, 50, 100]
param_names: ClassVar[list[str]] = ["checkpoint_count"]
def setup(self, checkpoint_count: int) -> None:
"""Populate service with checkpoints."""
-1
View File
@@ -30,7 +30,6 @@ from cleveragents.application.services.component_resolver import ( # noqa: E402
ComponentResolver,
)
# ---------------------------------------------------------------------------
# Test protocol and implementations
# ---------------------------------------------------------------------------
+9 -9
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import importlib
import sys
from pathlib import Path
from typing import ClassVar
# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
@@ -29,7 +30,6 @@ from cleveragents.domain.models.core.correction import ( # noqa: E402
CorrectionMode,
)
# ---------------------------------------------------------------------------
# Mock implementations (minimal, for benchmarking)
# ---------------------------------------------------------------------------
@@ -82,8 +82,8 @@ def _make_service(
class CascadeCancelSuite:
"""Benchmark cascade with cancel-only (not-started) child plans."""
params = [1, 5, 10, 50, 100]
param_names = ["child_plan_count"]
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None:
self.svc, self.plan_ids = _make_service(
@@ -100,8 +100,8 @@ class CascadeCancelSuite:
class CascadeRollbackSuite:
"""Benchmark cascade with cancel+rollback (in-progress) child plans."""
params = [1, 5, 10, 50, 100]
param_names = ["child_plan_count"]
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None:
self.svc, self.plan_ids = _make_service(
@@ -115,8 +115,8 @@ class CascadeRollbackSuite:
class CascadeRejectionSuite:
"""Benchmark cascade rejection (applied child plans)."""
params = [1, 5, 10, 50, 100]
param_names = ["child_plan_count"]
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None:
self.svc, self.plan_ids = _make_service(
@@ -130,8 +130,8 @@ class CascadeRejectionSuite:
class CorrectionWithCascadeSuite:
"""Benchmark full execute_correction_with_cascade flow."""
params = [0, 5, 20, 50]
param_names = ["child_plan_count"]
params: ClassVar[list[int]] = [0, 5, 20, 50]
param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None:
plan_ids = [f"CP{i}" for i in range(child_plan_count)]
+1 -1
View File
@@ -17,7 +17,7 @@ try:
from cleveragents.application.services.decision_service import DecisionService
from cleveragents.config.settings import Settings
from cleveragents.domain.models.core.action import Action, ActionState
from cleveragents.domain.models.core.decision import Decision, DecisionType
from cleveragents.domain.models.core.decision import DecisionType
from cleveragents.domain.models.core.plan import (
NamespacedName,
Plan,
+3 -5
View File
@@ -13,11 +13,9 @@ import json
import sys
import tempfile
from pathlib import Path
from typing import ClassVar
try:
from cleveragents.resource.handlers.devcontainer import (
DevcontainerHandler,
)
from cleveragents.resource.handlers.discovery import (
DevcontainerDiscoveryResult,
discover_devcontainers,
@@ -48,8 +46,8 @@ class TimeDiscoveryThroughput:
"""Benchmark auto-discovery throughput on projects with many subdirs."""
timeout = 60
params: list[int] = [10, 50, 100]
param_names: list[str] = ["num_subdirs"]
params: ClassVar[list[int]] = [10, 50, 100]
param_names: ClassVar[list[str]] = ["num_subdirs"]
def setup(self, num_subdirs: int) -> None:
"""Create a temp directory with many subdirectories."""
+2 -3
View File
@@ -9,6 +9,7 @@ Measures the performance of:
from __future__ import annotations
import contextlib
import importlib
import sys
from pathlib import Path
@@ -81,10 +82,8 @@ class ContainerValidationSuite:
self.resolver.validate_container_available(self.container_types)
def time_validate_without_container_exception(self) -> None:
try:
with contextlib.suppress(ContainerUnavailableError):
self.resolver.validate_container_available(self.no_container_types)
except ContainerUnavailableError:
pass
def time_resolve_and_validate_host(self) -> None:
self.resolver.resolve_and_validate(
+15 -10
View File
@@ -12,6 +12,7 @@ from __future__ import annotations
import importlib
import sys
from pathlib import Path
from typing import ClassVar
# Ensure the local source tree is importable
_SRC = str(Path(__file__).resolve().parents[1] / "src")
@@ -27,8 +28,12 @@ from cleveragents.actor.reconciliation import ( # noqa: E402
ScopeInvariants,
reconcile_invariants,
)
from cleveragents.application.services.decision_service import DecisionService # noqa: E402
from cleveragents.application.services.invariant_service import InvariantService # noqa: E402
from cleveragents.application.services.decision_service import ( # noqa: E402
DecisionService,
)
from cleveragents.application.services.invariant_service import ( # noqa: E402
InvariantService,
)
from cleveragents.domain.models.core.invariant import ( # noqa: E402
Invariant,
InvariantScope,
@@ -86,8 +91,8 @@ def _build_conflicting_scope_invariants(n_conflicts: int) -> ScopeInvariants:
class ReconciliationAlgorithmSuite:
"""Benchmark the pure reconciliation algorithm (no I/O)."""
params = [1, 10, 50, 100]
param_names = ["invariants_per_scope"]
params: ClassVar[list[int]] = [1, 10, 50, 100]
param_names: ClassVar[list[str]] = ["invariants_per_scope"]
def setup(self, n: int) -> None:
self._scope_invs = _build_scope_invariants(
@@ -102,8 +107,8 @@ class ReconciliationAlgorithmSuite:
class ConflictResolutionSuite:
"""Benchmark conflict detection and resolution."""
params = [1, 10, 50, 100]
param_names = ["conflict_count"]
params: ClassVar[list[int]] = [1, 10, 50, 100]
param_names: ClassVar[list[str]] = ["conflict_count"]
def setup(self, n: int) -> None:
self._scope_invs = _build_conflicting_scope_invariants(n)
@@ -116,8 +121,8 @@ class ConflictResolutionSuite:
class FullPipelineSuite:
"""Benchmark the full actor pipeline (collect + reconcile + record)."""
params = [1, 10, 50]
param_names = ["invariants_per_scope"]
params: ClassVar[list[int]] = [1, 10, 50]
param_names: ClassVar[list[str]] = ["invariants_per_scope"]
def setup(self, n: int) -> None:
self._n = n
@@ -153,8 +158,8 @@ class FullPipelineSuite:
class DecisionRecordingSuite:
"""Benchmark decision recording throughput."""
params = [1, 10, 50, 100]
param_names = ["invariant_count"]
params: ClassVar[list[int]] = [1, 10, 50, 100]
param_names: ClassVar[list[str]] = ["invariant_count"]
def setup(self, n: int) -> None:
self._n = n
-1
View File
@@ -9,7 +9,6 @@ from __future__ import annotations
import importlib
import sys
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock
_SRC = str(Path(__file__).resolve().parents[1] / "src")
+3 -2
View File
@@ -7,6 +7,7 @@ Measures the overhead of permission checks in both local mode
from __future__ import annotations
import os
from typing import ClassVar
from cleveragents.application.services.permission_service import (
PermissionService,
@@ -65,8 +66,8 @@ class TimePermissionCheckLocal:
class TimePermissionCheckServer:
"""Benchmark permission checks in server mode."""
params: list[int] = [0, 10, 100]
param_names: list[str] = ["num_bindings"]
params: ClassVar[list[int]] = [0, 10, 100]
param_names: ClassVar[list[str]] = ["num_bindings"]
def setup(self, num_bindings: int) -> None:
os.environ[_SERVER_MODE_ENV] = "true"
+5 -4
View File
@@ -11,6 +11,7 @@ from __future__ import annotations
import sys
from pathlib import Path
from types import SimpleNamespace
from typing import ClassVar
try:
from cleveragents.application.services.correction_service import CorrectionService
@@ -59,8 +60,8 @@ def _build_tree_from_decisions(
class TreeBuildingSuite:
"""Benchmark decision tree adjacency-list construction."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["chain_depth"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["chain_depth"]
def setup(self, chain_depth: int) -> None:
"""Prepare decision chains."""
@@ -74,8 +75,8 @@ class TreeBuildingSuite:
class AnalyzeWithTreeSuite:
"""Benchmark analyze_impact when decision_tree is pre-built."""
params: list[int] = [10, 100, 500]
param_names: list[str] = ["chain_depth"]
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["chain_depth"]
def setup(self, chain_depth: int) -> None:
"""Prepare CorrectionService and tree."""
@@ -0,0 +1,158 @@
"""ASV benchmarks for Layer 4 — Predictive Error Prevention.
Measures pattern matching performance with large databases,
recording throughput, and guidance generation latency.
"""
from __future__ import annotations
from cleveragents.application.services.error_pattern_service import (
ErrorPatternService,
)
from cleveragents.infrastructure.database.error_pattern_repository import (
ErrorPatternRepository,
)
def _make_service(num_patterns: int = 0) -> ErrorPatternService:
"""Create a service and optionally seed it with patterns."""
repo = ErrorPatternRepository()
service = ErrorPatternService(repository=repo)
for i in range(num_patterns):
service.record_failure(
pattern_text=f"Error pattern number {i} in module_{i % 10}",
failure_description=f"Failure description {i}",
preventive_checks=(f"Check {i}-a", f"Check {i}-b"),
keywords=(f"pattern{i}", f"module_{i % 10}", "error"),
)
return service
class TimeRecordSingle:
"""Benchmark recording a single failure pattern."""
timeout = 30
def setup(self) -> None:
self.service = _make_service()
def time_record_single_failure(self) -> None:
self.service.record_failure(
pattern_text="Benchmark failure pattern",
failure_description="Test failure",
preventive_checks=("Check A", "Check B"),
keywords=("benchmark", "failure"),
)
class TimeRecordBatch:
"""Benchmark recording a batch of failure patterns."""
timeout = 60
def setup(self) -> None:
self.service = _make_service()
def time_record_100_failures(self) -> None:
for i in range(100):
self.service.record_failure(
pattern_text=f"Batch failure pattern {i}",
failure_description=f"Failure {i}",
preventive_checks=(f"Check {i}",),
keywords=(f"batch{i}",),
)
class TimeMatchSmallDB:
"""Benchmark pattern matching against a small database (10 patterns)."""
timeout = 30
def setup(self) -> None:
self.service = _make_service(num_patterns=10)
def time_match_context_small(self) -> None:
self.service.match_patterns("We need to fix error pattern number 5 in module_5")
class TimeMatchMediumDB:
"""Benchmark pattern matching against a medium database (100 patterns)."""
timeout = 60
def setup(self) -> None:
self.service = _make_service(num_patterns=100)
def time_match_context_medium(self) -> None:
self.service.match_patterns(
"We need to fix error pattern number 50 in module_5"
)
class TimeMatchLargeDB:
"""Benchmark pattern matching against a large database (1000 patterns)."""
timeout = 120
def setup(self) -> None:
self.service = _make_service(num_patterns=1000)
def time_match_context_large(self) -> None:
self.service.match_patterns(
"We need to fix error pattern number 500 in module_5"
)
def time_match_no_results_large(self) -> None:
self.service.match_patterns(
"Completely unrelated context about cooking recipes"
)
class TimeGuidanceFormatting:
"""Benchmark guidance formatting after matching."""
timeout = 30
def setup(self) -> None:
self.service = _make_service(num_patterns=50)
self.guidance = self.service.match_patterns("We need to fix error in module_5")
def time_format_guidance(self) -> None:
self.guidance.format_for_context()
class TimeStatistics:
"""Benchmark statistics computation."""
timeout = 30
def setup(self) -> None:
self.service = _make_service(num_patterns=500)
def time_get_statistics(self) -> None:
self.service.get_statistics()
def time_list_patterns(self) -> None:
self.service.list_patterns()
class TimeDuplicateRecording:
"""Benchmark recording duplicate patterns (frequency updates)."""
timeout = 60
def setup(self) -> None:
self.service = _make_service()
self.service.record_failure(
pattern_text="Recurring failure",
failure_description="Initial occurrence",
preventive_checks=("Check A",),
keywords=("recurring",),
)
def time_record_duplicate_100_times(self) -> None:
for i in range(100):
self.service.record_failure(
pattern_text="Recurring failure",
failure_description=f"Occurrence {i + 2}",
)
-2
View File
@@ -19,7 +19,6 @@ try:
)
from cleveragents.domain.models.core.automation_profile import (
BUILTIN_PROFILES,
AutomationProfile,
)
from cleveragents.domain.models.core.escalation import (
ConfidenceFactors,
@@ -33,7 +32,6 @@ except ModuleNotFoundError:
)
from cleveragents.domain.models.core.automation_profile import (
BUILTIN_PROFILES,
AutomationProfile,
)
from cleveragents.domain.models.core.escalation import (
ConfidenceFactors,
+2 -2
View File
@@ -16,7 +16,7 @@ _SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.application.services.semantic_validation_rules import (
from cleveragents.application.services.semantic_validation_rules import ( # noqa: E402
APIMisuseRule,
BrokenReferenceRule,
DuplicateImportRule,
@@ -26,7 +26,7 @@ from cleveragents.application.services.semantic_validation_rules import (
SemanticValidationSeverity,
SyntaxCheckRule,
)
from cleveragents.application.services.semantic_validation_service import (
from cleveragents.application.services.semantic_validation_service import ( # noqa: E402
SemanticValidationCache,
SemanticValidationService,
create_default_registry,
@@ -24,8 +24,14 @@ importlib.reload(cleveragents)
from cleveragents.domain.models.acms.crp import ( # noqa: E402
AssembledContext as CRPAssembledContext,
)
from cleveragents.domain.models.acms.crp import ( # noqa: E402
ContextBudget as CRPContextBudget,
)
from cleveragents.domain.models.acms.crp import ( # noqa: E402
ContextFragment as CRPContextFragment,
)
from cleveragents.domain.models.acms.crp import ( # noqa: E402
FragmentProvenance as CRPFragmentProvenance,
)
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402