feat(correcting-plans): implement Predictive Error Prevention (Layer 4) with Error Pattern Database
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 54s
CI / unit_tests (pull_request) Successful in 2m39s
CI / integration_tests (pull_request) Successful in 3m10s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m0s
CI / lint (push) Successful in 14s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 34s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m8s
CI / coverage (push) Successful in 5m10s
CI / benchmark-publish (push) Successful in 16m26s
CI / benchmark-regression (pull_request) Successful in 30m12s

Implement spec-mandated Layer 4 Predictive Error Prevention system:

- ErrorPattern domain model with pattern text, historical failures,
  preventive checks, frequency tracking, and keyword-based matching.
- ErrorPatternRepository with in-memory CRUD + context-matching query.
- ErrorPatternService with record_failure(), match_patterns(), and
  get_statistics() methods.
- Wire into plan execution via plan_lifecycle_service pre-execution hook.
- Add error pattern statistics to CLI diagnostics output.

Behave BDD: 11 scenarios covering recording, matching, formatting, stats.
Robot Framework: 3 integration smoke tests.
ASV benchmarks: pattern matching performance.

ISSUES CLOSED: #571
This commit was merged in pull request #643.
This commit is contained in:
2026-03-08 20:26:19 +00:00
parent 61fc70d84b
commit 583e6b7ea2
30 changed files with 1292 additions and 66 deletions
+1 -1
View File
@@ -8,8 +8,8 @@ Create Date: 2026-03-03 00:00:00
from collections.abc import Sequence from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision: str = "m6_003_async_jobs_table" revision: str = "m6_003_async_jobs_table"
+9 -8
View File
@@ -13,6 +13,7 @@ from __future__ import annotations
import importlib import importlib
import sys import sys
from pathlib import Path from pathlib import Path
from typing import ClassVar
_SRC = str(Path(__file__).resolve().parents[1] / "src") _SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path: if _SRC not in sys.path:
@@ -59,8 +60,8 @@ def _make_fragments(n: int, *, unique_nodes: bool = True) -> list[ContextFragmen
class ContentHashDeduplicatorSuite: class ContentHashDeduplicatorSuite:
"""Benchmark ContentHashDeduplicator throughput.""" """Benchmark ContentHashDeduplicator throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["fragment_count"] param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.dedup = ContentHashDeduplicator() self.dedup = ContentHashDeduplicator()
@@ -85,8 +86,8 @@ class ContentHashDeduplicatorSuite:
class MaxDepthResolverSuite: class MaxDepthResolverSuite:
"""Benchmark MaxDepthResolver throughput.""" """Benchmark MaxDepthResolver throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["fragment_count"] param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.resolver = MaxDepthResolver() self.resolver = MaxDepthResolver()
@@ -112,8 +113,8 @@ class MaxDepthResolverSuite:
class WeightedCompositeScorerSuite: class WeightedCompositeScorerSuite:
"""Benchmark WeightedCompositeScorer throughput.""" """Benchmark WeightedCompositeScorer throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["fragment_count"] param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.scorer = WeightedCompositeScorer() self.scorer = WeightedCompositeScorer()
@@ -129,8 +130,8 @@ class WeightedCompositeScorerSuite:
class GreedyKnapsackPackerSuite: class GreedyKnapsackPackerSuite:
"""Benchmark GreedyKnapsackPacker throughput.""" """Benchmark GreedyKnapsackPacker throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["fragment_count"] param_names: ClassVar[list[str]] = ["fragment_count"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.packer = GreedyKnapsackPacker() self.packer = GreedyKnapsackPacker()
+11 -10
View File
@@ -14,6 +14,7 @@ from __future__ import annotations
import importlib import importlib
import sys import sys
from pathlib import Path from pathlib import Path
from typing import ClassVar
_SRC = str(Path(__file__).resolve().parents[1] / "src") _SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path: if _SRC not in sys.path:
@@ -71,8 +72,8 @@ def _make_fragments(
class RelevanceCoherenceOrdererSuite: class RelevanceCoherenceOrdererSuite:
"""Benchmark RelevanceCoherenceOrderer throughput.""" """Benchmark RelevanceCoherenceOrderer throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["num_fragments"] param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.orderer = RelevanceCoherenceOrderer() self.orderer = RelevanceCoherenceOrderer()
@@ -85,8 +86,8 @@ class RelevanceCoherenceOrdererSuite:
class ProvenancePreambleGeneratorSuite: class ProvenancePreambleGeneratorSuite:
"""Benchmark ProvenancePreambleGenerator throughput.""" """Benchmark ProvenancePreambleGenerator throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["num_fragments"] param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.generator = ProvenancePreambleGenerator() self.generator = ProvenancePreambleGenerator()
@@ -99,8 +100,8 @@ class ProvenancePreambleGeneratorSuite:
class ArceStrategySuite: class ArceStrategySuite:
"""Benchmark ArceStrategy throughput.""" """Benchmark ArceStrategy throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["num_fragments"] param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.strategy = ArceStrategy(max_iterations=3) self.strategy = ArceStrategy(max_iterations=3)
@@ -114,8 +115,8 @@ class ArceStrategySuite:
class TemporalArchaeologyStrategySuite: class TemporalArchaeologyStrategySuite:
"""Benchmark TemporalArchaeologyStrategy throughput.""" """Benchmark TemporalArchaeologyStrategy throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["num_fragments"] param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.strategy = TemporalArchaeologyStrategy() self.strategy = TemporalArchaeologyStrategy()
@@ -129,8 +130,8 @@ class TemporalArchaeologyStrategySuite:
class PlanDecisionContextStrategySuite: class PlanDecisionContextStrategySuite:
"""Benchmark PlanDecisionContextStrategy throughput.""" """Benchmark PlanDecisionContextStrategy throughput."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["num_fragments"] param_names: ClassVar[list[str]] = ["num_fragments"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self.strategy = PlanDecisionContextStrategy() self.strategy = PlanDecisionContextStrategy()
+3 -3
View File
@@ -133,7 +133,7 @@ class AsyncJobTransitionSuite:
phase="execute", phase="execute",
status=AsyncJobStatus.QUEUED, status=AsyncJobStatus.QUEUED,
) )
job.is_terminal _ = job.is_terminal
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -270,12 +270,12 @@ class CancellationTokenSuite:
def time_check_not_cancelled(self) -> None: def time_check_not_cancelled(self) -> None:
token = CancellationToken() token = CancellationToken()
token.is_cancelled _ = token.is_cancelled
def time_cancel_and_check(self) -> None: def time_cancel_and_check(self) -> None:
token = CancellationToken() token = CancellationToken()
token.cancel() token.cancel()
token.is_cancelled _ = token.is_cancelled
def time_reset_token(self) -> None: def time_reset_token(self) -> None:
token = CancellationToken() token = CancellationToken()
+6 -4
View File
@@ -6,6 +6,8 @@ overhead across varying checkpoint counts.
from __future__ import annotations from __future__ import annotations
from typing import ClassVar
from cleveragents.application.services.checkpoint_service import CheckpointService from cleveragents.application.services.checkpoint_service import CheckpointService
from cleveragents.domain.models.core.checkpoint import CheckpointRetentionPolicy from cleveragents.domain.models.core.checkpoint import CheckpointRetentionPolicy
@@ -49,8 +51,8 @@ class CheckpointCreateSuite:
class CheckpointListSuite: class CheckpointListSuite:
"""Benchmark listing checkpoints at varying plan sizes.""" """Benchmark listing checkpoints at varying plan sizes."""
params: list[int] = [10, 50, 100] params: ClassVar[list[int]] = [10, 50, 100]
param_names: list[str] = ["checkpoint_count"] param_names: ClassVar[list[str]] = ["checkpoint_count"]
def setup(self, checkpoint_count: int) -> None: def setup(self, checkpoint_count: int) -> None:
"""Populate service with checkpoints.""" """Populate service with checkpoints."""
@@ -90,8 +92,8 @@ class RollbackSuite:
class PruneSuite: class PruneSuite:
"""Benchmark checkpoint pruning at varying counts.""" """Benchmark checkpoint pruning at varying counts."""
params: list[int] = [20, 50, 100] params: ClassVar[list[int]] = [20, 50, 100]
param_names: list[str] = ["checkpoint_count"] param_names: ClassVar[list[str]] = ["checkpoint_count"]
def setup(self, checkpoint_count: int) -> None: def setup(self, checkpoint_count: int) -> None:
"""Populate service with checkpoints.""" """Populate service with checkpoints."""
-1
View File
@@ -30,7 +30,6 @@ from cleveragents.application.services.component_resolver import ( # noqa: E402
ComponentResolver, ComponentResolver,
) )
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Test protocol and implementations # Test protocol and implementations
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+9 -9
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import importlib import importlib
import sys import sys
from pathlib import Path from pathlib import Path
from typing import ClassVar
# Ensure the local *source* tree is importable even when ASV has an # Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed. # older build of the package installed.
@@ -29,7 +30,6 @@ from cleveragents.domain.models.core.correction import ( # noqa: E402
CorrectionMode, CorrectionMode,
) )
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Mock implementations (minimal, for benchmarking) # Mock implementations (minimal, for benchmarking)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -82,8 +82,8 @@ def _make_service(
class CascadeCancelSuite: class CascadeCancelSuite:
"""Benchmark cascade with cancel-only (not-started) child plans.""" """Benchmark cascade with cancel-only (not-started) child plans."""
params = [1, 5, 10, 50, 100] params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
param_names = ["child_plan_count"] param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None: def setup(self, child_plan_count: int) -> None:
self.svc, self.plan_ids = _make_service( self.svc, self.plan_ids = _make_service(
@@ -100,8 +100,8 @@ class CascadeCancelSuite:
class CascadeRollbackSuite: class CascadeRollbackSuite:
"""Benchmark cascade with cancel+rollback (in-progress) child plans.""" """Benchmark cascade with cancel+rollback (in-progress) child plans."""
params = [1, 5, 10, 50, 100] params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
param_names = ["child_plan_count"] param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None: def setup(self, child_plan_count: int) -> None:
self.svc, self.plan_ids = _make_service( self.svc, self.plan_ids = _make_service(
@@ -115,8 +115,8 @@ class CascadeRollbackSuite:
class CascadeRejectionSuite: class CascadeRejectionSuite:
"""Benchmark cascade rejection (applied child plans).""" """Benchmark cascade rejection (applied child plans)."""
params = [1, 5, 10, 50, 100] params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
param_names = ["child_plan_count"] param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None: def setup(self, child_plan_count: int) -> None:
self.svc, self.plan_ids = _make_service( self.svc, self.plan_ids = _make_service(
@@ -130,8 +130,8 @@ class CascadeRejectionSuite:
class CorrectionWithCascadeSuite: class CorrectionWithCascadeSuite:
"""Benchmark full execute_correction_with_cascade flow.""" """Benchmark full execute_correction_with_cascade flow."""
params = [0, 5, 20, 50] params: ClassVar[list[int]] = [0, 5, 20, 50]
param_names = ["child_plan_count"] param_names: ClassVar[list[str]] = ["child_plan_count"]
def setup(self, child_plan_count: int) -> None: def setup(self, child_plan_count: int) -> None:
plan_ids = [f"CP{i}" for i in range(child_plan_count)] plan_ids = [f"CP{i}" for i in range(child_plan_count)]
+1 -1
View File
@@ -17,7 +17,7 @@ try:
from cleveragents.application.services.decision_service import DecisionService from cleveragents.application.services.decision_service import DecisionService
from cleveragents.config.settings import Settings from cleveragents.config.settings import Settings
from cleveragents.domain.models.core.action import Action, ActionState from cleveragents.domain.models.core.action import Action, ActionState
from cleveragents.domain.models.core.decision import Decision, DecisionType from cleveragents.domain.models.core.decision import DecisionType
from cleveragents.domain.models.core.plan import ( from cleveragents.domain.models.core.plan import (
NamespacedName, NamespacedName,
Plan, Plan,
+3 -5
View File
@@ -13,11 +13,9 @@ import json
import sys import sys
import tempfile import tempfile
from pathlib import Path from pathlib import Path
from typing import ClassVar
try: try:
from cleveragents.resource.handlers.devcontainer import (
DevcontainerHandler,
)
from cleveragents.resource.handlers.discovery import ( from cleveragents.resource.handlers.discovery import (
DevcontainerDiscoveryResult, DevcontainerDiscoveryResult,
discover_devcontainers, discover_devcontainers,
@@ -48,8 +46,8 @@ class TimeDiscoveryThroughput:
"""Benchmark auto-discovery throughput on projects with many subdirs.""" """Benchmark auto-discovery throughput on projects with many subdirs."""
timeout = 60 timeout = 60
params: list[int] = [10, 50, 100] params: ClassVar[list[int]] = [10, 50, 100]
param_names: list[str] = ["num_subdirs"] param_names: ClassVar[list[str]] = ["num_subdirs"]
def setup(self, num_subdirs: int) -> None: def setup(self, num_subdirs: int) -> None:
"""Create a temp directory with many subdirectories.""" """Create a temp directory with many subdirectories."""
+2 -3
View File
@@ -9,6 +9,7 @@ Measures the performance of:
from __future__ import annotations from __future__ import annotations
import contextlib
import importlib import importlib
import sys import sys
from pathlib import Path from pathlib import Path
@@ -81,10 +82,8 @@ class ContainerValidationSuite:
self.resolver.validate_container_available(self.container_types) self.resolver.validate_container_available(self.container_types)
def time_validate_without_container_exception(self) -> None: def time_validate_without_container_exception(self) -> None:
try: with contextlib.suppress(ContainerUnavailableError):
self.resolver.validate_container_available(self.no_container_types) self.resolver.validate_container_available(self.no_container_types)
except ContainerUnavailableError:
pass
def time_resolve_and_validate_host(self) -> None: def time_resolve_and_validate_host(self) -> None:
self.resolver.resolve_and_validate( self.resolver.resolve_and_validate(
+15 -10
View File
@@ -12,6 +12,7 @@ from __future__ import annotations
import importlib import importlib
import sys import sys
from pathlib import Path from pathlib import Path
from typing import ClassVar
# Ensure the local source tree is importable # Ensure the local source tree is importable
_SRC = str(Path(__file__).resolve().parents[1] / "src") _SRC = str(Path(__file__).resolve().parents[1] / "src")
@@ -27,8 +28,12 @@ from cleveragents.actor.reconciliation import ( # noqa: E402
ScopeInvariants, ScopeInvariants,
reconcile_invariants, reconcile_invariants,
) )
from cleveragents.application.services.decision_service import DecisionService # noqa: E402 from cleveragents.application.services.decision_service import ( # noqa: E402
from cleveragents.application.services.invariant_service import InvariantService # noqa: E402 DecisionService,
)
from cleveragents.application.services.invariant_service import ( # noqa: E402
InvariantService,
)
from cleveragents.domain.models.core.invariant import ( # noqa: E402 from cleveragents.domain.models.core.invariant import ( # noqa: E402
Invariant, Invariant,
InvariantScope, InvariantScope,
@@ -86,8 +91,8 @@ def _build_conflicting_scope_invariants(n_conflicts: int) -> ScopeInvariants:
class ReconciliationAlgorithmSuite: class ReconciliationAlgorithmSuite:
"""Benchmark the pure reconciliation algorithm (no I/O).""" """Benchmark the pure reconciliation algorithm (no I/O)."""
params = [1, 10, 50, 100] params: ClassVar[list[int]] = [1, 10, 50, 100]
param_names = ["invariants_per_scope"] param_names: ClassVar[list[str]] = ["invariants_per_scope"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self._scope_invs = _build_scope_invariants( self._scope_invs = _build_scope_invariants(
@@ -102,8 +107,8 @@ class ReconciliationAlgorithmSuite:
class ConflictResolutionSuite: class ConflictResolutionSuite:
"""Benchmark conflict detection and resolution.""" """Benchmark conflict detection and resolution."""
params = [1, 10, 50, 100] params: ClassVar[list[int]] = [1, 10, 50, 100]
param_names = ["conflict_count"] param_names: ClassVar[list[str]] = ["conflict_count"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self._scope_invs = _build_conflicting_scope_invariants(n) self._scope_invs = _build_conflicting_scope_invariants(n)
@@ -116,8 +121,8 @@ class ConflictResolutionSuite:
class FullPipelineSuite: class FullPipelineSuite:
"""Benchmark the full actor pipeline (collect + reconcile + record).""" """Benchmark the full actor pipeline (collect + reconcile + record)."""
params = [1, 10, 50] params: ClassVar[list[int]] = [1, 10, 50]
param_names = ["invariants_per_scope"] param_names: ClassVar[list[str]] = ["invariants_per_scope"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self._n = n self._n = n
@@ -153,8 +158,8 @@ class FullPipelineSuite:
class DecisionRecordingSuite: class DecisionRecordingSuite:
"""Benchmark decision recording throughput.""" """Benchmark decision recording throughput."""
params = [1, 10, 50, 100] params: ClassVar[list[int]] = [1, 10, 50, 100]
param_names = ["invariant_count"] param_names: ClassVar[list[str]] = ["invariant_count"]
def setup(self, n: int) -> None: def setup(self, n: int) -> None:
self._n = n self._n = n
-1
View File
@@ -9,7 +9,6 @@ from __future__ import annotations
import importlib import importlib
import sys import sys
from pathlib import Path from pathlib import Path
from typing import Any
from unittest.mock import MagicMock from unittest.mock import MagicMock
_SRC = str(Path(__file__).resolve().parents[1] / "src") _SRC = str(Path(__file__).resolve().parents[1] / "src")
+3 -2
View File
@@ -7,6 +7,7 @@ Measures the overhead of permission checks in both local mode
from __future__ import annotations from __future__ import annotations
import os import os
from typing import ClassVar
from cleveragents.application.services.permission_service import ( from cleveragents.application.services.permission_service import (
PermissionService, PermissionService,
@@ -65,8 +66,8 @@ class TimePermissionCheckLocal:
class TimePermissionCheckServer: class TimePermissionCheckServer:
"""Benchmark permission checks in server mode.""" """Benchmark permission checks in server mode."""
params: list[int] = [0, 10, 100] params: ClassVar[list[int]] = [0, 10, 100]
param_names: list[str] = ["num_bindings"] param_names: ClassVar[list[str]] = ["num_bindings"]
def setup(self, num_bindings: int) -> None: def setup(self, num_bindings: int) -> None:
os.environ[_SERVER_MODE_ENV] = "true" os.environ[_SERVER_MODE_ENV] = "true"
+5 -4
View File
@@ -11,6 +11,7 @@ from __future__ import annotations
import sys import sys
from pathlib import Path from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
from typing import ClassVar
try: try:
from cleveragents.application.services.correction_service import CorrectionService from cleveragents.application.services.correction_service import CorrectionService
@@ -59,8 +60,8 @@ def _build_tree_from_decisions(
class TreeBuildingSuite: class TreeBuildingSuite:
"""Benchmark decision tree adjacency-list construction.""" """Benchmark decision tree adjacency-list construction."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["chain_depth"] param_names: ClassVar[list[str]] = ["chain_depth"]
def setup(self, chain_depth: int) -> None: def setup(self, chain_depth: int) -> None:
"""Prepare decision chains.""" """Prepare decision chains."""
@@ -74,8 +75,8 @@ class TreeBuildingSuite:
class AnalyzeWithTreeSuite: class AnalyzeWithTreeSuite:
"""Benchmark analyze_impact when decision_tree is pre-built.""" """Benchmark analyze_impact when decision_tree is pre-built."""
params: list[int] = [10, 100, 500] params: ClassVar[list[int]] = [10, 100, 500]
param_names: list[str] = ["chain_depth"] param_names: ClassVar[list[str]] = ["chain_depth"]
def setup(self, chain_depth: int) -> None: def setup(self, chain_depth: int) -> None:
"""Prepare CorrectionService and tree.""" """Prepare CorrectionService and tree."""
@@ -0,0 +1,158 @@
"""ASV benchmarks for Layer 4 — Predictive Error Prevention.
Measures pattern matching performance with large databases,
recording throughput, and guidance generation latency.
"""
from __future__ import annotations
from cleveragents.application.services.error_pattern_service import (
ErrorPatternService,
)
from cleveragents.infrastructure.database.error_pattern_repository import (
ErrorPatternRepository,
)
def _make_service(num_patterns: int = 0) -> ErrorPatternService:
"""Create a service and optionally seed it with patterns."""
repo = ErrorPatternRepository()
service = ErrorPatternService(repository=repo)
for i in range(num_patterns):
service.record_failure(
pattern_text=f"Error pattern number {i} in module_{i % 10}",
failure_description=f"Failure description {i}",
preventive_checks=(f"Check {i}-a", f"Check {i}-b"),
keywords=(f"pattern{i}", f"module_{i % 10}", "error"),
)
return service
class TimeRecordSingle:
"""Benchmark recording a single failure pattern."""
timeout = 30
def setup(self) -> None:
self.service = _make_service()
def time_record_single_failure(self) -> None:
self.service.record_failure(
pattern_text="Benchmark failure pattern",
failure_description="Test failure",
preventive_checks=("Check A", "Check B"),
keywords=("benchmark", "failure"),
)
class TimeRecordBatch:
"""Benchmark recording a batch of failure patterns."""
timeout = 60
def setup(self) -> None:
self.service = _make_service()
def time_record_100_failures(self) -> None:
for i in range(100):
self.service.record_failure(
pattern_text=f"Batch failure pattern {i}",
failure_description=f"Failure {i}",
preventive_checks=(f"Check {i}",),
keywords=(f"batch{i}",),
)
class TimeMatchSmallDB:
"""Benchmark pattern matching against a small database (10 patterns)."""
timeout = 30
def setup(self) -> None:
self.service = _make_service(num_patterns=10)
def time_match_context_small(self) -> None:
self.service.match_patterns("We need to fix error pattern number 5 in module_5")
class TimeMatchMediumDB:
"""Benchmark pattern matching against a medium database (100 patterns)."""
timeout = 60
def setup(self) -> None:
self.service = _make_service(num_patterns=100)
def time_match_context_medium(self) -> None:
self.service.match_patterns(
"We need to fix error pattern number 50 in module_5"
)
class TimeMatchLargeDB:
"""Benchmark pattern matching against a large database (1000 patterns)."""
timeout = 120
def setup(self) -> None:
self.service = _make_service(num_patterns=1000)
def time_match_context_large(self) -> None:
self.service.match_patterns(
"We need to fix error pattern number 500 in module_5"
)
def time_match_no_results_large(self) -> None:
self.service.match_patterns(
"Completely unrelated context about cooking recipes"
)
class TimeGuidanceFormatting:
"""Benchmark guidance formatting after matching."""
timeout = 30
def setup(self) -> None:
self.service = _make_service(num_patterns=50)
self.guidance = self.service.match_patterns("We need to fix error in module_5")
def time_format_guidance(self) -> None:
self.guidance.format_for_context()
class TimeStatistics:
"""Benchmark statistics computation."""
timeout = 30
def setup(self) -> None:
self.service = _make_service(num_patterns=500)
def time_get_statistics(self) -> None:
self.service.get_statistics()
def time_list_patterns(self) -> None:
self.service.list_patterns()
class TimeDuplicateRecording:
"""Benchmark recording duplicate patterns (frequency updates)."""
timeout = 60
def setup(self) -> None:
self.service = _make_service()
self.service.record_failure(
pattern_text="Recurring failure",
failure_description="Initial occurrence",
preventive_checks=("Check A",),
keywords=("recurring",),
)
def time_record_duplicate_100_times(self) -> None:
for i in range(100):
self.service.record_failure(
pattern_text="Recurring failure",
failure_description=f"Occurrence {i + 2}",
)
-2
View File
@@ -19,7 +19,6 @@ try:
) )
from cleveragents.domain.models.core.automation_profile import ( from cleveragents.domain.models.core.automation_profile import (
BUILTIN_PROFILES, BUILTIN_PROFILES,
AutomationProfile,
) )
from cleveragents.domain.models.core.escalation import ( from cleveragents.domain.models.core.escalation import (
ConfidenceFactors, ConfidenceFactors,
@@ -33,7 +32,6 @@ except ModuleNotFoundError:
) )
from cleveragents.domain.models.core.automation_profile import ( from cleveragents.domain.models.core.automation_profile import (
BUILTIN_PROFILES, BUILTIN_PROFILES,
AutomationProfile,
) )
from cleveragents.domain.models.core.escalation import ( from cleveragents.domain.models.core.escalation import (
ConfidenceFactors, ConfidenceFactors,
+2 -2
View File
@@ -16,7 +16,7 @@ _SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path: if _SRC not in sys.path:
sys.path.insert(0, _SRC) sys.path.insert(0, _SRC)
from cleveragents.application.services.semantic_validation_rules import ( from cleveragents.application.services.semantic_validation_rules import ( # noqa: E402
APIMisuseRule, APIMisuseRule,
BrokenReferenceRule, BrokenReferenceRule,
DuplicateImportRule, DuplicateImportRule,
@@ -26,7 +26,7 @@ from cleveragents.application.services.semantic_validation_rules import (
SemanticValidationSeverity, SemanticValidationSeverity,
SyntaxCheckRule, SyntaxCheckRule,
) )
from cleveragents.application.services.semantic_validation_service import ( from cleveragents.application.services.semantic_validation_service import ( # noqa: E402
SemanticValidationCache, SemanticValidationCache,
SemanticValidationService, SemanticValidationService,
create_default_registry, create_default_registry,
@@ -24,8 +24,14 @@ importlib.reload(cleveragents)
from cleveragents.domain.models.acms.crp import ( # noqa: E402 from cleveragents.domain.models.acms.crp import ( # noqa: E402
AssembledContext as CRPAssembledContext, AssembledContext as CRPAssembledContext,
)
from cleveragents.domain.models.acms.crp import ( # noqa: E402
ContextBudget as CRPContextBudget, ContextBudget as CRPContextBudget,
)
from cleveragents.domain.models.acms.crp import ( # noqa: E402
ContextFragment as CRPContextFragment, ContextFragment as CRPContextFragment,
)
from cleveragents.domain.models.acms.crp import ( # noqa: E402
FragmentProvenance as CRPFragmentProvenance, FragmentProvenance as CRPFragmentProvenance,
) )
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402 from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
@@ -0,0 +1,76 @@
Feature: Predictive Error Prevention — Layer 4 Error Pattern Database
As a plan executor
I want historical failure patterns to be consulted before execution
So that preventive guidance prevents recurring errors
Scenario: pep-record Recording a new failure pattern creates an entry
Given a fresh error pattern service
When I record a failure with pattern "Race condition in payment module" and failure "Payment confirmation timeout"
Then the error pattern database should contain 1 pattern
And the pattern should have frequency 1
Scenario: pep-record Recording the same pattern again increments frequency
Given a fresh error pattern service
When I record a failure with pattern "Race condition in payment module" and failure "Payment confirmation timeout"
And I record a failure with pattern "Race condition in payment module" and failure "Idempotency key collision"
Then the error pattern database should contain 1 pattern
And the pattern should have frequency 2
And the pattern should have 2 historical failures
Scenario: pep-match Matching context text against known patterns returns guidance
Given a fresh error pattern service
And a recorded pattern "Async conversion" with preventive check "Add explicit transaction boundaries"
When I match context "We need to perform async conversion in the payment module"
Then preventive guidance should contain "Add explicit transaction boundaries"
Scenario: pep-match No matching patterns returns empty guidance
Given a fresh error pattern service
And a recorded pattern "Async conversion" with preventive check "Add transaction boundaries"
When I match context "Simple refactoring of the UI component"
Then preventive guidance should be empty
Scenario: pep-format Preventive guidance formats for actor context
Given a fresh error pattern service
And a recorded pattern "Memory leak" with preventive check "Check resource cleanup"
And a recorded pattern "Memory leak" with preventive check "Verify dispose patterns"
When I match context "Fix the memory leak in the cache service"
Then the formatted guidance should contain "PREVENTIVE GUIDANCE"
And the formatted guidance should contain "Check resource cleanup"
Scenario: pep-stats Statistics report pattern counts
Given a fresh error pattern service
And a recorded pattern "Race condition" with preventive check "Add locks"
And a recorded pattern "Memory leak" with preventive check "Check cleanup"
When I request error pattern statistics
Then the statistics should show 2 total patterns
Scenario: pep-extract Keywords are automatically extracted from pattern text
Given a fresh error pattern service
When I record a failure with pattern "Race condition in payment processing module" and failure "Timeout"
Then the recorded pattern should have keywords including "race"
And the recorded pattern should have keywords including "payment"
Scenario: pep-multi Multiple patterns match and checks are deduplicated
Given a fresh error pattern service
And a recorded pattern "async conversion" with preventive check "Add transaction boundaries" and keywords "async,conversion"
And a recorded pattern "async timeout" with preventive check "Add retry logic" and keywords "async,timeout"
When I match context "Handle async conversion with proper timeout handling"
Then preventive guidance should contain "Add transaction boundaries"
And preventive guidance should contain "Add retry logic"
Scenario: pep-repo-get Repository get returns pattern by ID
Given a fresh error pattern service
When I record a failure with pattern "Test pattern" and failure "Test failure"
Then the recorded pattern can be retrieved by its ID
Scenario: pep-repo-delete Repository delete removes a pattern
Given a fresh error pattern service
When I record a failure with pattern "Deletable pattern" and failure "Will be deleted"
And I delete the last recorded pattern
Then the error pattern database should contain 0 patterns
Scenario: pep-fallback-match Pattern without keywords matches by description
Given a fresh error pattern service
And a recorded pattern with no keywords for "timeout error" with check "Add timeout handling"
When I match context "We encountered a timeout error in production"
Then preventive guidance should contain "Add timeout handling"
@@ -0,0 +1,172 @@
"""Step definitions for predictive_error_prevention.feature.
Exercises the Error Pattern Database (Layer 4: Predictive Error Prevention)
including recording, matching, formatting, statistics, and keyword extraction.
"""
from __future__ import annotations
from typing import Any
from behave import given, then, when
from cleveragents.application.services.error_pattern_service import (
ErrorPatternService,
)
from cleveragents.domain.models.core.error_pattern import (
ErrorPattern,
PreventiveGuidance,
)
from cleveragents.infrastructure.database.error_pattern_repository import (
ErrorPatternRepository,
)
@given("a fresh error pattern service")
def step_pep_fresh_service(context: Any) -> None:
context.pep_service = ErrorPatternService()
context.pep_last_pattern = None
context.pep_guidance: PreventiveGuidance | None = None
context.pep_statistics: dict[str, int | float] | None = None
@given(
'a recorded pattern "{pattern}" with preventive check "{check}" and keywords "{keywords}"'
)
def step_pep_record_pattern_with_check_and_keywords(
context: Any, pattern: str, check: str, keywords: str
) -> None:
kw_tuple = tuple(k.strip() for k in keywords.split(","))
context.pep_last_pattern = context.pep_service.record_failure(
pattern_text=pattern,
failure_description=f"Historical failure for {pattern}",
preventive_checks=(check,),
keywords=kw_tuple,
)
@given('a recorded pattern "{pattern}" with preventive check "{check}"')
def step_pep_record_pattern_with_check(context: Any, pattern: str, check: str) -> None:
context.pep_last_pattern = context.pep_service.record_failure(
pattern_text=pattern,
failure_description=f"Historical failure for {pattern}",
preventive_checks=(check,),
)
@when('I record a failure with pattern "{pattern}" and failure "{failure}"')
def step_pep_record_failure(context: Any, pattern: str, failure: str) -> None:
context.pep_last_pattern = context.pep_service.record_failure(
pattern_text=pattern,
failure_description=failure,
)
@when('I match context "{text}"')
def step_pep_match_context(context: Any, text: str) -> None:
context.pep_guidance = context.pep_service.match_patterns(text)
@when("I request error pattern statistics")
def step_pep_request_statistics(context: Any) -> None:
context.pep_statistics = context.pep_service.get_statistics()
@then("the error pattern database should contain {count:d} pattern")
def step_pep_assert_pattern_count(context: Any, count: int) -> None:
patterns = context.pep_service.list_patterns()
assert len(patterns) == count, f"Expected {count} patterns, got {len(patterns)}"
@then("the pattern should have frequency {freq:d}")
def step_pep_assert_frequency(context: Any, freq: int) -> None:
patterns = context.pep_service.list_patterns()
assert len(patterns) > 0, "No patterns found"
actual = patterns[0].frequency
assert actual == freq, f"Expected frequency {freq}, got {actual}"
@then("the pattern should have {count:d} historical failures")
def step_pep_assert_historical_failures_count(context: Any, count: int) -> None:
patterns = context.pep_service.list_patterns()
assert len(patterns) > 0, "No patterns found"
actual = len(patterns[0].historical_failures)
assert actual == count, f"Expected {count} historical failures, got {actual}"
@then('preventive guidance should contain "{text}"')
def step_pep_assert_guidance_contains(context: Any, text: str) -> None:
assert context.pep_guidance is not None, "No guidance available"
all_checks = context.pep_guidance.preventive_checks
assert any(text in c for c in all_checks), (
f"Expected guidance to contain '{text}', got {all_checks}"
)
@then("preventive guidance should be empty")
def step_pep_assert_guidance_empty(context: Any) -> None:
assert context.pep_guidance is not None, "No guidance available"
assert not context.pep_guidance.has_guidance, (
f"Expected empty guidance, got {context.pep_guidance.preventive_checks}"
)
@then('the formatted guidance should contain "{text}"')
def step_pep_assert_formatted_contains(context: Any, text: str) -> None:
assert context.pep_guidance is not None, "No guidance available"
formatted = context.pep_guidance.format_for_context()
assert text in formatted, (
f"Expected formatted guidance to contain '{text}', got: {formatted}"
)
@then("the statistics should show {count:d} total patterns")
def step_pep_assert_stats_total(context: Any, count: int) -> None:
assert context.pep_statistics is not None, "No statistics available"
actual = context.pep_statistics["total_patterns"]
assert actual == count, f"Expected {count} total patterns, got {actual}"
@then('the recorded pattern should have keywords including "{keyword}"')
def step_pep_assert_keyword(context: Any, keyword: str) -> None:
assert context.pep_last_pattern is not None, "No pattern recorded"
keywords_lower = [k.lower() for k in context.pep_last_pattern.keywords]
assert keyword.lower() in keywords_lower, (
f"Expected keyword '{keyword}' in {context.pep_last_pattern.keywords}"
)
@then("the recorded pattern can be retrieved by its ID")
def step_pep_retrieve_by_id(context: Any) -> None:
assert context.pep_last_pattern is not None, "No pattern recorded"
repo = ErrorPatternRepository()
repo.create(context.pep_last_pattern)
retrieved = repo.get(context.pep_last_pattern.pattern_id)
assert retrieved is not None, "Pattern not found by ID"
assert retrieved.pattern_id == context.pep_last_pattern.pattern_id
@when("I delete the last recorded pattern")
def step_pep_delete_last_pattern(context: Any) -> None:
assert context.pep_last_pattern is not None, "No pattern recorded"
# Access the internal repo to delete
deleted = context.pep_service._repo.delete(context.pep_last_pattern.pattern_id)
assert deleted, "Delete returned False"
@then("the error pattern database should contain {count:d} patterns")
def step_pep_assert_pattern_count_plural(context: Any, count: int) -> None:
patterns = context.pep_service.list_patterns()
assert len(patterns) == count, f"Expected {count} patterns, got {len(patterns)}"
@given('a recorded pattern with no keywords for "{pattern}" with check "{check}"')
def step_pep_record_pattern_no_keywords(context: Any, pattern: str, check: str) -> None:
ep = ErrorPattern(
pattern=pattern,
historical_failures=("Initial failure",),
preventive_checks=(check,),
keywords=(), # No keywords — forces fallback matching
)
context.pep_service._repo.create(ep)
context.pep_last_pattern = ep
@@ -0,0 +1,236 @@
"""Step definitions for system_diagnostics_coverage.feature.
Covers uncovered paths in cli/commands/system.py:
- _check_file_permissions: writable-but-not-readable (line 344)
- _check_stale_locks: locks table exists paths (lines 367-388)
- _check_async_worker_health: async enabled path (lines 405-417)
- _check_error_patterns: exception path (lines 438-442)
All step patterns use the 'sysdiag_' prefix to avoid AmbiguousStep collisions
with existing steps in cli_core_steps.py and coverage_security_template_boost_steps.py.
"""
from __future__ import annotations
import tempfile
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
__all__: list[str] = []
# ===========================================================================
# Helpers
# ===========================================================================
def _sysdiag_mock_settings(**overrides: Any) -> Any:
"""Create mock settings with sensible defaults for diagnostic tests."""
tmpdir = tempfile.mkdtemp()
s = MagicMock()
s.database_url = overrides.get("database_url", f"sqlite:///{tmpdir}/test.db")
s.data_dir = overrides.get("data_dir", Path(tmpdir))
s.storage_path = overrides.get("storage_path", Path(tmpdir))
s.log_dir = overrides.get("log_dir", Path(tmpdir) / "logs")
s.default_automation_profile = "auto"
s.debug_enabled = False
s.async_enabled = overrides.get("async_enabled", False)
s.async_max_workers = overrides.get("async_max_workers", 4)
s.async_poll_interval = overrides.get("async_poll_interval", 1.0)
s.has_provider_configured = MagicMock(return_value=False)
s.configured_providers = []
return s
# ===========================================================================
# Given steps
# ===========================================================================
@given("a sysdiag test environment")
def step_sysdiag_env(context: Context) -> None:
context.sysdiag_result = {}
context.sysdiag_tmpdir = tempfile.mkdtemp()
@given("a data dir that exists")
def step_sysdiag_data_dir_exists(context: Context) -> None:
data_dir = Path(context.sysdiag_tmpdir) / "data"
data_dir.mkdir(parents=True, exist_ok=True)
context.sysdiag_data_dir = data_dir
# ===========================================================================
# When steps — _check_file_permissions edge cases
# ===========================================================================
@when("I run the file permissions check with read access denied")
def step_sysdiag_perms_no_read(context: Context) -> None:
from cleveragents.cli.commands.system import _check_file_permissions
ms = _sysdiag_mock_settings(data_dir=context.sysdiag_data_dir)
def fake_access(path: Any, mode: int) -> bool:
"""Simulate writable-but-not-readable directory."""
import os as _os
if mode == _os.R_OK:
return False
return mode == _os.W_OK
with (
patch("cleveragents.config.settings.get_settings", return_value=ms),
patch("cleveragents.cli.commands.system.os.access", side_effect=fake_access),
):
context.sysdiag_result = _check_file_permissions()
@when("I run the file permissions check with no access at all")
def step_sysdiag_perms_no_access(context: Context) -> None:
from cleveragents.cli.commands.system import _check_file_permissions
ms = _sysdiag_mock_settings(data_dir=context.sysdiag_data_dir)
with (
patch("cleveragents.config.settings.get_settings", return_value=ms),
patch("cleveragents.cli.commands.system.os.access", return_value=False),
):
context.sysdiag_result = _check_file_permissions()
# ===========================================================================
# When steps — _check_stale_locks edge cases
# ===========================================================================
@when("I run the stale locks check with {count:d} stale locks in the table")
def step_sysdiag_stale_locks_count(context: Context, count: int) -> None:
from cleveragents.cli.commands.system import _check_stale_locks
mock_engine = MagicMock()
mock_inspector = MagicMock()
mock_inspector.get_table_names.return_value = ["locks", "other_table"]
mock_lock_service = MagicMock()
mock_lock_service.count_stale_locks.return_value = count
with (
patch(
"cleveragents.cli.commands.system.get_database_url",
return_value="sqlite:///test.db",
),
patch(
"cleveragents.cli.commands.system.create_engine",
return_value=mock_engine,
),
patch(
"cleveragents.cli.commands.system.sa_inspect",
return_value=mock_inspector,
),
patch(
"cleveragents.cli.commands.system.LockService",
return_value=mock_lock_service,
),
):
context.sysdiag_result = _check_stale_locks()
@when("I run the stale locks check with a database error")
def step_sysdiag_stale_locks_error(context: Context) -> None:
from cleveragents.cli.commands.system import _check_stale_locks
with patch(
"cleveragents.cli.commands.system.get_database_url",
side_effect=RuntimeError("database unreachable"),
):
context.sysdiag_result = _check_stale_locks()
# ===========================================================================
# When steps — _check_async_worker_health edge cases
# ===========================================================================
@when("I run the async worker health check with async enabled")
def step_sysdiag_async_enabled(context: Context) -> None:
from cleveragents.cli.commands.system import _check_async_worker_health
ms = _sysdiag_mock_settings(
async_enabled=True,
async_max_workers=8,
async_poll_interval=2.5,
)
with patch("cleveragents.config.settings.get_settings", return_value=ms):
context.sysdiag_result = _check_async_worker_health()
@when("I run the async worker health check with a settings error")
def step_sysdiag_async_error(context: Context) -> None:
from cleveragents.cli.commands.system import _check_async_worker_health
with patch(
"cleveragents.config.settings.get_settings",
side_effect=RuntimeError("settings unavailable"),
):
context.sysdiag_result = _check_async_worker_health()
# ===========================================================================
# When steps — _check_error_patterns exception path
# ===========================================================================
@when("I run the error patterns check with a service error")
def step_sysdiag_error_patterns_error(context: Context) -> None:
from cleveragents.cli.commands.system import _check_error_patterns
with patch(
"cleveragents.application.services.error_pattern_service.ErrorPatternService",
side_effect=RuntimeError("service init failed"),
):
context.sysdiag_result = _check_error_patterns()
# ===========================================================================
# Then steps
# ===========================================================================
@then('the sysdiag check status should be "{expected}"')
def step_sysdiag_status(context: Context, expected: str) -> None:
from cleveragents.cli.commands.system import CheckStatus
status_map = {
"ok": CheckStatus.OK,
"warn": CheckStatus.WARN,
"error": CheckStatus.ERROR,
}
actual = context.sysdiag_result["status"]
assert actual == status_map[expected], (
f"Expected status '{expected}', got '{actual}'"
)
@then('the sysdiag check details should be "{expected}"')
def step_sysdiag_details_exact(context: Context, expected: str) -> None:
actual = context.sysdiag_result["details"]
assert actual == expected, f"Expected details '{expected}', got '{actual}'"
@then('the sysdiag check details should contain "{text}"')
def step_sysdiag_details_contains(context: Context, text: str) -> None:
details = context.sysdiag_result["details"]
assert text in details, f"Expected '{text}' in details: '{details}'"
@then("the sysdiag check should have a recommendation")
def step_sysdiag_has_recommendation(context: Context) -> None:
rec = context.sysdiag_result.get("recommendation")
assert rec is not None and len(rec) > 0, (
f"Expected a non-empty recommendation, got: {rec!r}"
)
@@ -0,0 +1,74 @@
@coverage_boost
Feature: System diagnostics uncovered code paths
As a developer ensuring thorough test coverage
I want to exercise uncovered diagnostic check paths in system.py
So that stale-lock, async-worker, error-pattern, and file-permission edge cases are tested
# ===========================================================================
# _check_file_permissions — line 344: writable-but-not-readable data dir
# ===========================================================================
Scenario: File permissions check on data dir that is writable but not readable
Given a sysdiag test environment
And a data dir that exists
When I run the file permissions check with read access denied
Then the sysdiag check status should be "warn"
And the sysdiag check details should be "data dir w"
Scenario: File permissions check on data dir that is neither readable nor writable
Given a sysdiag test environment
And a data dir that exists
When I run the file permissions check with no access at all
Then the sysdiag check status should be "error"
And the sysdiag check details should be "data dir no access"
# ===========================================================================
# _check_stale_locks — lines 367-388
# ===========================================================================
Scenario: Stale locks check finds zero stale locks in existing table
Given a sysdiag test environment
When I run the stale locks check with 0 stale locks in the table
Then the sysdiag check status should be "ok"
And the sysdiag check details should be "0 stale locks"
Scenario: Stale locks check finds stale locks present
Given a sysdiag test environment
When I run the stale locks check with 3 stale locks in the table
Then the sysdiag check status should be "warn"
And the sysdiag check details should contain "3 stale lock(s) found"
And the sysdiag check should have a recommendation
Scenario: Stale locks check handles database errors gracefully
Given a sysdiag test environment
When I run the stale locks check with a database error
Then the sysdiag check status should be "warn"
And the sysdiag check details should be "unable to check"
# ===========================================================================
# _check_async_worker_health — lines 405-417
# ===========================================================================
Scenario: Async worker health check when async is enabled
Given a sysdiag test environment
When I run the async worker health check with async enabled
Then the sysdiag check status should be "ok"
And the sysdiag check details should contain "enabled"
And the sysdiag check details should contain "max_workers="
And the sysdiag check details should contain "poll_interval="
Scenario: Async worker health check handles settings errors gracefully
Given a sysdiag test environment
When I run the async worker health check with a settings error
Then the sysdiag check status should be "warn"
And the sysdiag check details should be "unable to check"
# ===========================================================================
# _check_error_patterns — lines 438-442
# ===========================================================================
Scenario: Error pattern check when service initialization fails
Given a sysdiag test environment
When I run the error patterns check with a service error
Then the sysdiag check status should be "ok"
And the sysdiag check details should be "empty (no patterns recorded)"
@@ -0,0 +1,99 @@
"""Helper utilities for Predictive Error Prevention Robot integration tests.
Covers Layer 4: Error Pattern Database recording, matching, and
preventive guidance generation.
"""
from __future__ import annotations
from cleveragents.application.services.error_pattern_service import (
ErrorPatternService,
)
from cleveragents.domain.models.core.error_pattern import ErrorPattern
def _record_and_verify_pattern() -> None:
"""Integration test: record a failure and verify pattern creation."""
service = ErrorPatternService()
pattern = service.record_failure(
pattern_text="Race condition in payment module",
failure_description="Payment confirmation timeout",
preventive_checks=("Add explicit locking",),
keywords=("race", "payment"),
)
assert isinstance(pattern, ErrorPattern), (
f"Expected ErrorPattern, got {type(pattern)}"
)
assert pattern.frequency == 1, f"Expected frequency 1, got {pattern.frequency}"
assert len(pattern.historical_failures) == 1, (
f"Expected 1 historical failure, got {len(pattern.historical_failures)}"
)
assert pattern.pattern == "Race condition in payment module"
stats = service.get_statistics()
assert stats["total_patterns"] == 1, (
f"Expected 1 pattern, got {stats['total_patterns']}"
)
print("record-and-verify-ok")
def _match_patterns_returns_guidance() -> None:
"""Integration test: match context against patterns returns guidance."""
service = ErrorPatternService()
service.record_failure(
pattern_text="Async conversion failure",
failure_description="Timeout during async conversion",
preventive_checks=("Add transaction boundaries", "Use retry logic"),
keywords=("async", "conversion"),
)
guidance = service.match_patterns(
"We need to perform async conversion in the payment module"
)
assert guidance.has_guidance, "Expected guidance to be non-empty"
assert len(guidance.preventive_checks) == 2, (
f"Expected 2 checks, got {len(guidance.preventive_checks)}"
)
assert "Add transaction boundaries" in guidance.preventive_checks
assert "Use retry logic" in guidance.preventive_checks
formatted = guidance.format_for_context()
assert "PREVENTIVE GUIDANCE" in formatted, (
f"Expected header in formatted output: {formatted}"
)
print("match-patterns-guidance-ok")
def _frequency_increments_on_duplicate() -> None:
"""Integration test: duplicate pattern recording increments frequency."""
service = ErrorPatternService()
service.record_failure(
pattern_text="Memory leak in cache",
failure_description="OOM after 24h",
preventive_checks=("Check dispose patterns",),
keywords=("memory", "leak", "cache"),
)
updated = service.record_failure(
pattern_text="Memory leak in cache",
failure_description="High memory usage after 12h",
)
assert updated.frequency == 2, f"Expected frequency 2, got {updated.frequency}"
assert len(updated.historical_failures) == 2, (
f"Expected 2 failures, got {len(updated.historical_failures)}"
)
stats = service.get_statistics()
assert stats["total_patterns"] == 1, (
f"Expected 1 pattern, got {stats['total_patterns']}"
)
assert stats["total_occurrences"] == 2, (
f"Expected 2 occurrences, got {stats['total_occurrences']}"
)
print("frequency-increments-ok")
if __name__ == "__main__":
_record_and_verify_pattern()
_match_patterns_returns_guidance()
_frequency_increments_on_duplicate()
print("all-predictive-error-prevention-tests-passed")
+48
View File
@@ -0,0 +1,48 @@
*** Settings ***
Documentation Layer 4 - Predictive Error Prevention integration smoke tests
Library OperatingSystem
Library Process
*** Variables ***
${ERROR_PATTERN_MODEL} ${CURDIR}/../src/cleveragents/domain/models/core/error_pattern.py
${ERROR_PATTERN_REPO} ${CURDIR}/../src/cleveragents/infrastructure/database/error_pattern_repository.py
${ERROR_PATTERN_SERVICE} ${CURDIR}/../src/cleveragents/application/services/error_pattern_service.py
*** Test Cases ***
Error Pattern Domain Model Exists
[Documentation] Verify the error pattern domain model module is present
File Should Exist ${ERROR_PATTERN_MODEL}
${content}= Get File ${ERROR_PATTERN_MODEL}
Should Contain ${content} class ErrorPattern
Should Contain ${content} class PreventiveGuidance
Should Contain ${content} pattern_id
Should Contain ${content} historical_failures
Should Contain ${content} preventive_checks
Should Contain ${content} frequency
Should Contain ${content} keywords
Should Contain ${content} format_for_context
Error Pattern Repository Module Exists
[Documentation] Verify the error pattern repository module is present
File Should Exist ${ERROR_PATTERN_REPO}
${content}= Get File ${ERROR_PATTERN_REPO}
Should Contain ${content} class ErrorPatternRepository
Should Contain ${content} def create(
Should Contain ${content} def get(
Should Contain ${content} def list_all(
Should Contain ${content} def update(
Should Contain ${content} def delete(
Should Contain ${content} def match_context(
Should Contain ${content} def count(
Should Contain ${content} def statistics(
Error Pattern Service Provides Recording Matching And Statistics
[Documentation] Verify the error pattern service has all required methods
File Should Exist ${ERROR_PATTERN_SERVICE}
${content}= Get File ${ERROR_PATTERN_SERVICE}
Should Contain ${content} class ErrorPatternService
Should Contain ${content} def record_failure(
Should Contain ${content} def match_patterns(
Should Contain ${content} def get_statistics(
Should Contain ${content} def list_patterns(
Should Contain ${content} _extract_keywords
@@ -0,0 +1,151 @@
"""Error Pattern Service — records failures and provides preventive guidance."""
from __future__ import annotations
from datetime import UTC, datetime
from cleveragents.domain.models.core.error_pattern import (
ErrorPattern,
PreventiveGuidance,
)
from cleveragents.infrastructure.database.error_pattern_repository import (
ErrorPatternRepository,
)
class ErrorPatternService:
"""Service layer for the Error Pattern Database.
Implements Layer 4: Predictive Error Prevention.
"""
def __init__(self, repository: ErrorPatternRepository | None = None) -> None:
self._repo = repository or ErrorPatternRepository()
def record_failure(
self,
pattern_text: str,
failure_description: str,
preventive_checks: tuple[str, ...] = (),
keywords: tuple[str, ...] = (),
) -> ErrorPattern:
"""Record a new failure or update frequency of an existing pattern."""
# Check if similar pattern already exists
existing = self._find_existing_pattern(pattern_text)
if existing is not None:
# Merge preventive checks (accumulate, don't replace)
merged_checks = list(existing.preventive_checks)
for check in preventive_checks:
if check not in merged_checks:
merged_checks.append(check)
updated = ErrorPattern(
pattern_id=existing.pattern_id,
pattern=existing.pattern,
historical_failures=(
*existing.historical_failures,
failure_description,
),
preventive_checks=tuple(merged_checks),
frequency=existing.frequency + 1,
created_at=existing.created_at,
last_seen=datetime.now(UTC),
keywords=keywords if keywords else existing.keywords,
)
return self._repo.update(updated)
new_pattern = ErrorPattern(
pattern=pattern_text,
historical_failures=(failure_description,),
preventive_checks=preventive_checks,
keywords=keywords or self._extract_keywords(pattern_text),
)
return self._repo.create(new_pattern)
def match_patterns(self, context_text: str) -> PreventiveGuidance:
"""Match context against known error patterns.
Returns preventive guidance for matched patterns.
"""
matched = self._repo.match_context(context_text)
if not matched:
return PreventiveGuidance()
all_checks: list[str] = []
seen: set[str] = set()
for pattern in matched:
for check in pattern.preventive_checks:
if check not in seen:
all_checks.append(check)
seen.add(check)
return PreventiveGuidance(
matched_patterns=tuple(matched),
preventive_checks=tuple(all_checks),
)
def get_statistics(self) -> dict[str, int | float]:
"""Return error pattern database statistics."""
return self._repo.statistics()
def list_patterns(self) -> list[ErrorPattern]:
"""List all error patterns, most recent first."""
return self._repo.list_all()
def _find_existing_pattern(self, pattern_text: str) -> ErrorPattern | None:
"""Find an existing pattern by exact text match."""
pattern_lower = pattern_text.lower()
for p in self._repo.list_all():
if p.pattern.lower() == pattern_lower:
return p
return None
def _extract_keywords(self, text: str) -> tuple[str, ...]:
"""Extract significant keywords from pattern text for matching."""
stop_words = frozenset(
{
"a",
"an",
"the",
"in",
"on",
"at",
"to",
"for",
"of",
"with",
"and",
"or",
"but",
"is",
"was",
"are",
"were",
"be",
"been",
"has",
"had",
"have",
"do",
"does",
"did",
"will",
"would",
"could",
"should",
"may",
"might",
"can",
"this",
"that",
"it",
"its",
"not",
"no",
"from",
"by",
"as",
}
)
words = text.lower().split()
keywords = [w for w in words if len(w) > 2 and w not in stop_words]
return tuple(keywords[:10]) # Limit to 10 keywords
@@ -90,6 +90,9 @@ from cleveragents.infrastructure.events.types import EventType
if TYPE_CHECKING: if TYPE_CHECKING:
from cleveragents.application.services.async_worker import InMemoryJobStore from cleveragents.application.services.async_worker import InMemoryJobStore
from cleveragents.application.services.decision_service import DecisionService from cleveragents.application.services.decision_service import DecisionService
from cleveragents.application.services.error_pattern_service import (
ErrorPatternService,
)
from cleveragents.config.settings import Settings from cleveragents.config.settings import Settings
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
from cleveragents.infrastructure.events.protocol import EventBus from cleveragents.infrastructure.events.protocol import EventBus
@@ -159,6 +162,7 @@ class PlanLifecycleService:
decision_service: DecisionService | None = None, decision_service: DecisionService | None = None,
event_bus: EventBus | None = None, event_bus: EventBus | None = None,
job_store: InMemoryJobStore | None = None, job_store: InMemoryJobStore | None = None,
error_pattern_service: ErrorPatternService | None = None,
): ):
"""Initialize the plan lifecycle service. """Initialize the plan lifecycle service.
@@ -181,12 +185,19 @@ class PlanLifecycleService:
async jobs when ``settings.async_enabled`` is True. async jobs when ``settings.async_enabled`` is True.
When ``None``, async job creation is silently skipped When ``None``, async job creation is silently skipped
even if async is enabled. even if async is enabled.
error_pattern_service: Optional
:class:`ErrorPatternService` for Layer 4 Predictive
Error Prevention. When provided, the service is
consulted before the Execute phase to inject preventive
guidance. When ``None``, predictive prevention is
silently skipped.
""" """
self.settings = settings self.settings = settings
self.unit_of_work = unit_of_work self.unit_of_work = unit_of_work
self.decision_service = decision_service self.decision_service = decision_service
self.event_bus = event_bus self.event_bus = event_bus
self._job_store = job_store self._job_store = job_store
self.error_pattern_service = error_pattern_service
self._logger = logger.bind(service="plan_lifecycle") self._logger = logger.bind(service="plan_lifecycle")
self.preflight_guardrail = PlanPreflightGuardrail() self.preflight_guardrail = PlanPreflightGuardrail()
@@ -231,6 +242,42 @@ class PlanLifecycleService:
exc_info=True, exc_info=True,
) )
def _consult_error_patterns(self, plan: Plan) -> None:
"""Consult the Error Pattern Database before execution.
When an ``ErrorPatternService`` is configured, the plan's
description is matched against known error patterns. Any
resulting preventive guidance is stored in the plan's
``error_details`` dict under the ``preventive_guidance`` key
so that downstream actors can incorporate it.
Failures are logged but never propagated preventive guidance
must not block lifecycle transitions.
"""
if self.error_pattern_service is None:
return
try:
context_text = plan.description or ""
guidance = self.error_pattern_service.match_patterns(context_text)
if guidance.has_guidance:
formatted = guidance.format_for_context()
existing = dict(plan.error_details) if plan.error_details else {}
existing["preventive_guidance"] = formatted
plan.error_details = existing
self._logger.info(
"Preventive guidance injected",
plan_id=plan.identity.plan_id,
matched_patterns=len(guidance.matched_patterns),
checks=len(guidance.preventive_checks),
)
except Exception:
self._logger.warning(
"error_pattern_consultation_failed",
plan_id=plan.identity.plan_id,
exc_info=True,
)
def _persist_action_create(self, action: Action, ctx: Any) -> None: def _persist_action_create(self, action: Action, ctx: Any) -> None:
"""Persist a new action via the repository. """Persist a new action via the repository.
@@ -914,6 +961,9 @@ class PlanLifecycleService:
plan_id, plan.phase, plan.state or ProcessingState.QUEUED plan_id, plan.phase, plan.state or ProcessingState.QUEUED
) )
# Layer 4: Consult Error Pattern Database for preventive guidance
self._consult_error_patterns(plan)
# Transition to Execute phase # Transition to Execute phase
plan.phase = PlanPhase.EXECUTE plan.phase = PlanPhase.EXECUTE
plan.processing_state = ProcessingState.QUEUED plan.processing_state = ProcessingState.QUEUED
+26
View File
@@ -418,6 +418,31 @@ def _check_async_worker_health() -> dict[str, Any]:
} }
def _check_error_patterns() -> dict[str, Any]:
"""Check the Error Pattern Database status."""
try:
from cleveragents.application.services.error_pattern_service import (
ErrorPatternService,
)
service = ErrorPatternService()
stats = service.get_statistics()
total = stats["total_patterns"]
return {
"name": "Error Pattern DB",
"status": CheckStatus.OK,
"details": (
f"{total} patterns, {stats['total_occurrences']} total occurrences"
),
}
except Exception:
return {
"name": "Error Pattern DB",
"status": CheckStatus.OK,
"details": "empty (no patterns recorded)",
}
def build_diagnostics_data() -> dict[str, Any]: def build_diagnostics_data() -> dict[str, Any]:
"""Run all diagnostic checks and return structured results.""" """Run all diagnostic checks and return structured results."""
start = time.monotonic() start = time.monotonic()
@@ -433,6 +458,7 @@ def build_diagnostics_data() -> dict[str, Any]:
checks.append(_check_stale_locks()) checks.append(_check_stale_locks())
checks.append(_check_async_worker_health()) checks.append(_check_async_worker_health())
checks.append(_check_error_patterns())
elapsed = time.monotonic() - start elapsed = time.monotonic() - start
@@ -108,6 +108,12 @@ from cleveragents.domain.models.core.diff_review import (
ReviewArtifact, ReviewArtifact,
) )
# Error pattern domain models (Layer 4: Predictive Error Prevention)
from cleveragents.domain.models.core.error_pattern import (
ErrorPattern,
PreventiveGuidance,
)
# Error recovery models # Error recovery models
from cleveragents.domain.models.core.error_recovery import ( from cleveragents.domain.models.core.error_recovery import (
ErrorCategory, ErrorCategory,
@@ -356,6 +362,7 @@ __all__ = [
"DoDSummary", "DoDSummary",
"ErrorCategory", "ErrorCategory",
"ErrorHistory", "ErrorHistory",
"ErrorPattern",
"ErrorRecord", "ErrorRecord",
"ErrorRecoveryPolicy", "ErrorRecoveryPolicy",
"EscalationDecision", "EscalationDecision",
@@ -407,6 +414,7 @@ __all__ = [
"PlanResult", "PlanResult",
"PlanStatus", "PlanStatus",
"PlanTimestamps", "PlanTimestamps",
"PreventiveGuidance",
"ProcessingState", "ProcessingState",
"Project", "Project",
"ProjectContextPolicy", "ProjectContextPolicy",
@@ -0,0 +1,55 @@
"""Error Pattern domain model for Predictive Error Prevention (Layer 4)."""
from __future__ import annotations
from datetime import UTC, datetime
from pydantic import BaseModel, Field
from ulid import ULID
class ErrorPattern(BaseModel, frozen=True):
"""A recorded error pattern with failure history and preventive checks."""
pattern_id: str = Field(default_factory=lambda: str(ULID()))
pattern: str = Field(min_length=1, description="Description of the error pattern")
historical_failures: tuple[str, ...] = Field(
default_factory=tuple,
description="List of historical failure descriptions matching this pattern",
)
preventive_checks: tuple[str, ...] = Field(
default_factory=tuple,
description="List of preventive check descriptions to apply",
)
frequency: int = Field(
default=1, ge=1, description="Number of times pattern was observed"
)
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
last_seen: datetime = Field(default_factory=lambda: datetime.now(UTC))
# Keywords extracted from pattern for matching
keywords: tuple[str, ...] = Field(
default_factory=tuple,
description="Keywords for pattern matching against plan context",
)
class PreventiveGuidance(BaseModel, frozen=True):
"""Guidance to inject into actor context before plan execution."""
matched_patterns: tuple[ErrorPattern, ...] = Field(default_factory=tuple)
preventive_checks: tuple[str, ...] = Field(default_factory=tuple)
@property
def has_guidance(self) -> bool:
"""Return True if there are any preventive checks."""
return len(self.preventive_checks) > 0
def format_for_context(self) -> str:
"""Format preventive guidance as text for inclusion in actor context."""
if not self.has_guidance:
return ""
lines: list[str] = ["[PREVENTIVE GUIDANCE — from Error Pattern Database]"]
for i, check in enumerate(self.preventive_checks, 1):
lines.append(f" {i}. {check}")
return "\n".join(lines)
@@ -0,0 +1,63 @@
"""Error Pattern Repository — in-memory store for error patterns."""
from __future__ import annotations
from cleveragents.domain.models.core.error_pattern import ErrorPattern
class ErrorPatternRepository:
"""In-memory CRUD + pattern-matching repository for error patterns."""
def __init__(self) -> None:
self._patterns: dict[str, ErrorPattern] = {}
def create(self, pattern: ErrorPattern) -> ErrorPattern:
"""Create a new error pattern."""
self._patterns[pattern.pattern_id] = pattern
return pattern
def get(self, pattern_id: str) -> ErrorPattern | None:
"""Get a pattern by ID."""
return self._patterns.get(pattern_id)
def list_all(self) -> list[ErrorPattern]:
"""List all patterns, sorted by last_seen descending."""
return sorted(self._patterns.values(), key=lambda p: p.last_seen, reverse=True)
def update(self, pattern: ErrorPattern) -> ErrorPattern:
"""Update an existing pattern."""
self._patterns[pattern.pattern_id] = pattern
return pattern
def delete(self, pattern_id: str) -> bool:
"""Delete a pattern by ID. Returns True if deleted."""
return self._patterns.pop(pattern_id, None) is not None
def match_context(self, context_text: str) -> list[ErrorPattern]:
"""Find patterns whose keywords appear in the context text."""
context_lower = context_text.lower()
matched: list[ErrorPattern] = []
for pattern in self._patterns.values():
if pattern.keywords:
if any(kw.lower() in context_lower for kw in pattern.keywords):
matched.append(pattern)
else:
# Fallback: check if pattern description appears in context
if pattern.pattern.lower() in context_lower:
matched.append(pattern)
return sorted(matched, key=lambda p: p.frequency, reverse=True)
def count(self) -> int:
"""Return the number of patterns stored."""
return len(self._patterns)
def statistics(self) -> dict[str, int | float]:
"""Return aggregate statistics about stored patterns."""
patterns = self.list_all()
total = len(patterns)
total_freq = sum(p.frequency for p in patterns)
return {
"total_patterns": total,
"total_occurrences": total_freq,
"avg_frequency": total_freq / total if total > 0 else 0.0,
}