feat(correcting-plans): implement Predictive Error Prevention (Layer 4) with Error Pattern Database
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 54s
CI / unit_tests (pull_request) Successful in 2m39s
CI / integration_tests (pull_request) Successful in 3m10s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m0s
CI / lint (push) Successful in 14s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 34s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m8s
CI / coverage (push) Successful in 5m10s
CI / benchmark-publish (push) Successful in 16m26s
CI / benchmark-regression (pull_request) Successful in 30m12s
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 54s
CI / unit_tests (pull_request) Successful in 2m39s
CI / integration_tests (pull_request) Successful in 3m10s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m0s
CI / lint (push) Successful in 14s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 34s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m8s
CI / coverage (push) Successful in 5m10s
CI / benchmark-publish (push) Successful in 16m26s
CI / benchmark-regression (pull_request) Successful in 30m12s
Implement spec-mandated Layer 4 Predictive Error Prevention system: - ErrorPattern domain model with pattern text, historical failures, preventive checks, frequency tracking, and keyword-based matching. - ErrorPatternRepository with in-memory CRUD + context-matching query. - ErrorPatternService with record_failure(), match_patterns(), and get_statistics() methods. - Wire into plan execution via plan_lifecycle_service pre-execution hook. - Add error pattern statistics to CLI diagnostics output. Behave BDD: 11 scenarios covering recording, matching, formatting, stats. Robot Framework: 3 integration smoke tests. ASV benchmarks: pattern matching performance. ISSUES CLOSED: #571
This commit was merged in pull request #643.
This commit is contained in:
@@ -8,8 +8,8 @@ Create Date: 2026-03-03 00:00:00
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "m6_003_async_jobs_table"
|
||||
|
||||
@@ -13,6 +13,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
@@ -59,8 +60,8 @@ def _make_fragments(n: int, *, unique_nodes: bool = True) -> list[ContextFragmen
|
||||
class ContentHashDeduplicatorSuite:
|
||||
"""Benchmark ContentHashDeduplicator throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.dedup = ContentHashDeduplicator()
|
||||
@@ -85,8 +86,8 @@ class ContentHashDeduplicatorSuite:
|
||||
class MaxDepthResolverSuite:
|
||||
"""Benchmark MaxDepthResolver throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.resolver = MaxDepthResolver()
|
||||
@@ -112,8 +113,8 @@ class MaxDepthResolverSuite:
|
||||
class WeightedCompositeScorerSuite:
|
||||
"""Benchmark WeightedCompositeScorer throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.scorer = WeightedCompositeScorer()
|
||||
@@ -129,8 +130,8 @@ class WeightedCompositeScorerSuite:
|
||||
class GreedyKnapsackPackerSuite:
|
||||
"""Benchmark GreedyKnapsackPacker throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.packer = GreedyKnapsackPacker()
|
||||
|
||||
@@ -14,6 +14,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
@@ -71,8 +72,8 @@ def _make_fragments(
|
||||
class RelevanceCoherenceOrdererSuite:
|
||||
"""Benchmark RelevanceCoherenceOrderer throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.orderer = RelevanceCoherenceOrderer()
|
||||
@@ -85,8 +86,8 @@ class RelevanceCoherenceOrdererSuite:
|
||||
class ProvenancePreambleGeneratorSuite:
|
||||
"""Benchmark ProvenancePreambleGenerator throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.generator = ProvenancePreambleGenerator()
|
||||
@@ -99,8 +100,8 @@ class ProvenancePreambleGeneratorSuite:
|
||||
class ArceStrategySuite:
|
||||
"""Benchmark ArceStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = ArceStrategy(max_iterations=3)
|
||||
@@ -114,8 +115,8 @@ class ArceStrategySuite:
|
||||
class TemporalArchaeologyStrategySuite:
|
||||
"""Benchmark TemporalArchaeologyStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = TemporalArchaeologyStrategy()
|
||||
@@ -129,8 +130,8 @@ class TemporalArchaeologyStrategySuite:
|
||||
class PlanDecisionContextStrategySuite:
|
||||
"""Benchmark PlanDecisionContextStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = PlanDecisionContextStrategy()
|
||||
|
||||
@@ -133,7 +133,7 @@ class AsyncJobTransitionSuite:
|
||||
phase="execute",
|
||||
status=AsyncJobStatus.QUEUED,
|
||||
)
|
||||
job.is_terminal
|
||||
_ = job.is_terminal
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -270,12 +270,12 @@ class CancellationTokenSuite:
|
||||
|
||||
def time_check_not_cancelled(self) -> None:
|
||||
token = CancellationToken()
|
||||
token.is_cancelled
|
||||
_ = token.is_cancelled
|
||||
|
||||
def time_cancel_and_check(self) -> None:
|
||||
token = CancellationToken()
|
||||
token.cancel()
|
||||
token.is_cancelled
|
||||
_ = token.is_cancelled
|
||||
|
||||
def time_reset_token(self) -> None:
|
||||
token = CancellationToken()
|
||||
|
||||
@@ -6,6 +6,8 @@ overhead across varying checkpoint counts.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
from cleveragents.application.services.checkpoint_service import CheckpointService
|
||||
from cleveragents.domain.models.core.checkpoint import CheckpointRetentionPolicy
|
||||
|
||||
@@ -49,8 +51,8 @@ class CheckpointCreateSuite:
|
||||
class CheckpointListSuite:
|
||||
"""Benchmark listing checkpoints at varying plan sizes."""
|
||||
|
||||
params: list[int] = [10, 50, 100]
|
||||
param_names: list[str] = ["checkpoint_count"]
|
||||
params: ClassVar[list[int]] = [10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["checkpoint_count"]
|
||||
|
||||
def setup(self, checkpoint_count: int) -> None:
|
||||
"""Populate service with checkpoints."""
|
||||
@@ -90,8 +92,8 @@ class RollbackSuite:
|
||||
class PruneSuite:
|
||||
"""Benchmark checkpoint pruning at varying counts."""
|
||||
|
||||
params: list[int] = [20, 50, 100]
|
||||
param_names: list[str] = ["checkpoint_count"]
|
||||
params: ClassVar[list[int]] = [20, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["checkpoint_count"]
|
||||
|
||||
def setup(self, checkpoint_count: int) -> None:
|
||||
"""Populate service with checkpoints."""
|
||||
|
||||
@@ -30,7 +30,6 @@ from cleveragents.application.services.component_resolver import ( # noqa: E402
|
||||
ComponentResolver,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test protocol and implementations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
@@ -29,7 +30,6 @@ from cleveragents.domain.models.core.correction import ( # noqa: E402
|
||||
CorrectionMode,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock implementations (minimal, for benchmarking)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -82,8 +82,8 @@ def _make_service(
|
||||
class CascadeCancelSuite:
|
||||
"""Benchmark cascade with cancel-only (not-started) child plans."""
|
||||
|
||||
params = [1, 5, 10, 50, 100]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
self.svc, self.plan_ids = _make_service(
|
||||
@@ -100,8 +100,8 @@ class CascadeCancelSuite:
|
||||
class CascadeRollbackSuite:
|
||||
"""Benchmark cascade with cancel+rollback (in-progress) child plans."""
|
||||
|
||||
params = [1, 5, 10, 50, 100]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
self.svc, self.plan_ids = _make_service(
|
||||
@@ -115,8 +115,8 @@ class CascadeRollbackSuite:
|
||||
class CascadeRejectionSuite:
|
||||
"""Benchmark cascade rejection (applied child plans)."""
|
||||
|
||||
params = [1, 5, 10, 50, 100]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
self.svc, self.plan_ids = _make_service(
|
||||
@@ -130,8 +130,8 @@ class CascadeRejectionSuite:
|
||||
class CorrectionWithCascadeSuite:
|
||||
"""Benchmark full execute_correction_with_cascade flow."""
|
||||
|
||||
params = [0, 5, 20, 50]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [0, 5, 20, 50]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
plan_ids = [f"CP{i}" for i in range(child_plan_count)]
|
||||
|
||||
@@ -17,7 +17,7 @@ try:
|
||||
from cleveragents.application.services.decision_service import DecisionService
|
||||
from cleveragents.config.settings import Settings
|
||||
from cleveragents.domain.models.core.action import Action, ActionState
|
||||
from cleveragents.domain.models.core.decision import Decision, DecisionType
|
||||
from cleveragents.domain.models.core.decision import DecisionType
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
NamespacedName,
|
||||
Plan,
|
||||
|
||||
@@ -13,11 +13,9 @@ import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
try:
|
||||
from cleveragents.resource.handlers.devcontainer import (
|
||||
DevcontainerHandler,
|
||||
)
|
||||
from cleveragents.resource.handlers.discovery import (
|
||||
DevcontainerDiscoveryResult,
|
||||
discover_devcontainers,
|
||||
@@ -48,8 +46,8 @@ class TimeDiscoveryThroughput:
|
||||
"""Benchmark auto-discovery throughput on projects with many subdirs."""
|
||||
|
||||
timeout = 60
|
||||
params: list[int] = [10, 50, 100]
|
||||
param_names: list[str] = ["num_subdirs"]
|
||||
params: ClassVar[list[int]] = [10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["num_subdirs"]
|
||||
|
||||
def setup(self, num_subdirs: int) -> None:
|
||||
"""Create a temp directory with many subdirectories."""
|
||||
|
||||
@@ -9,6 +9,7 @@ Measures the performance of:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -81,10 +82,8 @@ class ContainerValidationSuite:
|
||||
self.resolver.validate_container_available(self.container_types)
|
||||
|
||||
def time_validate_without_container_exception(self) -> None:
|
||||
try:
|
||||
with contextlib.suppress(ContainerUnavailableError):
|
||||
self.resolver.validate_container_available(self.no_container_types)
|
||||
except ContainerUnavailableError:
|
||||
pass
|
||||
|
||||
def time_resolve_and_validate_host(self) -> None:
|
||||
self.resolver.resolve_and_validate(
|
||||
|
||||
@@ -12,6 +12,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
# Ensure the local source tree is importable
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
@@ -27,8 +28,12 @@ from cleveragents.actor.reconciliation import ( # noqa: E402
|
||||
ScopeInvariants,
|
||||
reconcile_invariants,
|
||||
)
|
||||
from cleveragents.application.services.decision_service import DecisionService # noqa: E402
|
||||
from cleveragents.application.services.invariant_service import InvariantService # noqa: E402
|
||||
from cleveragents.application.services.decision_service import ( # noqa: E402
|
||||
DecisionService,
|
||||
)
|
||||
from cleveragents.application.services.invariant_service import ( # noqa: E402
|
||||
InvariantService,
|
||||
)
|
||||
from cleveragents.domain.models.core.invariant import ( # noqa: E402
|
||||
Invariant,
|
||||
InvariantScope,
|
||||
@@ -86,8 +91,8 @@ def _build_conflicting_scope_invariants(n_conflicts: int) -> ScopeInvariants:
|
||||
class ReconciliationAlgorithmSuite:
|
||||
"""Benchmark the pure reconciliation algorithm (no I/O)."""
|
||||
|
||||
params = [1, 10, 50, 100]
|
||||
param_names = ["invariants_per_scope"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["invariants_per_scope"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._scope_invs = _build_scope_invariants(
|
||||
@@ -102,8 +107,8 @@ class ReconciliationAlgorithmSuite:
|
||||
class ConflictResolutionSuite:
|
||||
"""Benchmark conflict detection and resolution."""
|
||||
|
||||
params = [1, 10, 50, 100]
|
||||
param_names = ["conflict_count"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["conflict_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._scope_invs = _build_conflicting_scope_invariants(n)
|
||||
@@ -116,8 +121,8 @@ class ConflictResolutionSuite:
|
||||
class FullPipelineSuite:
|
||||
"""Benchmark the full actor pipeline (collect + reconcile + record)."""
|
||||
|
||||
params = [1, 10, 50]
|
||||
param_names = ["invariants_per_scope"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50]
|
||||
param_names: ClassVar[list[str]] = ["invariants_per_scope"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._n = n
|
||||
@@ -153,8 +158,8 @@ class FullPipelineSuite:
|
||||
class DecisionRecordingSuite:
|
||||
"""Benchmark decision recording throughput."""
|
||||
|
||||
params = [1, 10, 50, 100]
|
||||
param_names = ["invariant_count"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["invariant_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._n = n
|
||||
|
||||
@@ -9,7 +9,6 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
|
||||
@@ -7,6 +7,7 @@ Measures the overhead of permission checks in both local mode
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import ClassVar
|
||||
|
||||
from cleveragents.application.services.permission_service import (
|
||||
PermissionService,
|
||||
@@ -65,8 +66,8 @@ class TimePermissionCheckLocal:
|
||||
class TimePermissionCheckServer:
|
||||
"""Benchmark permission checks in server mode."""
|
||||
|
||||
params: list[int] = [0, 10, 100]
|
||||
param_names: list[str] = ["num_bindings"]
|
||||
params: ClassVar[list[int]] = [0, 10, 100]
|
||||
param_names: ClassVar[list[str]] = ["num_bindings"]
|
||||
|
||||
def setup(self, num_bindings: int) -> None:
|
||||
os.environ[_SERVER_MODE_ENV] = "true"
|
||||
|
||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import ClassVar
|
||||
|
||||
try:
|
||||
from cleveragents.application.services.correction_service import CorrectionService
|
||||
@@ -59,8 +60,8 @@ def _build_tree_from_decisions(
|
||||
class TreeBuildingSuite:
|
||||
"""Benchmark decision tree adjacency-list construction."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["chain_depth"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["chain_depth"]
|
||||
|
||||
def setup(self, chain_depth: int) -> None:
|
||||
"""Prepare decision chains."""
|
||||
@@ -74,8 +75,8 @@ class TreeBuildingSuite:
|
||||
class AnalyzeWithTreeSuite:
|
||||
"""Benchmark analyze_impact when decision_tree is pre-built."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["chain_depth"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["chain_depth"]
|
||||
|
||||
def setup(self, chain_depth: int) -> None:
|
||||
"""Prepare CorrectionService and tree."""
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""ASV benchmarks for Layer 4 — Predictive Error Prevention.
|
||||
|
||||
Measures pattern matching performance with large databases,
|
||||
recording throughput, and guidance generation latency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cleveragents.application.services.error_pattern_service import (
|
||||
ErrorPatternService,
|
||||
)
|
||||
from cleveragents.infrastructure.database.error_pattern_repository import (
|
||||
ErrorPatternRepository,
|
||||
)
|
||||
|
||||
|
||||
def _make_service(num_patterns: int = 0) -> ErrorPatternService:
|
||||
"""Create a service and optionally seed it with patterns."""
|
||||
repo = ErrorPatternRepository()
|
||||
service = ErrorPatternService(repository=repo)
|
||||
for i in range(num_patterns):
|
||||
service.record_failure(
|
||||
pattern_text=f"Error pattern number {i} in module_{i % 10}",
|
||||
failure_description=f"Failure description {i}",
|
||||
preventive_checks=(f"Check {i}-a", f"Check {i}-b"),
|
||||
keywords=(f"pattern{i}", f"module_{i % 10}", "error"),
|
||||
)
|
||||
return service
|
||||
|
||||
|
||||
class TimeRecordSingle:
|
||||
"""Benchmark recording a single failure pattern."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service()
|
||||
|
||||
def time_record_single_failure(self) -> None:
|
||||
self.service.record_failure(
|
||||
pattern_text="Benchmark failure pattern",
|
||||
failure_description="Test failure",
|
||||
preventive_checks=("Check A", "Check B"),
|
||||
keywords=("benchmark", "failure"),
|
||||
)
|
||||
|
||||
|
||||
class TimeRecordBatch:
|
||||
"""Benchmark recording a batch of failure patterns."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service()
|
||||
|
||||
def time_record_100_failures(self) -> None:
|
||||
for i in range(100):
|
||||
self.service.record_failure(
|
||||
pattern_text=f"Batch failure pattern {i}",
|
||||
failure_description=f"Failure {i}",
|
||||
preventive_checks=(f"Check {i}",),
|
||||
keywords=(f"batch{i}",),
|
||||
)
|
||||
|
||||
|
||||
class TimeMatchSmallDB:
|
||||
"""Benchmark pattern matching against a small database (10 patterns)."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=10)
|
||||
|
||||
def time_match_context_small(self) -> None:
|
||||
self.service.match_patterns("We need to fix error pattern number 5 in module_5")
|
||||
|
||||
|
||||
class TimeMatchMediumDB:
|
||||
"""Benchmark pattern matching against a medium database (100 patterns)."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=100)
|
||||
|
||||
def time_match_context_medium(self) -> None:
|
||||
self.service.match_patterns(
|
||||
"We need to fix error pattern number 50 in module_5"
|
||||
)
|
||||
|
||||
|
||||
class TimeMatchLargeDB:
|
||||
"""Benchmark pattern matching against a large database (1000 patterns)."""
|
||||
|
||||
timeout = 120
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=1000)
|
||||
|
||||
def time_match_context_large(self) -> None:
|
||||
self.service.match_patterns(
|
||||
"We need to fix error pattern number 500 in module_5"
|
||||
)
|
||||
|
||||
def time_match_no_results_large(self) -> None:
|
||||
self.service.match_patterns(
|
||||
"Completely unrelated context about cooking recipes"
|
||||
)
|
||||
|
||||
|
||||
class TimeGuidanceFormatting:
|
||||
"""Benchmark guidance formatting after matching."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=50)
|
||||
self.guidance = self.service.match_patterns("We need to fix error in module_5")
|
||||
|
||||
def time_format_guidance(self) -> None:
|
||||
self.guidance.format_for_context()
|
||||
|
||||
|
||||
class TimeStatistics:
|
||||
"""Benchmark statistics computation."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=500)
|
||||
|
||||
def time_get_statistics(self) -> None:
|
||||
self.service.get_statistics()
|
||||
|
||||
def time_list_patterns(self) -> None:
|
||||
self.service.list_patterns()
|
||||
|
||||
|
||||
class TimeDuplicateRecording:
|
||||
"""Benchmark recording duplicate patterns (frequency updates)."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service()
|
||||
self.service.record_failure(
|
||||
pattern_text="Recurring failure",
|
||||
failure_description="Initial occurrence",
|
||||
preventive_checks=("Check A",),
|
||||
keywords=("recurring",),
|
||||
)
|
||||
|
||||
def time_record_duplicate_100_times(self) -> None:
|
||||
for i in range(100):
|
||||
self.service.record_failure(
|
||||
pattern_text="Recurring failure",
|
||||
failure_description=f"Occurrence {i + 2}",
|
||||
)
|
||||
@@ -19,7 +19,6 @@ try:
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_profile import (
|
||||
BUILTIN_PROFILES,
|
||||
AutomationProfile,
|
||||
)
|
||||
from cleveragents.domain.models.core.escalation import (
|
||||
ConfidenceFactors,
|
||||
@@ -33,7 +32,6 @@ except ModuleNotFoundError:
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_profile import (
|
||||
BUILTIN_PROFILES,
|
||||
AutomationProfile,
|
||||
)
|
||||
from cleveragents.domain.models.core.escalation import (
|
||||
ConfidenceFactors,
|
||||
|
||||
@@ -16,7 +16,7 @@ _SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from cleveragents.application.services.semantic_validation_rules import (
|
||||
from cleveragents.application.services.semantic_validation_rules import ( # noqa: E402
|
||||
APIMisuseRule,
|
||||
BrokenReferenceRule,
|
||||
DuplicateImportRule,
|
||||
@@ -26,7 +26,7 @@ from cleveragents.application.services.semantic_validation_rules import (
|
||||
SemanticValidationSeverity,
|
||||
SyntaxCheckRule,
|
||||
)
|
||||
from cleveragents.application.services.semantic_validation_service import (
|
||||
from cleveragents.application.services.semantic_validation_service import ( # noqa: E402
|
||||
SemanticValidationCache,
|
||||
SemanticValidationService,
|
||||
create_default_registry,
|
||||
|
||||
@@ -24,8 +24,14 @@ importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
AssembledContext as CRPAssembledContext,
|
||||
)
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
ContextBudget as CRPContextBudget,
|
||||
)
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
ContextFragment as CRPContextFragment,
|
||||
)
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
FragmentProvenance as CRPFragmentProvenance,
|
||||
)
|
||||
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
Feature: Predictive Error Prevention — Layer 4 Error Pattern Database
|
||||
As a plan executor
|
||||
I want historical failure patterns to be consulted before execution
|
||||
So that preventive guidance prevents recurring errors
|
||||
|
||||
Scenario: pep-record Recording a new failure pattern creates an entry
|
||||
Given a fresh error pattern service
|
||||
When I record a failure with pattern "Race condition in payment module" and failure "Payment confirmation timeout"
|
||||
Then the error pattern database should contain 1 pattern
|
||||
And the pattern should have frequency 1
|
||||
|
||||
Scenario: pep-record Recording the same pattern again increments frequency
|
||||
Given a fresh error pattern service
|
||||
When I record a failure with pattern "Race condition in payment module" and failure "Payment confirmation timeout"
|
||||
And I record a failure with pattern "Race condition in payment module" and failure "Idempotency key collision"
|
||||
Then the error pattern database should contain 1 pattern
|
||||
And the pattern should have frequency 2
|
||||
And the pattern should have 2 historical failures
|
||||
|
||||
Scenario: pep-match Matching context text against known patterns returns guidance
|
||||
Given a fresh error pattern service
|
||||
And a recorded pattern "Async conversion" with preventive check "Add explicit transaction boundaries"
|
||||
When I match context "We need to perform async conversion in the payment module"
|
||||
Then preventive guidance should contain "Add explicit transaction boundaries"
|
||||
|
||||
Scenario: pep-match No matching patterns returns empty guidance
|
||||
Given a fresh error pattern service
|
||||
And a recorded pattern "Async conversion" with preventive check "Add transaction boundaries"
|
||||
When I match context "Simple refactoring of the UI component"
|
||||
Then preventive guidance should be empty
|
||||
|
||||
Scenario: pep-format Preventive guidance formats for actor context
|
||||
Given a fresh error pattern service
|
||||
And a recorded pattern "Memory leak" with preventive check "Check resource cleanup"
|
||||
And a recorded pattern "Memory leak" with preventive check "Verify dispose patterns"
|
||||
When I match context "Fix the memory leak in the cache service"
|
||||
Then the formatted guidance should contain "PREVENTIVE GUIDANCE"
|
||||
And the formatted guidance should contain "Check resource cleanup"
|
||||
|
||||
Scenario: pep-stats Statistics report pattern counts
|
||||
Given a fresh error pattern service
|
||||
And a recorded pattern "Race condition" with preventive check "Add locks"
|
||||
And a recorded pattern "Memory leak" with preventive check "Check cleanup"
|
||||
When I request error pattern statistics
|
||||
Then the statistics should show 2 total patterns
|
||||
|
||||
Scenario: pep-extract Keywords are automatically extracted from pattern text
|
||||
Given a fresh error pattern service
|
||||
When I record a failure with pattern "Race condition in payment processing module" and failure "Timeout"
|
||||
Then the recorded pattern should have keywords including "race"
|
||||
And the recorded pattern should have keywords including "payment"
|
||||
|
||||
Scenario: pep-multi Multiple patterns match and checks are deduplicated
|
||||
Given a fresh error pattern service
|
||||
And a recorded pattern "async conversion" with preventive check "Add transaction boundaries" and keywords "async,conversion"
|
||||
And a recorded pattern "async timeout" with preventive check "Add retry logic" and keywords "async,timeout"
|
||||
When I match context "Handle async conversion with proper timeout handling"
|
||||
Then preventive guidance should contain "Add transaction boundaries"
|
||||
And preventive guidance should contain "Add retry logic"
|
||||
|
||||
Scenario: pep-repo-get Repository get returns pattern by ID
|
||||
Given a fresh error pattern service
|
||||
When I record a failure with pattern "Test pattern" and failure "Test failure"
|
||||
Then the recorded pattern can be retrieved by its ID
|
||||
|
||||
Scenario: pep-repo-delete Repository delete removes a pattern
|
||||
Given a fresh error pattern service
|
||||
When I record a failure with pattern "Deletable pattern" and failure "Will be deleted"
|
||||
And I delete the last recorded pattern
|
||||
Then the error pattern database should contain 0 patterns
|
||||
|
||||
Scenario: pep-fallback-match Pattern without keywords matches by description
|
||||
Given a fresh error pattern service
|
||||
And a recorded pattern with no keywords for "timeout error" with check "Add timeout handling"
|
||||
When I match context "We encountered a timeout error in production"
|
||||
Then preventive guidance should contain "Add timeout handling"
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Step definitions for predictive_error_prevention.feature.
|
||||
|
||||
Exercises the Error Pattern Database (Layer 4: Predictive Error Prevention)
|
||||
including recording, matching, formatting, statistics, and keyword extraction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
|
||||
from cleveragents.application.services.error_pattern_service import (
|
||||
ErrorPatternService,
|
||||
)
|
||||
from cleveragents.domain.models.core.error_pattern import (
|
||||
ErrorPattern,
|
||||
PreventiveGuidance,
|
||||
)
|
||||
from cleveragents.infrastructure.database.error_pattern_repository import (
|
||||
ErrorPatternRepository,
|
||||
)
|
||||
|
||||
|
||||
@given("a fresh error pattern service")
|
||||
def step_pep_fresh_service(context: Any) -> None:
|
||||
context.pep_service = ErrorPatternService()
|
||||
context.pep_last_pattern = None
|
||||
context.pep_guidance: PreventiveGuidance | None = None
|
||||
context.pep_statistics: dict[str, int | float] | None = None
|
||||
|
||||
|
||||
@given(
|
||||
'a recorded pattern "{pattern}" with preventive check "{check}" and keywords "{keywords}"'
|
||||
)
|
||||
def step_pep_record_pattern_with_check_and_keywords(
|
||||
context: Any, pattern: str, check: str, keywords: str
|
||||
) -> None:
|
||||
kw_tuple = tuple(k.strip() for k in keywords.split(","))
|
||||
context.pep_last_pattern = context.pep_service.record_failure(
|
||||
pattern_text=pattern,
|
||||
failure_description=f"Historical failure for {pattern}",
|
||||
preventive_checks=(check,),
|
||||
keywords=kw_tuple,
|
||||
)
|
||||
|
||||
|
||||
@given('a recorded pattern "{pattern}" with preventive check "{check}"')
|
||||
def step_pep_record_pattern_with_check(context: Any, pattern: str, check: str) -> None:
|
||||
context.pep_last_pattern = context.pep_service.record_failure(
|
||||
pattern_text=pattern,
|
||||
failure_description=f"Historical failure for {pattern}",
|
||||
preventive_checks=(check,),
|
||||
)
|
||||
|
||||
|
||||
@when('I record a failure with pattern "{pattern}" and failure "{failure}"')
|
||||
def step_pep_record_failure(context: Any, pattern: str, failure: str) -> None:
|
||||
context.pep_last_pattern = context.pep_service.record_failure(
|
||||
pattern_text=pattern,
|
||||
failure_description=failure,
|
||||
)
|
||||
|
||||
|
||||
@when('I match context "{text}"')
|
||||
def step_pep_match_context(context: Any, text: str) -> None:
|
||||
context.pep_guidance = context.pep_service.match_patterns(text)
|
||||
|
||||
|
||||
@when("I request error pattern statistics")
|
||||
def step_pep_request_statistics(context: Any) -> None:
|
||||
context.pep_statistics = context.pep_service.get_statistics()
|
||||
|
||||
|
||||
@then("the error pattern database should contain {count:d} pattern")
|
||||
def step_pep_assert_pattern_count(context: Any, count: int) -> None:
|
||||
patterns = context.pep_service.list_patterns()
|
||||
assert len(patterns) == count, f"Expected {count} patterns, got {len(patterns)}"
|
||||
|
||||
|
||||
@then("the pattern should have frequency {freq:d}")
|
||||
def step_pep_assert_frequency(context: Any, freq: int) -> None:
|
||||
patterns = context.pep_service.list_patterns()
|
||||
assert len(patterns) > 0, "No patterns found"
|
||||
actual = patterns[0].frequency
|
||||
assert actual == freq, f"Expected frequency {freq}, got {actual}"
|
||||
|
||||
|
||||
@then("the pattern should have {count:d} historical failures")
|
||||
def step_pep_assert_historical_failures_count(context: Any, count: int) -> None:
|
||||
patterns = context.pep_service.list_patterns()
|
||||
assert len(patterns) > 0, "No patterns found"
|
||||
actual = len(patterns[0].historical_failures)
|
||||
assert actual == count, f"Expected {count} historical failures, got {actual}"
|
||||
|
||||
|
||||
@then('preventive guidance should contain "{text}"')
|
||||
def step_pep_assert_guidance_contains(context: Any, text: str) -> None:
|
||||
assert context.pep_guidance is not None, "No guidance available"
|
||||
all_checks = context.pep_guidance.preventive_checks
|
||||
assert any(text in c for c in all_checks), (
|
||||
f"Expected guidance to contain '{text}', got {all_checks}"
|
||||
)
|
||||
|
||||
|
||||
@then("preventive guidance should be empty")
|
||||
def step_pep_assert_guidance_empty(context: Any) -> None:
|
||||
assert context.pep_guidance is not None, "No guidance available"
|
||||
assert not context.pep_guidance.has_guidance, (
|
||||
f"Expected empty guidance, got {context.pep_guidance.preventive_checks}"
|
||||
)
|
||||
|
||||
|
||||
@then('the formatted guidance should contain "{text}"')
|
||||
def step_pep_assert_formatted_contains(context: Any, text: str) -> None:
|
||||
assert context.pep_guidance is not None, "No guidance available"
|
||||
formatted = context.pep_guidance.format_for_context()
|
||||
assert text in formatted, (
|
||||
f"Expected formatted guidance to contain '{text}', got: {formatted}"
|
||||
)
|
||||
|
||||
|
||||
@then("the statistics should show {count:d} total patterns")
|
||||
def step_pep_assert_stats_total(context: Any, count: int) -> None:
|
||||
assert context.pep_statistics is not None, "No statistics available"
|
||||
actual = context.pep_statistics["total_patterns"]
|
||||
assert actual == count, f"Expected {count} total patterns, got {actual}"
|
||||
|
||||
|
||||
@then('the recorded pattern should have keywords including "{keyword}"')
|
||||
def step_pep_assert_keyword(context: Any, keyword: str) -> None:
|
||||
assert context.pep_last_pattern is not None, "No pattern recorded"
|
||||
keywords_lower = [k.lower() for k in context.pep_last_pattern.keywords]
|
||||
assert keyword.lower() in keywords_lower, (
|
||||
f"Expected keyword '{keyword}' in {context.pep_last_pattern.keywords}"
|
||||
)
|
||||
|
||||
|
||||
@then("the recorded pattern can be retrieved by its ID")
|
||||
def step_pep_retrieve_by_id(context: Any) -> None:
|
||||
assert context.pep_last_pattern is not None, "No pattern recorded"
|
||||
repo = ErrorPatternRepository()
|
||||
repo.create(context.pep_last_pattern)
|
||||
retrieved = repo.get(context.pep_last_pattern.pattern_id)
|
||||
assert retrieved is not None, "Pattern not found by ID"
|
||||
assert retrieved.pattern_id == context.pep_last_pattern.pattern_id
|
||||
|
||||
|
||||
@when("I delete the last recorded pattern")
|
||||
def step_pep_delete_last_pattern(context: Any) -> None:
|
||||
assert context.pep_last_pattern is not None, "No pattern recorded"
|
||||
# Access the internal repo to delete
|
||||
deleted = context.pep_service._repo.delete(context.pep_last_pattern.pattern_id)
|
||||
assert deleted, "Delete returned False"
|
||||
|
||||
|
||||
@then("the error pattern database should contain {count:d} patterns")
|
||||
def step_pep_assert_pattern_count_plural(context: Any, count: int) -> None:
|
||||
patterns = context.pep_service.list_patterns()
|
||||
assert len(patterns) == count, f"Expected {count} patterns, got {len(patterns)}"
|
||||
|
||||
|
||||
@given('a recorded pattern with no keywords for "{pattern}" with check "{check}"')
|
||||
def step_pep_record_pattern_no_keywords(context: Any, pattern: str, check: str) -> None:
|
||||
ep = ErrorPattern(
|
||||
pattern=pattern,
|
||||
historical_failures=("Initial failure",),
|
||||
preventive_checks=(check,),
|
||||
keywords=(), # No keywords — forces fallback matching
|
||||
)
|
||||
context.pep_service._repo.create(ep)
|
||||
context.pep_last_pattern = ep
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Step definitions for system_diagnostics_coverage.feature.
|
||||
|
||||
Covers uncovered paths in cli/commands/system.py:
|
||||
- _check_file_permissions: writable-but-not-readable (line 344)
|
||||
- _check_stale_locks: locks table exists paths (lines 367-388)
|
||||
- _check_async_worker_health: async enabled path (lines 405-417)
|
||||
- _check_error_patterns: exception path (lines 438-442)
|
||||
|
||||
All step patterns use the 'sysdiag_' prefix to avoid AmbiguousStep collisions
|
||||
with existing steps in cli_core_steps.py and coverage_security_template_boost_steps.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Helpers
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _sysdiag_mock_settings(**overrides: Any) -> Any:
|
||||
"""Create mock settings with sensible defaults for diagnostic tests."""
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
s = MagicMock()
|
||||
s.database_url = overrides.get("database_url", f"sqlite:///{tmpdir}/test.db")
|
||||
s.data_dir = overrides.get("data_dir", Path(tmpdir))
|
||||
s.storage_path = overrides.get("storage_path", Path(tmpdir))
|
||||
s.log_dir = overrides.get("log_dir", Path(tmpdir) / "logs")
|
||||
s.default_automation_profile = "auto"
|
||||
s.debug_enabled = False
|
||||
s.async_enabled = overrides.get("async_enabled", False)
|
||||
s.async_max_workers = overrides.get("async_max_workers", 4)
|
||||
s.async_poll_interval = overrides.get("async_poll_interval", 1.0)
|
||||
s.has_provider_configured = MagicMock(return_value=False)
|
||||
s.configured_providers = []
|
||||
return s
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Given steps
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@given("a sysdiag test environment")
|
||||
def step_sysdiag_env(context: Context) -> None:
|
||||
context.sysdiag_result = {}
|
||||
context.sysdiag_tmpdir = tempfile.mkdtemp()
|
||||
|
||||
|
||||
@given("a data dir that exists")
|
||||
def step_sysdiag_data_dir_exists(context: Context) -> None:
|
||||
data_dir = Path(context.sysdiag_tmpdir) / "data"
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
context.sysdiag_data_dir = data_dir
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# When steps — _check_file_permissions edge cases
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@when("I run the file permissions check with read access denied")
|
||||
def step_sysdiag_perms_no_read(context: Context) -> None:
|
||||
from cleveragents.cli.commands.system import _check_file_permissions
|
||||
|
||||
ms = _sysdiag_mock_settings(data_dir=context.sysdiag_data_dir)
|
||||
|
||||
def fake_access(path: Any, mode: int) -> bool:
|
||||
"""Simulate writable-but-not-readable directory."""
|
||||
import os as _os
|
||||
|
||||
if mode == _os.R_OK:
|
||||
return False
|
||||
return mode == _os.W_OK
|
||||
|
||||
with (
|
||||
patch("cleveragents.config.settings.get_settings", return_value=ms),
|
||||
patch("cleveragents.cli.commands.system.os.access", side_effect=fake_access),
|
||||
):
|
||||
context.sysdiag_result = _check_file_permissions()
|
||||
|
||||
|
||||
@when("I run the file permissions check with no access at all")
|
||||
def step_sysdiag_perms_no_access(context: Context) -> None:
|
||||
from cleveragents.cli.commands.system import _check_file_permissions
|
||||
|
||||
ms = _sysdiag_mock_settings(data_dir=context.sysdiag_data_dir)
|
||||
|
||||
with (
|
||||
patch("cleveragents.config.settings.get_settings", return_value=ms),
|
||||
patch("cleveragents.cli.commands.system.os.access", return_value=False),
|
||||
):
|
||||
context.sysdiag_result = _check_file_permissions()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# When steps — _check_stale_locks edge cases
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@when("I run the stale locks check with {count:d} stale locks in the table")
|
||||
def step_sysdiag_stale_locks_count(context: Context, count: int) -> None:
|
||||
from cleveragents.cli.commands.system import _check_stale_locks
|
||||
|
||||
mock_engine = MagicMock()
|
||||
mock_inspector = MagicMock()
|
||||
mock_inspector.get_table_names.return_value = ["locks", "other_table"]
|
||||
|
||||
mock_lock_service = MagicMock()
|
||||
mock_lock_service.count_stale_locks.return_value = count
|
||||
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.system.get_database_url",
|
||||
return_value="sqlite:///test.db",
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.system.create_engine",
|
||||
return_value=mock_engine,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.system.sa_inspect",
|
||||
return_value=mock_inspector,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.system.LockService",
|
||||
return_value=mock_lock_service,
|
||||
),
|
||||
):
|
||||
context.sysdiag_result = _check_stale_locks()
|
||||
|
||||
|
||||
@when("I run the stale locks check with a database error")
|
||||
def step_sysdiag_stale_locks_error(context: Context) -> None:
|
||||
from cleveragents.cli.commands.system import _check_stale_locks
|
||||
|
||||
with patch(
|
||||
"cleveragents.cli.commands.system.get_database_url",
|
||||
side_effect=RuntimeError("database unreachable"),
|
||||
):
|
||||
context.sysdiag_result = _check_stale_locks()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# When steps — _check_async_worker_health edge cases
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@when("I run the async worker health check with async enabled")
|
||||
def step_sysdiag_async_enabled(context: Context) -> None:
|
||||
from cleveragents.cli.commands.system import _check_async_worker_health
|
||||
|
||||
ms = _sysdiag_mock_settings(
|
||||
async_enabled=True,
|
||||
async_max_workers=8,
|
||||
async_poll_interval=2.5,
|
||||
)
|
||||
with patch("cleveragents.config.settings.get_settings", return_value=ms):
|
||||
context.sysdiag_result = _check_async_worker_health()
|
||||
|
||||
|
||||
@when("I run the async worker health check with a settings error")
|
||||
def step_sysdiag_async_error(context: Context) -> None:
|
||||
from cleveragents.cli.commands.system import _check_async_worker_health
|
||||
|
||||
with patch(
|
||||
"cleveragents.config.settings.get_settings",
|
||||
side_effect=RuntimeError("settings unavailable"),
|
||||
):
|
||||
context.sysdiag_result = _check_async_worker_health()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# When steps — _check_error_patterns exception path
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@when("I run the error patterns check with a service error")
|
||||
def step_sysdiag_error_patterns_error(context: Context) -> None:
|
||||
from cleveragents.cli.commands.system import _check_error_patterns
|
||||
|
||||
with patch(
|
||||
"cleveragents.application.services.error_pattern_service.ErrorPatternService",
|
||||
side_effect=RuntimeError("service init failed"),
|
||||
):
|
||||
context.sysdiag_result = _check_error_patterns()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Then steps
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@then('the sysdiag check status should be "{expected}"')
|
||||
def step_sysdiag_status(context: Context, expected: str) -> None:
|
||||
from cleveragents.cli.commands.system import CheckStatus
|
||||
|
||||
status_map = {
|
||||
"ok": CheckStatus.OK,
|
||||
"warn": CheckStatus.WARN,
|
||||
"error": CheckStatus.ERROR,
|
||||
}
|
||||
actual = context.sysdiag_result["status"]
|
||||
assert actual == status_map[expected], (
|
||||
f"Expected status '{expected}', got '{actual}'"
|
||||
)
|
||||
|
||||
|
||||
@then('the sysdiag check details should be "{expected}"')
|
||||
def step_sysdiag_details_exact(context: Context, expected: str) -> None:
|
||||
actual = context.sysdiag_result["details"]
|
||||
assert actual == expected, f"Expected details '{expected}', got '{actual}'"
|
||||
|
||||
|
||||
@then('the sysdiag check details should contain "{text}"')
|
||||
def step_sysdiag_details_contains(context: Context, text: str) -> None:
|
||||
details = context.sysdiag_result["details"]
|
||||
assert text in details, f"Expected '{text}' in details: '{details}'"
|
||||
|
||||
|
||||
@then("the sysdiag check should have a recommendation")
|
||||
def step_sysdiag_has_recommendation(context: Context) -> None:
|
||||
rec = context.sysdiag_result.get("recommendation")
|
||||
assert rec is not None and len(rec) > 0, (
|
||||
f"Expected a non-empty recommendation, got: {rec!r}"
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
@coverage_boost
|
||||
Feature: System diagnostics uncovered code paths
|
||||
As a developer ensuring thorough test coverage
|
||||
I want to exercise uncovered diagnostic check paths in system.py
|
||||
So that stale-lock, async-worker, error-pattern, and file-permission edge cases are tested
|
||||
|
||||
# ===========================================================================
|
||||
# _check_file_permissions — line 344: writable-but-not-readable data dir
|
||||
# ===========================================================================
|
||||
|
||||
Scenario: File permissions check on data dir that is writable but not readable
|
||||
Given a sysdiag test environment
|
||||
And a data dir that exists
|
||||
When I run the file permissions check with read access denied
|
||||
Then the sysdiag check status should be "warn"
|
||||
And the sysdiag check details should be "data dir w"
|
||||
|
||||
Scenario: File permissions check on data dir that is neither readable nor writable
|
||||
Given a sysdiag test environment
|
||||
And a data dir that exists
|
||||
When I run the file permissions check with no access at all
|
||||
Then the sysdiag check status should be "error"
|
||||
And the sysdiag check details should be "data dir no access"
|
||||
|
||||
# ===========================================================================
|
||||
# _check_stale_locks — lines 367-388
|
||||
# ===========================================================================
|
||||
|
||||
Scenario: Stale locks check finds zero stale locks in existing table
|
||||
Given a sysdiag test environment
|
||||
When I run the stale locks check with 0 stale locks in the table
|
||||
Then the sysdiag check status should be "ok"
|
||||
And the sysdiag check details should be "0 stale locks"
|
||||
|
||||
Scenario: Stale locks check finds stale locks present
|
||||
Given a sysdiag test environment
|
||||
When I run the stale locks check with 3 stale locks in the table
|
||||
Then the sysdiag check status should be "warn"
|
||||
And the sysdiag check details should contain "3 stale lock(s) found"
|
||||
And the sysdiag check should have a recommendation
|
||||
|
||||
Scenario: Stale locks check handles database errors gracefully
|
||||
Given a sysdiag test environment
|
||||
When I run the stale locks check with a database error
|
||||
Then the sysdiag check status should be "warn"
|
||||
And the sysdiag check details should be "unable to check"
|
||||
|
||||
# ===========================================================================
|
||||
# _check_async_worker_health — lines 405-417
|
||||
# ===========================================================================
|
||||
|
||||
Scenario: Async worker health check when async is enabled
|
||||
Given a sysdiag test environment
|
||||
When I run the async worker health check with async enabled
|
||||
Then the sysdiag check status should be "ok"
|
||||
And the sysdiag check details should contain "enabled"
|
||||
And the sysdiag check details should contain "max_workers="
|
||||
And the sysdiag check details should contain "poll_interval="
|
||||
|
||||
Scenario: Async worker health check handles settings errors gracefully
|
||||
Given a sysdiag test environment
|
||||
When I run the async worker health check with a settings error
|
||||
Then the sysdiag check status should be "warn"
|
||||
And the sysdiag check details should be "unable to check"
|
||||
|
||||
# ===========================================================================
|
||||
# _check_error_patterns — lines 438-442
|
||||
# ===========================================================================
|
||||
|
||||
Scenario: Error pattern check when service initialization fails
|
||||
Given a sysdiag test environment
|
||||
When I run the error patterns check with a service error
|
||||
Then the sysdiag check status should be "ok"
|
||||
And the sysdiag check details should be "empty (no patterns recorded)"
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Helper utilities for Predictive Error Prevention Robot integration tests.
|
||||
|
||||
Covers Layer 4: Error Pattern Database — recording, matching, and
|
||||
preventive guidance generation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cleveragents.application.services.error_pattern_service import (
|
||||
ErrorPatternService,
|
||||
)
|
||||
from cleveragents.domain.models.core.error_pattern import ErrorPattern
|
||||
|
||||
|
||||
def _record_and_verify_pattern() -> None:
|
||||
"""Integration test: record a failure and verify pattern creation."""
|
||||
service = ErrorPatternService()
|
||||
pattern = service.record_failure(
|
||||
pattern_text="Race condition in payment module",
|
||||
failure_description="Payment confirmation timeout",
|
||||
preventive_checks=("Add explicit locking",),
|
||||
keywords=("race", "payment"),
|
||||
)
|
||||
assert isinstance(pattern, ErrorPattern), (
|
||||
f"Expected ErrorPattern, got {type(pattern)}"
|
||||
)
|
||||
assert pattern.frequency == 1, f"Expected frequency 1, got {pattern.frequency}"
|
||||
assert len(pattern.historical_failures) == 1, (
|
||||
f"Expected 1 historical failure, got {len(pattern.historical_failures)}"
|
||||
)
|
||||
assert pattern.pattern == "Race condition in payment module"
|
||||
|
||||
stats = service.get_statistics()
|
||||
assert stats["total_patterns"] == 1, (
|
||||
f"Expected 1 pattern, got {stats['total_patterns']}"
|
||||
)
|
||||
print("record-and-verify-ok")
|
||||
|
||||
|
||||
def _match_patterns_returns_guidance() -> None:
|
||||
"""Integration test: match context against patterns returns guidance."""
|
||||
service = ErrorPatternService()
|
||||
service.record_failure(
|
||||
pattern_text="Async conversion failure",
|
||||
failure_description="Timeout during async conversion",
|
||||
preventive_checks=("Add transaction boundaries", "Use retry logic"),
|
||||
keywords=("async", "conversion"),
|
||||
)
|
||||
|
||||
guidance = service.match_patterns(
|
||||
"We need to perform async conversion in the payment module"
|
||||
)
|
||||
assert guidance.has_guidance, "Expected guidance to be non-empty"
|
||||
assert len(guidance.preventive_checks) == 2, (
|
||||
f"Expected 2 checks, got {len(guidance.preventive_checks)}"
|
||||
)
|
||||
assert "Add transaction boundaries" in guidance.preventive_checks
|
||||
assert "Use retry logic" in guidance.preventive_checks
|
||||
|
||||
formatted = guidance.format_for_context()
|
||||
assert "PREVENTIVE GUIDANCE" in formatted, (
|
||||
f"Expected header in formatted output: {formatted}"
|
||||
)
|
||||
print("match-patterns-guidance-ok")
|
||||
|
||||
|
||||
def _frequency_increments_on_duplicate() -> None:
|
||||
"""Integration test: duplicate pattern recording increments frequency."""
|
||||
service = ErrorPatternService()
|
||||
service.record_failure(
|
||||
pattern_text="Memory leak in cache",
|
||||
failure_description="OOM after 24h",
|
||||
preventive_checks=("Check dispose patterns",),
|
||||
keywords=("memory", "leak", "cache"),
|
||||
)
|
||||
updated = service.record_failure(
|
||||
pattern_text="Memory leak in cache",
|
||||
failure_description="High memory usage after 12h",
|
||||
)
|
||||
assert updated.frequency == 2, f"Expected frequency 2, got {updated.frequency}"
|
||||
assert len(updated.historical_failures) == 2, (
|
||||
f"Expected 2 failures, got {len(updated.historical_failures)}"
|
||||
)
|
||||
|
||||
stats = service.get_statistics()
|
||||
assert stats["total_patterns"] == 1, (
|
||||
f"Expected 1 pattern, got {stats['total_patterns']}"
|
||||
)
|
||||
assert stats["total_occurrences"] == 2, (
|
||||
f"Expected 2 occurrences, got {stats['total_occurrences']}"
|
||||
)
|
||||
print("frequency-increments-ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_record_and_verify_pattern()
|
||||
_match_patterns_returns_guidance()
|
||||
_frequency_increments_on_duplicate()
|
||||
print("all-predictive-error-prevention-tests-passed")
|
||||
@@ -0,0 +1,48 @@
|
||||
*** Settings ***
|
||||
Documentation Layer 4 - Predictive Error Prevention integration smoke tests
|
||||
Library OperatingSystem
|
||||
Library Process
|
||||
|
||||
*** Variables ***
|
||||
${ERROR_PATTERN_MODEL} ${CURDIR}/../src/cleveragents/domain/models/core/error_pattern.py
|
||||
${ERROR_PATTERN_REPO} ${CURDIR}/../src/cleveragents/infrastructure/database/error_pattern_repository.py
|
||||
${ERROR_PATTERN_SERVICE} ${CURDIR}/../src/cleveragents/application/services/error_pattern_service.py
|
||||
|
||||
*** Test Cases ***
|
||||
Error Pattern Domain Model Exists
|
||||
[Documentation] Verify the error pattern domain model module is present
|
||||
File Should Exist ${ERROR_PATTERN_MODEL}
|
||||
${content}= Get File ${ERROR_PATTERN_MODEL}
|
||||
Should Contain ${content} class ErrorPattern
|
||||
Should Contain ${content} class PreventiveGuidance
|
||||
Should Contain ${content} pattern_id
|
||||
Should Contain ${content} historical_failures
|
||||
Should Contain ${content} preventive_checks
|
||||
Should Contain ${content} frequency
|
||||
Should Contain ${content} keywords
|
||||
Should Contain ${content} format_for_context
|
||||
|
||||
Error Pattern Repository Module Exists
|
||||
[Documentation] Verify the error pattern repository module is present
|
||||
File Should Exist ${ERROR_PATTERN_REPO}
|
||||
${content}= Get File ${ERROR_PATTERN_REPO}
|
||||
Should Contain ${content} class ErrorPatternRepository
|
||||
Should Contain ${content} def create(
|
||||
Should Contain ${content} def get(
|
||||
Should Contain ${content} def list_all(
|
||||
Should Contain ${content} def update(
|
||||
Should Contain ${content} def delete(
|
||||
Should Contain ${content} def match_context(
|
||||
Should Contain ${content} def count(
|
||||
Should Contain ${content} def statistics(
|
||||
|
||||
Error Pattern Service Provides Recording Matching And Statistics
|
||||
[Documentation] Verify the error pattern service has all required methods
|
||||
File Should Exist ${ERROR_PATTERN_SERVICE}
|
||||
${content}= Get File ${ERROR_PATTERN_SERVICE}
|
||||
Should Contain ${content} class ErrorPatternService
|
||||
Should Contain ${content} def record_failure(
|
||||
Should Contain ${content} def match_patterns(
|
||||
Should Contain ${content} def get_statistics(
|
||||
Should Contain ${content} def list_patterns(
|
||||
Should Contain ${content} _extract_keywords
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Error Pattern Service — records failures and provides preventive guidance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from cleveragents.domain.models.core.error_pattern import (
|
||||
ErrorPattern,
|
||||
PreventiveGuidance,
|
||||
)
|
||||
from cleveragents.infrastructure.database.error_pattern_repository import (
|
||||
ErrorPatternRepository,
|
||||
)
|
||||
|
||||
|
||||
class ErrorPatternService:
|
||||
"""Service layer for the Error Pattern Database.
|
||||
|
||||
Implements Layer 4: Predictive Error Prevention.
|
||||
"""
|
||||
|
||||
def __init__(self, repository: ErrorPatternRepository | None = None) -> None:
|
||||
self._repo = repository or ErrorPatternRepository()
|
||||
|
||||
def record_failure(
|
||||
self,
|
||||
pattern_text: str,
|
||||
failure_description: str,
|
||||
preventive_checks: tuple[str, ...] = (),
|
||||
keywords: tuple[str, ...] = (),
|
||||
) -> ErrorPattern:
|
||||
"""Record a new failure or update frequency of an existing pattern."""
|
||||
# Check if similar pattern already exists
|
||||
existing = self._find_existing_pattern(pattern_text)
|
||||
if existing is not None:
|
||||
# Merge preventive checks (accumulate, don't replace)
|
||||
merged_checks = list(existing.preventive_checks)
|
||||
for check in preventive_checks:
|
||||
if check not in merged_checks:
|
||||
merged_checks.append(check)
|
||||
updated = ErrorPattern(
|
||||
pattern_id=existing.pattern_id,
|
||||
pattern=existing.pattern,
|
||||
historical_failures=(
|
||||
*existing.historical_failures,
|
||||
failure_description,
|
||||
),
|
||||
preventive_checks=tuple(merged_checks),
|
||||
frequency=existing.frequency + 1,
|
||||
created_at=existing.created_at,
|
||||
last_seen=datetime.now(UTC),
|
||||
keywords=keywords if keywords else existing.keywords,
|
||||
)
|
||||
return self._repo.update(updated)
|
||||
|
||||
new_pattern = ErrorPattern(
|
||||
pattern=pattern_text,
|
||||
historical_failures=(failure_description,),
|
||||
preventive_checks=preventive_checks,
|
||||
keywords=keywords or self._extract_keywords(pattern_text),
|
||||
)
|
||||
return self._repo.create(new_pattern)
|
||||
|
||||
def match_patterns(self, context_text: str) -> PreventiveGuidance:
|
||||
"""Match context against known error patterns.
|
||||
|
||||
Returns preventive guidance for matched patterns.
|
||||
"""
|
||||
matched = self._repo.match_context(context_text)
|
||||
if not matched:
|
||||
return PreventiveGuidance()
|
||||
|
||||
all_checks: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for pattern in matched:
|
||||
for check in pattern.preventive_checks:
|
||||
if check not in seen:
|
||||
all_checks.append(check)
|
||||
seen.add(check)
|
||||
|
||||
return PreventiveGuidance(
|
||||
matched_patterns=tuple(matched),
|
||||
preventive_checks=tuple(all_checks),
|
||||
)
|
||||
|
||||
def get_statistics(self) -> dict[str, int | float]:
|
||||
"""Return error pattern database statistics."""
|
||||
return self._repo.statistics()
|
||||
|
||||
def list_patterns(self) -> list[ErrorPattern]:
|
||||
"""List all error patterns, most recent first."""
|
||||
return self._repo.list_all()
|
||||
|
||||
def _find_existing_pattern(self, pattern_text: str) -> ErrorPattern | None:
|
||||
"""Find an existing pattern by exact text match."""
|
||||
pattern_lower = pattern_text.lower()
|
||||
for p in self._repo.list_all():
|
||||
if p.pattern.lower() == pattern_lower:
|
||||
return p
|
||||
return None
|
||||
|
||||
def _extract_keywords(self, text: str) -> tuple[str, ...]:
|
||||
"""Extract significant keywords from pattern text for matching."""
|
||||
stop_words = frozenset(
|
||||
{
|
||||
"a",
|
||||
"an",
|
||||
"the",
|
||||
"in",
|
||||
"on",
|
||||
"at",
|
||||
"to",
|
||||
"for",
|
||||
"of",
|
||||
"with",
|
||||
"and",
|
||||
"or",
|
||||
"but",
|
||||
"is",
|
||||
"was",
|
||||
"are",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"has",
|
||||
"had",
|
||||
"have",
|
||||
"do",
|
||||
"does",
|
||||
"did",
|
||||
"will",
|
||||
"would",
|
||||
"could",
|
||||
"should",
|
||||
"may",
|
||||
"might",
|
||||
"can",
|
||||
"this",
|
||||
"that",
|
||||
"it",
|
||||
"its",
|
||||
"not",
|
||||
"no",
|
||||
"from",
|
||||
"by",
|
||||
"as",
|
||||
}
|
||||
)
|
||||
words = text.lower().split()
|
||||
keywords = [w for w in words if len(w) > 2 and w not in stop_words]
|
||||
return tuple(keywords[:10]) # Limit to 10 keywords
|
||||
@@ -90,6 +90,9 @@ from cleveragents.infrastructure.events.types import EventType
|
||||
if TYPE_CHECKING:
|
||||
from cleveragents.application.services.async_worker import InMemoryJobStore
|
||||
from cleveragents.application.services.decision_service import DecisionService
|
||||
from cleveragents.application.services.error_pattern_service import (
|
||||
ErrorPatternService,
|
||||
)
|
||||
from cleveragents.config.settings import Settings
|
||||
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
|
||||
from cleveragents.infrastructure.events.protocol import EventBus
|
||||
@@ -159,6 +162,7 @@ class PlanLifecycleService:
|
||||
decision_service: DecisionService | None = None,
|
||||
event_bus: EventBus | None = None,
|
||||
job_store: InMemoryJobStore | None = None,
|
||||
error_pattern_service: ErrorPatternService | None = None,
|
||||
):
|
||||
"""Initialize the plan lifecycle service.
|
||||
|
||||
@@ -181,12 +185,19 @@ class PlanLifecycleService:
|
||||
async jobs when ``settings.async_enabled`` is True.
|
||||
When ``None``, async job creation is silently skipped
|
||||
even if async is enabled.
|
||||
error_pattern_service: Optional
|
||||
:class:`ErrorPatternService` for Layer 4 Predictive
|
||||
Error Prevention. When provided, the service is
|
||||
consulted before the Execute phase to inject preventive
|
||||
guidance. When ``None``, predictive prevention is
|
||||
silently skipped.
|
||||
"""
|
||||
self.settings = settings
|
||||
self.unit_of_work = unit_of_work
|
||||
self.decision_service = decision_service
|
||||
self.event_bus = event_bus
|
||||
self._job_store = job_store
|
||||
self.error_pattern_service = error_pattern_service
|
||||
self._logger = logger.bind(service="plan_lifecycle")
|
||||
self.preflight_guardrail = PlanPreflightGuardrail()
|
||||
|
||||
@@ -231,6 +242,42 @@ class PlanLifecycleService:
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _consult_error_patterns(self, plan: Plan) -> None:
|
||||
"""Consult the Error Pattern Database before execution.
|
||||
|
||||
When an ``ErrorPatternService`` is configured, the plan's
|
||||
description is matched against known error patterns. Any
|
||||
resulting preventive guidance is stored in the plan's
|
||||
``error_details`` dict under the ``preventive_guidance`` key
|
||||
so that downstream actors can incorporate it.
|
||||
|
||||
Failures are logged but never propagated — preventive guidance
|
||||
must not block lifecycle transitions.
|
||||
"""
|
||||
if self.error_pattern_service is None:
|
||||
return
|
||||
|
||||
try:
|
||||
context_text = plan.description or ""
|
||||
guidance = self.error_pattern_service.match_patterns(context_text)
|
||||
if guidance.has_guidance:
|
||||
formatted = guidance.format_for_context()
|
||||
existing = dict(plan.error_details) if plan.error_details else {}
|
||||
existing["preventive_guidance"] = formatted
|
||||
plan.error_details = existing
|
||||
self._logger.info(
|
||||
"Preventive guidance injected",
|
||||
plan_id=plan.identity.plan_id,
|
||||
matched_patterns=len(guidance.matched_patterns),
|
||||
checks=len(guidance.preventive_checks),
|
||||
)
|
||||
except Exception:
|
||||
self._logger.warning(
|
||||
"error_pattern_consultation_failed",
|
||||
plan_id=plan.identity.plan_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _persist_action_create(self, action: Action, ctx: Any) -> None:
|
||||
"""Persist a new action via the repository.
|
||||
|
||||
@@ -914,6 +961,9 @@ class PlanLifecycleService:
|
||||
plan_id, plan.phase, plan.state or ProcessingState.QUEUED
|
||||
)
|
||||
|
||||
# Layer 4: Consult Error Pattern Database for preventive guidance
|
||||
self._consult_error_patterns(plan)
|
||||
|
||||
# Transition to Execute phase
|
||||
plan.phase = PlanPhase.EXECUTE
|
||||
plan.processing_state = ProcessingState.QUEUED
|
||||
|
||||
@@ -418,6 +418,31 @@ def _check_async_worker_health() -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _check_error_patterns() -> dict[str, Any]:
|
||||
"""Check the Error Pattern Database status."""
|
||||
try:
|
||||
from cleveragents.application.services.error_pattern_service import (
|
||||
ErrorPatternService,
|
||||
)
|
||||
|
||||
service = ErrorPatternService()
|
||||
stats = service.get_statistics()
|
||||
total = stats["total_patterns"]
|
||||
return {
|
||||
"name": "Error Pattern DB",
|
||||
"status": CheckStatus.OK,
|
||||
"details": (
|
||||
f"{total} patterns, {stats['total_occurrences']} total occurrences"
|
||||
),
|
||||
}
|
||||
except Exception:
|
||||
return {
|
||||
"name": "Error Pattern DB",
|
||||
"status": CheckStatus.OK,
|
||||
"details": "empty (no patterns recorded)",
|
||||
}
|
||||
|
||||
|
||||
def build_diagnostics_data() -> dict[str, Any]:
|
||||
"""Run all diagnostic checks and return structured results."""
|
||||
start = time.monotonic()
|
||||
@@ -433,6 +458,7 @@ def build_diagnostics_data() -> dict[str, Any]:
|
||||
checks.append(_check_stale_locks())
|
||||
|
||||
checks.append(_check_async_worker_health())
|
||||
checks.append(_check_error_patterns())
|
||||
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
|
||||
@@ -108,6 +108,12 @@ from cleveragents.domain.models.core.diff_review import (
|
||||
ReviewArtifact,
|
||||
)
|
||||
|
||||
# Error pattern domain models (Layer 4: Predictive Error Prevention)
|
||||
from cleveragents.domain.models.core.error_pattern import (
|
||||
ErrorPattern,
|
||||
PreventiveGuidance,
|
||||
)
|
||||
|
||||
# Error recovery models
|
||||
from cleveragents.domain.models.core.error_recovery import (
|
||||
ErrorCategory,
|
||||
@@ -356,6 +362,7 @@ __all__ = [
|
||||
"DoDSummary",
|
||||
"ErrorCategory",
|
||||
"ErrorHistory",
|
||||
"ErrorPattern",
|
||||
"ErrorRecord",
|
||||
"ErrorRecoveryPolicy",
|
||||
"EscalationDecision",
|
||||
@@ -407,6 +414,7 @@ __all__ = [
|
||||
"PlanResult",
|
||||
"PlanStatus",
|
||||
"PlanTimestamps",
|
||||
"PreventiveGuidance",
|
||||
"ProcessingState",
|
||||
"Project",
|
||||
"ProjectContextPolicy",
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Error Pattern domain model for Predictive Error Prevention (Layer 4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from ulid import ULID
|
||||
|
||||
|
||||
class ErrorPattern(BaseModel, frozen=True):
|
||||
"""A recorded error pattern with failure history and preventive checks."""
|
||||
|
||||
pattern_id: str = Field(default_factory=lambda: str(ULID()))
|
||||
pattern: str = Field(min_length=1, description="Description of the error pattern")
|
||||
historical_failures: tuple[str, ...] = Field(
|
||||
default_factory=tuple,
|
||||
description="List of historical failure descriptions matching this pattern",
|
||||
)
|
||||
preventive_checks: tuple[str, ...] = Field(
|
||||
default_factory=tuple,
|
||||
description="List of preventive check descriptions to apply",
|
||||
)
|
||||
frequency: int = Field(
|
||||
default=1, ge=1, description="Number of times pattern was observed"
|
||||
)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
last_seen: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
# Keywords extracted from pattern for matching
|
||||
keywords: tuple[str, ...] = Field(
|
||||
default_factory=tuple,
|
||||
description="Keywords for pattern matching against plan context",
|
||||
)
|
||||
|
||||
|
||||
class PreventiveGuidance(BaseModel, frozen=True):
|
||||
"""Guidance to inject into actor context before plan execution."""
|
||||
|
||||
matched_patterns: tuple[ErrorPattern, ...] = Field(default_factory=tuple)
|
||||
preventive_checks: tuple[str, ...] = Field(default_factory=tuple)
|
||||
|
||||
@property
|
||||
def has_guidance(self) -> bool:
|
||||
"""Return True if there are any preventive checks."""
|
||||
return len(self.preventive_checks) > 0
|
||||
|
||||
def format_for_context(self) -> str:
|
||||
"""Format preventive guidance as text for inclusion in actor context."""
|
||||
if not self.has_guidance:
|
||||
return ""
|
||||
lines: list[str] = ["[PREVENTIVE GUIDANCE — from Error Pattern Database]"]
|
||||
for i, check in enumerate(self.preventive_checks, 1):
|
||||
lines.append(f" {i}. {check}")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Error Pattern Repository — in-memory store for error patterns."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cleveragents.domain.models.core.error_pattern import ErrorPattern
|
||||
|
||||
|
||||
class ErrorPatternRepository:
|
||||
"""In-memory CRUD + pattern-matching repository for error patterns."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._patterns: dict[str, ErrorPattern] = {}
|
||||
|
||||
def create(self, pattern: ErrorPattern) -> ErrorPattern:
|
||||
"""Create a new error pattern."""
|
||||
self._patterns[pattern.pattern_id] = pattern
|
||||
return pattern
|
||||
|
||||
def get(self, pattern_id: str) -> ErrorPattern | None:
|
||||
"""Get a pattern by ID."""
|
||||
return self._patterns.get(pattern_id)
|
||||
|
||||
def list_all(self) -> list[ErrorPattern]:
|
||||
"""List all patterns, sorted by last_seen descending."""
|
||||
return sorted(self._patterns.values(), key=lambda p: p.last_seen, reverse=True)
|
||||
|
||||
def update(self, pattern: ErrorPattern) -> ErrorPattern:
|
||||
"""Update an existing pattern."""
|
||||
self._patterns[pattern.pattern_id] = pattern
|
||||
return pattern
|
||||
|
||||
def delete(self, pattern_id: str) -> bool:
|
||||
"""Delete a pattern by ID. Returns True if deleted."""
|
||||
return self._patterns.pop(pattern_id, None) is not None
|
||||
|
||||
def match_context(self, context_text: str) -> list[ErrorPattern]:
|
||||
"""Find patterns whose keywords appear in the context text."""
|
||||
context_lower = context_text.lower()
|
||||
matched: list[ErrorPattern] = []
|
||||
for pattern in self._patterns.values():
|
||||
if pattern.keywords:
|
||||
if any(kw.lower() in context_lower for kw in pattern.keywords):
|
||||
matched.append(pattern)
|
||||
else:
|
||||
# Fallback: check if pattern description appears in context
|
||||
if pattern.pattern.lower() in context_lower:
|
||||
matched.append(pattern)
|
||||
return sorted(matched, key=lambda p: p.frequency, reverse=True)
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the number of patterns stored."""
|
||||
return len(self._patterns)
|
||||
|
||||
def statistics(self) -> dict[str, int | float]:
|
||||
"""Return aggregate statistics about stored patterns."""
|
||||
patterns = self.list_all()
|
||||
total = len(patterns)
|
||||
total_freq = sum(p.frequency for p in patterns)
|
||||
return {
|
||||
"total_patterns": total,
|
||||
"total_occurrences": total_freq,
|
||||
"avg_frequency": total_freq / total if total > 0 else 0.0,
|
||||
}
|
||||
Reference in New Issue
Block a user