feat(correcting-plans): implement Predictive Error Prevention (Layer 4) with Error Pattern Database
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 54s
CI / unit_tests (pull_request) Successful in 2m39s
CI / integration_tests (pull_request) Successful in 3m10s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m0s
CI / lint (push) Successful in 14s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 34s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m8s
CI / coverage (push) Successful in 5m10s
CI / benchmark-publish (push) Successful in 16m26s
CI / benchmark-regression (pull_request) Successful in 30m12s
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 14s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 54s
CI / unit_tests (pull_request) Successful in 2m39s
CI / integration_tests (pull_request) Successful in 3m10s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m0s
CI / lint (push) Successful in 14s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 34s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m8s
CI / coverage (push) Successful in 5m10s
CI / benchmark-publish (push) Successful in 16m26s
CI / benchmark-regression (pull_request) Successful in 30m12s
Implement spec-mandated Layer 4 Predictive Error Prevention system: - ErrorPattern domain model with pattern text, historical failures, preventive checks, frequency tracking, and keyword-based matching. - ErrorPatternRepository with in-memory CRUD + context-matching query. - ErrorPatternService with record_failure(), match_patterns(), and get_statistics() methods. - Wire into plan execution via plan_lifecycle_service pre-execution hook. - Add error pattern statistics to CLI diagnostics output. Behave BDD: 11 scenarios covering recording, matching, formatting, stats. Robot Framework: 3 integration smoke tests. ASV benchmarks: pattern matching performance. ISSUES CLOSED: #571
This commit was merged in pull request #643.
This commit is contained in:
@@ -13,6 +13,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
@@ -59,8 +60,8 @@ def _make_fragments(n: int, *, unique_nodes: bool = True) -> list[ContextFragmen
|
||||
class ContentHashDeduplicatorSuite:
|
||||
"""Benchmark ContentHashDeduplicator throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.dedup = ContentHashDeduplicator()
|
||||
@@ -85,8 +86,8 @@ class ContentHashDeduplicatorSuite:
|
||||
class MaxDepthResolverSuite:
|
||||
"""Benchmark MaxDepthResolver throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.resolver = MaxDepthResolver()
|
||||
@@ -112,8 +113,8 @@ class MaxDepthResolverSuite:
|
||||
class WeightedCompositeScorerSuite:
|
||||
"""Benchmark WeightedCompositeScorer throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.scorer = WeightedCompositeScorer()
|
||||
@@ -129,8 +130,8 @@ class WeightedCompositeScorerSuite:
|
||||
class GreedyKnapsackPackerSuite:
|
||||
"""Benchmark GreedyKnapsackPacker throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["fragment_count"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["fragment_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.packer = GreedyKnapsackPacker()
|
||||
|
||||
@@ -14,6 +14,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
@@ -71,8 +72,8 @@ def _make_fragments(
|
||||
class RelevanceCoherenceOrdererSuite:
|
||||
"""Benchmark RelevanceCoherenceOrderer throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.orderer = RelevanceCoherenceOrderer()
|
||||
@@ -85,8 +86,8 @@ class RelevanceCoherenceOrdererSuite:
|
||||
class ProvenancePreambleGeneratorSuite:
|
||||
"""Benchmark ProvenancePreambleGenerator throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.generator = ProvenancePreambleGenerator()
|
||||
@@ -99,8 +100,8 @@ class ProvenancePreambleGeneratorSuite:
|
||||
class ArceStrategySuite:
|
||||
"""Benchmark ArceStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = ArceStrategy(max_iterations=3)
|
||||
@@ -114,8 +115,8 @@ class ArceStrategySuite:
|
||||
class TemporalArchaeologyStrategySuite:
|
||||
"""Benchmark TemporalArchaeologyStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = TemporalArchaeologyStrategy()
|
||||
@@ -129,8 +130,8 @@ class TemporalArchaeologyStrategySuite:
|
||||
class PlanDecisionContextStrategySuite:
|
||||
"""Benchmark PlanDecisionContextStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = PlanDecisionContextStrategy()
|
||||
|
||||
@@ -133,7 +133,7 @@ class AsyncJobTransitionSuite:
|
||||
phase="execute",
|
||||
status=AsyncJobStatus.QUEUED,
|
||||
)
|
||||
job.is_terminal
|
||||
_ = job.is_terminal
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -270,12 +270,12 @@ class CancellationTokenSuite:
|
||||
|
||||
def time_check_not_cancelled(self) -> None:
|
||||
token = CancellationToken()
|
||||
token.is_cancelled
|
||||
_ = token.is_cancelled
|
||||
|
||||
def time_cancel_and_check(self) -> None:
|
||||
token = CancellationToken()
|
||||
token.cancel()
|
||||
token.is_cancelled
|
||||
_ = token.is_cancelled
|
||||
|
||||
def time_reset_token(self) -> None:
|
||||
token = CancellationToken()
|
||||
|
||||
@@ -6,6 +6,8 @@ overhead across varying checkpoint counts.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
from cleveragents.application.services.checkpoint_service import CheckpointService
|
||||
from cleveragents.domain.models.core.checkpoint import CheckpointRetentionPolicy
|
||||
|
||||
@@ -49,8 +51,8 @@ class CheckpointCreateSuite:
|
||||
class CheckpointListSuite:
|
||||
"""Benchmark listing checkpoints at varying plan sizes."""
|
||||
|
||||
params: list[int] = [10, 50, 100]
|
||||
param_names: list[str] = ["checkpoint_count"]
|
||||
params: ClassVar[list[int]] = [10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["checkpoint_count"]
|
||||
|
||||
def setup(self, checkpoint_count: int) -> None:
|
||||
"""Populate service with checkpoints."""
|
||||
@@ -90,8 +92,8 @@ class RollbackSuite:
|
||||
class PruneSuite:
|
||||
"""Benchmark checkpoint pruning at varying counts."""
|
||||
|
||||
params: list[int] = [20, 50, 100]
|
||||
param_names: list[str] = ["checkpoint_count"]
|
||||
params: ClassVar[list[int]] = [20, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["checkpoint_count"]
|
||||
|
||||
def setup(self, checkpoint_count: int) -> None:
|
||||
"""Populate service with checkpoints."""
|
||||
|
||||
@@ -30,7 +30,6 @@ from cleveragents.application.services.component_resolver import ( # noqa: E402
|
||||
ComponentResolver,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test protocol and implementations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
@@ -29,7 +30,6 @@ from cleveragents.domain.models.core.correction import ( # noqa: E402
|
||||
CorrectionMode,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock implementations (minimal, for benchmarking)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -82,8 +82,8 @@ def _make_service(
|
||||
class CascadeCancelSuite:
|
||||
"""Benchmark cascade with cancel-only (not-started) child plans."""
|
||||
|
||||
params = [1, 5, 10, 50, 100]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
self.svc, self.plan_ids = _make_service(
|
||||
@@ -100,8 +100,8 @@ class CascadeCancelSuite:
|
||||
class CascadeRollbackSuite:
|
||||
"""Benchmark cascade with cancel+rollback (in-progress) child plans."""
|
||||
|
||||
params = [1, 5, 10, 50, 100]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
self.svc, self.plan_ids = _make_service(
|
||||
@@ -115,8 +115,8 @@ class CascadeRollbackSuite:
|
||||
class CascadeRejectionSuite:
|
||||
"""Benchmark cascade rejection (applied child plans)."""
|
||||
|
||||
params = [1, 5, 10, 50, 100]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [1, 5, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
self.svc, self.plan_ids = _make_service(
|
||||
@@ -130,8 +130,8 @@ class CascadeRejectionSuite:
|
||||
class CorrectionWithCascadeSuite:
|
||||
"""Benchmark full execute_correction_with_cascade flow."""
|
||||
|
||||
params = [0, 5, 20, 50]
|
||||
param_names = ["child_plan_count"]
|
||||
params: ClassVar[list[int]] = [0, 5, 20, 50]
|
||||
param_names: ClassVar[list[str]] = ["child_plan_count"]
|
||||
|
||||
def setup(self, child_plan_count: int) -> None:
|
||||
plan_ids = [f"CP{i}" for i in range(child_plan_count)]
|
||||
|
||||
@@ -17,7 +17,7 @@ try:
|
||||
from cleveragents.application.services.decision_service import DecisionService
|
||||
from cleveragents.config.settings import Settings
|
||||
from cleveragents.domain.models.core.action import Action, ActionState
|
||||
from cleveragents.domain.models.core.decision import Decision, DecisionType
|
||||
from cleveragents.domain.models.core.decision import DecisionType
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
NamespacedName,
|
||||
Plan,
|
||||
|
||||
@@ -13,11 +13,9 @@ import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
try:
|
||||
from cleveragents.resource.handlers.devcontainer import (
|
||||
DevcontainerHandler,
|
||||
)
|
||||
from cleveragents.resource.handlers.discovery import (
|
||||
DevcontainerDiscoveryResult,
|
||||
discover_devcontainers,
|
||||
@@ -48,8 +46,8 @@ class TimeDiscoveryThroughput:
|
||||
"""Benchmark auto-discovery throughput on projects with many subdirs."""
|
||||
|
||||
timeout = 60
|
||||
params: list[int] = [10, 50, 100]
|
||||
param_names: list[str] = ["num_subdirs"]
|
||||
params: ClassVar[list[int]] = [10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["num_subdirs"]
|
||||
|
||||
def setup(self, num_subdirs: int) -> None:
|
||||
"""Create a temp directory with many subdirectories."""
|
||||
|
||||
@@ -9,6 +9,7 @@ Measures the performance of:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -81,10 +82,8 @@ class ContainerValidationSuite:
|
||||
self.resolver.validate_container_available(self.container_types)
|
||||
|
||||
def time_validate_without_container_exception(self) -> None:
|
||||
try:
|
||||
with contextlib.suppress(ContainerUnavailableError):
|
||||
self.resolver.validate_container_available(self.no_container_types)
|
||||
except ContainerUnavailableError:
|
||||
pass
|
||||
|
||||
def time_resolve_and_validate_host(self) -> None:
|
||||
self.resolver.resolve_and_validate(
|
||||
|
||||
@@ -12,6 +12,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
# Ensure the local source tree is importable
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
@@ -27,8 +28,12 @@ from cleveragents.actor.reconciliation import ( # noqa: E402
|
||||
ScopeInvariants,
|
||||
reconcile_invariants,
|
||||
)
|
||||
from cleveragents.application.services.decision_service import DecisionService # noqa: E402
|
||||
from cleveragents.application.services.invariant_service import InvariantService # noqa: E402
|
||||
from cleveragents.application.services.decision_service import ( # noqa: E402
|
||||
DecisionService,
|
||||
)
|
||||
from cleveragents.application.services.invariant_service import ( # noqa: E402
|
||||
InvariantService,
|
||||
)
|
||||
from cleveragents.domain.models.core.invariant import ( # noqa: E402
|
||||
Invariant,
|
||||
InvariantScope,
|
||||
@@ -86,8 +91,8 @@ def _build_conflicting_scope_invariants(n_conflicts: int) -> ScopeInvariants:
|
||||
class ReconciliationAlgorithmSuite:
|
||||
"""Benchmark the pure reconciliation algorithm (no I/O)."""
|
||||
|
||||
params = [1, 10, 50, 100]
|
||||
param_names = ["invariants_per_scope"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["invariants_per_scope"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._scope_invs = _build_scope_invariants(
|
||||
@@ -102,8 +107,8 @@ class ReconciliationAlgorithmSuite:
|
||||
class ConflictResolutionSuite:
|
||||
"""Benchmark conflict detection and resolution."""
|
||||
|
||||
params = [1, 10, 50, 100]
|
||||
param_names = ["conflict_count"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["conflict_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._scope_invs = _build_conflicting_scope_invariants(n)
|
||||
@@ -116,8 +121,8 @@ class ConflictResolutionSuite:
|
||||
class FullPipelineSuite:
|
||||
"""Benchmark the full actor pipeline (collect + reconcile + record)."""
|
||||
|
||||
params = [1, 10, 50]
|
||||
param_names = ["invariants_per_scope"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50]
|
||||
param_names: ClassVar[list[str]] = ["invariants_per_scope"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._n = n
|
||||
@@ -153,8 +158,8 @@ class FullPipelineSuite:
|
||||
class DecisionRecordingSuite:
|
||||
"""Benchmark decision recording throughput."""
|
||||
|
||||
params = [1, 10, 50, 100]
|
||||
param_names = ["invariant_count"]
|
||||
params: ClassVar[list[int]] = [1, 10, 50, 100]
|
||||
param_names: ClassVar[list[str]] = ["invariant_count"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self._n = n
|
||||
|
||||
@@ -9,7 +9,6 @@ from __future__ import annotations
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
|
||||
@@ -7,6 +7,7 @@ Measures the overhead of permission checks in both local mode
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import ClassVar
|
||||
|
||||
from cleveragents.application.services.permission_service import (
|
||||
PermissionService,
|
||||
@@ -65,8 +66,8 @@ class TimePermissionCheckLocal:
|
||||
class TimePermissionCheckServer:
|
||||
"""Benchmark permission checks in server mode."""
|
||||
|
||||
params: list[int] = [0, 10, 100]
|
||||
param_names: list[str] = ["num_bindings"]
|
||||
params: ClassVar[list[int]] = [0, 10, 100]
|
||||
param_names: ClassVar[list[str]] = ["num_bindings"]
|
||||
|
||||
def setup(self, num_bindings: int) -> None:
|
||||
os.environ[_SERVER_MODE_ENV] = "true"
|
||||
|
||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import ClassVar
|
||||
|
||||
try:
|
||||
from cleveragents.application.services.correction_service import CorrectionService
|
||||
@@ -59,8 +60,8 @@ def _build_tree_from_decisions(
|
||||
class TreeBuildingSuite:
|
||||
"""Benchmark decision tree adjacency-list construction."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["chain_depth"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["chain_depth"]
|
||||
|
||||
def setup(self, chain_depth: int) -> None:
|
||||
"""Prepare decision chains."""
|
||||
@@ -74,8 +75,8 @@ class TreeBuildingSuite:
|
||||
class AnalyzeWithTreeSuite:
|
||||
"""Benchmark analyze_impact when decision_tree is pre-built."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["chain_depth"]
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["chain_depth"]
|
||||
|
||||
def setup(self, chain_depth: int) -> None:
|
||||
"""Prepare CorrectionService and tree."""
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""ASV benchmarks for Layer 4 — Predictive Error Prevention.
|
||||
|
||||
Measures pattern matching performance with large databases,
|
||||
recording throughput, and guidance generation latency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cleveragents.application.services.error_pattern_service import (
|
||||
ErrorPatternService,
|
||||
)
|
||||
from cleveragents.infrastructure.database.error_pattern_repository import (
|
||||
ErrorPatternRepository,
|
||||
)
|
||||
|
||||
|
||||
def _make_service(num_patterns: int = 0) -> ErrorPatternService:
|
||||
"""Create a service and optionally seed it with patterns."""
|
||||
repo = ErrorPatternRepository()
|
||||
service = ErrorPatternService(repository=repo)
|
||||
for i in range(num_patterns):
|
||||
service.record_failure(
|
||||
pattern_text=f"Error pattern number {i} in module_{i % 10}",
|
||||
failure_description=f"Failure description {i}",
|
||||
preventive_checks=(f"Check {i}-a", f"Check {i}-b"),
|
||||
keywords=(f"pattern{i}", f"module_{i % 10}", "error"),
|
||||
)
|
||||
return service
|
||||
|
||||
|
||||
class TimeRecordSingle:
|
||||
"""Benchmark recording a single failure pattern."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service()
|
||||
|
||||
def time_record_single_failure(self) -> None:
|
||||
self.service.record_failure(
|
||||
pattern_text="Benchmark failure pattern",
|
||||
failure_description="Test failure",
|
||||
preventive_checks=("Check A", "Check B"),
|
||||
keywords=("benchmark", "failure"),
|
||||
)
|
||||
|
||||
|
||||
class TimeRecordBatch:
|
||||
"""Benchmark recording a batch of failure patterns."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service()
|
||||
|
||||
def time_record_100_failures(self) -> None:
|
||||
for i in range(100):
|
||||
self.service.record_failure(
|
||||
pattern_text=f"Batch failure pattern {i}",
|
||||
failure_description=f"Failure {i}",
|
||||
preventive_checks=(f"Check {i}",),
|
||||
keywords=(f"batch{i}",),
|
||||
)
|
||||
|
||||
|
||||
class TimeMatchSmallDB:
|
||||
"""Benchmark pattern matching against a small database (10 patterns)."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=10)
|
||||
|
||||
def time_match_context_small(self) -> None:
|
||||
self.service.match_patterns("We need to fix error pattern number 5 in module_5")
|
||||
|
||||
|
||||
class TimeMatchMediumDB:
|
||||
"""Benchmark pattern matching against a medium database (100 patterns)."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=100)
|
||||
|
||||
def time_match_context_medium(self) -> None:
|
||||
self.service.match_patterns(
|
||||
"We need to fix error pattern number 50 in module_5"
|
||||
)
|
||||
|
||||
|
||||
class TimeMatchLargeDB:
|
||||
"""Benchmark pattern matching against a large database (1000 patterns)."""
|
||||
|
||||
timeout = 120
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=1000)
|
||||
|
||||
def time_match_context_large(self) -> None:
|
||||
self.service.match_patterns(
|
||||
"We need to fix error pattern number 500 in module_5"
|
||||
)
|
||||
|
||||
def time_match_no_results_large(self) -> None:
|
||||
self.service.match_patterns(
|
||||
"Completely unrelated context about cooking recipes"
|
||||
)
|
||||
|
||||
|
||||
class TimeGuidanceFormatting:
|
||||
"""Benchmark guidance formatting after matching."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=50)
|
||||
self.guidance = self.service.match_patterns("We need to fix error in module_5")
|
||||
|
||||
def time_format_guidance(self) -> None:
|
||||
self.guidance.format_for_context()
|
||||
|
||||
|
||||
class TimeStatistics:
|
||||
"""Benchmark statistics computation."""
|
||||
|
||||
timeout = 30
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service(num_patterns=500)
|
||||
|
||||
def time_get_statistics(self) -> None:
|
||||
self.service.get_statistics()
|
||||
|
||||
def time_list_patterns(self) -> None:
|
||||
self.service.list_patterns()
|
||||
|
||||
|
||||
class TimeDuplicateRecording:
|
||||
"""Benchmark recording duplicate patterns (frequency updates)."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
self.service = _make_service()
|
||||
self.service.record_failure(
|
||||
pattern_text="Recurring failure",
|
||||
failure_description="Initial occurrence",
|
||||
preventive_checks=("Check A",),
|
||||
keywords=("recurring",),
|
||||
)
|
||||
|
||||
def time_record_duplicate_100_times(self) -> None:
|
||||
for i in range(100):
|
||||
self.service.record_failure(
|
||||
pattern_text="Recurring failure",
|
||||
failure_description=f"Occurrence {i + 2}",
|
||||
)
|
||||
@@ -19,7 +19,6 @@ try:
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_profile import (
|
||||
BUILTIN_PROFILES,
|
||||
AutomationProfile,
|
||||
)
|
||||
from cleveragents.domain.models.core.escalation import (
|
||||
ConfidenceFactors,
|
||||
@@ -33,7 +32,6 @@ except ModuleNotFoundError:
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_profile import (
|
||||
BUILTIN_PROFILES,
|
||||
AutomationProfile,
|
||||
)
|
||||
from cleveragents.domain.models.core.escalation import (
|
||||
ConfidenceFactors,
|
||||
|
||||
@@ -16,7 +16,7 @@ _SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from cleveragents.application.services.semantic_validation_rules import (
|
||||
from cleveragents.application.services.semantic_validation_rules import ( # noqa: E402
|
||||
APIMisuseRule,
|
||||
BrokenReferenceRule,
|
||||
DuplicateImportRule,
|
||||
@@ -26,7 +26,7 @@ from cleveragents.application.services.semantic_validation_rules import (
|
||||
SemanticValidationSeverity,
|
||||
SyntaxCheckRule,
|
||||
)
|
||||
from cleveragents.application.services.semantic_validation_service import (
|
||||
from cleveragents.application.services.semantic_validation_service import ( # noqa: E402
|
||||
SemanticValidationCache,
|
||||
SemanticValidationService,
|
||||
create_default_registry,
|
||||
|
||||
@@ -24,8 +24,14 @@ importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
AssembledContext as CRPAssembledContext,
|
||||
)
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
ContextBudget as CRPContextBudget,
|
||||
)
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
ContextFragment as CRPContextFragment,
|
||||
)
|
||||
from cleveragents.domain.models.acms.crp import ( # noqa: E402
|
||||
FragmentProvenance as CRPFragmentProvenance,
|
||||
)
|
||||
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
|
||||
|
||||
Reference in New Issue
Block a user