cleveragents-core/benchmarks/predictive_error_prevention_bench.py

"""ASV benchmarks for Layer 4 — Predictive Error Prevention.

Measures pattern matching performance with large databases,
recording throughput, and guidance generation latency.
"""

from __future__ import annotations

from cleveragents.application.services.error_pattern_service import (
    ErrorPatternService,
)
from cleveragents.infrastructure.database.error_pattern_repository import (
    ErrorPatternRepository,
)


def _make_service(num_patterns: int = 0) -> ErrorPatternService:
    """Create a service and optionally seed it with patterns."""
    repo = ErrorPatternRepository()
    service = ErrorPatternService(repository=repo)
    for i in range(num_patterns):
        service.record_failure(
            pattern_text=f"Error pattern number {i} in module_{i % 10}",
            failure_description=f"Failure description {i}",
            preventive_checks=(f"Check {i}-a", f"Check {i}-b"),
            keywords=(f"pattern{i}", f"module_{i % 10}", "error"),
        )
    return service


class TimeRecordSingle:
    """Benchmark recording a single failure pattern."""

    timeout = 30

    def setup(self) -> None:
        self.service = _make_service()

    def time_record_single_failure(self) -> None:
        self.service.record_failure(
            pattern_text="Benchmark failure pattern",
            failure_description="Test failure",
            preventive_checks=("Check A", "Check B"),
            keywords=("benchmark", "failure"),
        )


class TimeRecordBatch:
    """Benchmark recording a batch of failure patterns."""

    timeout = 60

    def setup(self) -> None:
        self.service = _make_service()

    def time_record_100_failures(self) -> None:
        for i in range(100):
            self.service.record_failure(
                pattern_text=f"Batch failure pattern {i}",
                failure_description=f"Failure {i}",
                preventive_checks=(f"Check {i}",),
                keywords=(f"batch{i}",),
            )


class TimeMatchSmallDB:
    """Benchmark pattern matching against a small database (10 patterns)."""

    timeout = 30

    def setup(self) -> None:
        self.service = _make_service(num_patterns=10)

    def time_match_context_small(self) -> None:
        self.service.match_patterns("We need to fix error pattern number 5 in module_5")


class TimeMatchMediumDB:
    """Benchmark pattern matching against a medium database (100 patterns)."""

    timeout = 60

    def setup(self) -> None:
        self.service = _make_service(num_patterns=100)

    def time_match_context_medium(self) -> None:
        self.service.match_patterns(
            "We need to fix error pattern number 50 in module_5"
        )


class TimeMatchLargeDB:
    """Benchmark pattern matching against a large database (1000 patterns)."""

    timeout = 120

    def setup(self) -> None:
        self.service = _make_service(num_patterns=1000)

    def time_match_context_large(self) -> None:
        self.service.match_patterns(
            "We need to fix error pattern number 500 in module_5"
        )

    def time_match_no_results_large(self) -> None:
        self.service.match_patterns(
            "Completely unrelated context about cooking recipes"
        )


class TimeGuidanceFormatting:
    """Benchmark guidance formatting after matching."""

    timeout = 30

    def setup(self) -> None:
        self.service = _make_service(num_patterns=50)
        self.guidance = self.service.match_patterns("We need to fix error in module_5")

    def time_format_guidance(self) -> None:
        self.guidance.format_for_context()


class TimeStatistics:
    """Benchmark statistics computation."""

    timeout = 30

    def setup(self) -> None:
        self.service = _make_service(num_patterns=500)

    def time_get_statistics(self) -> None:
        self.service.get_statistics()

    def time_list_patterns(self) -> None:
        self.service.list_patterns()


class TimeDuplicateRecording:
    """Benchmark recording duplicate patterns (frequency updates)."""

    timeout = 60

    def setup(self) -> None:
        self.service = _make_service()
        self.service.record_failure(
            pattern_text="Recurring failure",
            failure_description="Initial occurrence",
            preventive_checks=("Check A",),
            keywords=("recurring",),
        )

    def time_record_duplicate_100_times(self) -> None:
        for i in range(100):
            self.service.record_failure(
                pattern_text="Recurring failure",
                failure_description=f"Occurrence {i + 2}",
            )