cleveragents-core/benchmarks/semantic_validation_bench.py

"""ASV benchmarks for the semantic validation service.

Measures the performance of:
- Individual rule checks (syntax, import, reference, cycle, misuse, symbol)
- Rule registry operations (register, lookup, list)
- Validation cache operations (put, get, compute_hash)
- Service orchestration (check_file, as_pipeline_results)
"""

from __future__ import annotations

import sys
from pathlib import Path

_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

from cleveragents.application.services.semantic_validation_rules import (  # noqa: E402
    APIMisuseRule,
    BrokenReferenceRule,
    DuplicateImportRule,
    MissingImportRule,
    MissingSymbolRule,
    SemanticCheckResult,
    SemanticValidationSeverity,
    SyntaxCheckRule,
)
from cleveragents.application.services.semantic_validation_service import (  # noqa: E402
    SemanticValidationCache,
    SemanticValidationService,
    create_default_registry,
)

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------

VALID_PYTHON = """\
import os
import sys
import json

x = 10

def hello():
    return x

class Greeter:
    def greet(self):
        return hello()

def compute():
    return x + 1
"""

LARGE_PYTHON = VALID_PYTHON * 50  # ~800 lines


# ---------------------------------------------------------------------------
# Rule benchmark suites
# ---------------------------------------------------------------------------


class SyntaxCheckRuleSuite:
    """Benchmarks for SyntaxCheckRule."""

    timeout = 60

    def setup(self) -> None:
        self.rule = SyntaxCheckRule()
        self.source = VALID_PYTHON
        self.large = LARGE_PYTHON

    def time_syntax_check_small(self) -> None:
        self.rule.check(self.source, "test.py")

    def time_syntax_check_large(self) -> None:
        self.rule.check(self.large, "test.py")


class MissingImportRuleSuite:
    """Benchmarks for MissingImportRule."""

    timeout = 60

    def setup(self) -> None:
        self.rule = MissingImportRule()
        self.source = VALID_PYTHON

    def time_import_check(self) -> None:
        self.rule.check(self.source, "test.py")


class BrokenReferenceRuleSuite:
    """Benchmarks for BrokenReferenceRule."""

    timeout = 60

    def setup(self) -> None:
        self.rule = BrokenReferenceRule()
        self.source = VALID_PYTHON

    def time_reference_check(self) -> None:
        self.rule.check(self.source, "test.py")


class APIMisuseRuleSuite:
    """Benchmarks for APIMisuseRule."""

    timeout = 60

    def setup(self) -> None:
        self.rule = APIMisuseRule()
        self.source = VALID_PYTHON

    def time_api_misuse_check(self) -> None:
        self.rule.check(self.source, "test.py")


class DuplicateImportRuleSuite:
    """Benchmarks for DuplicateImportRule."""

    timeout = 60

    def setup(self) -> None:
        self.rule = DuplicateImportRule()
        self.source = VALID_PYTHON

    def time_duplicate_import_check(self) -> None:
        self.rule.check(self.source, "test.py")


class MissingSymbolRuleSuite:
    """Benchmarks for MissingSymbolRule."""

    timeout = 60

    def setup(self) -> None:
        self.rule = MissingSymbolRule()
        self.source = VALID_PYTHON

    def time_symbol_check(self) -> None:
        self.rule.check(self.source, "test.py")


# ---------------------------------------------------------------------------
# Registry benchmarks
# ---------------------------------------------------------------------------


class RegistrySuite:
    """Benchmarks for SemanticRuleRegistry."""

    timeout = 60

    def setup(self) -> None:
        self.registry = create_default_registry()

    def time_list_rules(self) -> None:
        self.registry.list_rules()

    def time_lookup_rule(self) -> None:
        self.registry.get("syntax_error")

    def time_all_rules(self) -> None:
        self.registry.all_rules()


# ---------------------------------------------------------------------------
# Cache benchmarks
# ---------------------------------------------------------------------------


class CacheSuite:
    """Benchmarks for SemanticValidationCache."""

    timeout = 60

    def setup(self) -> None:
        self.cache = SemanticValidationCache()
        self.result = SemanticCheckResult(
            passed=True,
            message="ok",
            severity=SemanticValidationSeverity.INFO,
        )
        self.cache.put("rule", "hash1", self.result)

    def time_compute_hash(self) -> None:
        SemanticValidationCache.compute_hash(VALID_PYTHON)

    def time_cache_get_hit(self) -> None:
        self.cache.get("rule", "hash1")

    def time_cache_get_miss(self) -> None:
        self.cache.get("rule", "missing")

    def time_cache_put(self) -> None:
        self.cache.put("rule", "hash2", self.result)


# ---------------------------------------------------------------------------
# Service benchmarks
# ---------------------------------------------------------------------------


class ServiceSuite:
    """Benchmarks for SemanticValidationService orchestration."""

    timeout = 120

    def setup(self) -> None:
        self.service = SemanticValidationService()
        self.source = VALID_PYTHON
        self.large = LARGE_PYTHON

    def time_check_file_small(self) -> None:
        # Clear cache to measure full check cost
        self.service.cache.clear()
        self.service.check_file(self.source, "test.py")

    def time_check_file_large(self) -> None:
        self.service.cache.clear()
        self.service.check_file(self.large, "test.py")

    def time_check_file_cached(self) -> None:
        # First call populates cache, second uses it
        self.service.check_file(self.source, "test.py")
        self.service.check_file(self.source, "test.py")

    def time_pipeline_results(self) -> None:
        self.service.cache.clear()
        self.service.as_pipeline_results(self.source, "test.py")