cleveragents-core/benchmarks/semantic_validation_suite_bench.py

"""ASV benchmarks for semantic validation suite runtime.

Measures the performance of running entire fixture suites through the
semantic validation rules, covering:
- Language-porting mismatch fixtures
- Dependency graph violation fixtures
- API surface change fixtures
- Cross-file symbol resolution fixtures
- Duplicate relative import detection fixtures

Uses :mod:`features.fixtures.validation.suite_helpers` for shared
loading, lookup, and rule-map construction.
"""

from __future__ import annotations

import sys
from pathlib import Path
from typing import Any

_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

_FIXTURES_DIR = str(
    Path(__file__).resolve().parents[1] / "features" / "fixtures" / "validation"
)
if _FIXTURES_DIR not in sys.path:
    sys.path.insert(0, _FIXTURES_DIR)

from suite_helpers import (  # noqa: E402, I001
    ALL_FIXTURE_FILENAMES,
    API_FILENAME,
    CIRCULAR_FILENAME,
    CROSS_FILE_FILENAME,
    DEPENDENCY_FILENAME,
    PORTING_FILENAME,
    RULE_MAP,
    load_fixtures,
)

from cleveragents.application.services.semantic_validation_rules import (  # noqa: E402
    SemanticCheckResult,
)
from cleveragents.application.services.semantic_validation_service import (  # noqa: E402
    SemanticValidationService,
)


# ---------------------------------------------------------------------------
# Fixture runner (uses shared RULE_MAP)
# ---------------------------------------------------------------------------


def _run_suite(fixtures: list[dict[str, Any]]) -> list[SemanticCheckResult]:
    """Run all fixtures through their expected rules."""
    results: list[SemanticCheckResult] = []
    for fixture in fixtures:
        rule_name: str = fixture["expected_rule"]
        rule = RULE_MAP[rule_name]
        result = rule.check(fixture["source"], fixture["filename"])
        results.append(result)
    return results


# ---------------------------------------------------------------------------
# Benchmark suites
# ---------------------------------------------------------------------------


class PortingFixtureSuite:
    """Benchmarks for language-porting mismatch fixture suite runtime."""

    timeout = 60

    def setup(self) -> None:
        self.fixtures = load_fixtures(PORTING_FILENAME)

    def time_run_all_porting_fixtures(self) -> None:
        _run_suite(self.fixtures)

    def time_load_porting_fixtures(self) -> None:
        load_fixtures(PORTING_FILENAME)


class DependencyFixtureSuite:
    """Benchmarks for dependency graph violation fixture suite runtime."""

    timeout = 60

    def setup(self) -> None:
        self.fixtures = load_fixtures(DEPENDENCY_FILENAME)

    def time_run_all_dependency_fixtures(self) -> None:
        _run_suite(self.fixtures)

    def time_load_dependency_fixtures(self) -> None:
        load_fixtures(DEPENDENCY_FILENAME)


class APISurfaceFixtureSuite:
    """Benchmarks for API surface change fixture suite runtime."""

    timeout = 60

    def setup(self) -> None:
        self.fixtures = load_fixtures(API_FILENAME)

    def time_run_all_api_fixtures(self) -> None:
        _run_suite(self.fixtures)

    def time_load_api_fixtures(self) -> None:
        load_fixtures(API_FILENAME)


class CrossFileFixtureSuite:
    """Benchmarks for cross-file symbol resolution fixture suite runtime."""

    timeout = 60

    def setup(self) -> None:
        self.fixtures = load_fixtures(CROSS_FILE_FILENAME)

    def time_run_all_cross_file_fixtures(self) -> None:
        _run_suite(self.fixtures)

    def time_load_cross_file_fixtures(self) -> None:
        load_fixtures(CROSS_FILE_FILENAME)


class CircularImportFixtureSuite:
    """Benchmarks for duplicate relative import fixture suite runtime."""

    timeout = 60

    def setup(self) -> None:
        self.fixtures = load_fixtures(CIRCULAR_FILENAME)

    def time_run_all_circular_fixtures(self) -> None:
        _run_suite(self.fixtures)

    def time_load_circular_fixtures(self) -> None:
        load_fixtures(CIRCULAR_FILENAME)


class FullSuiteRuntime:
    """Benchmarks for running all fixture suites together."""

    timeout = 120

    def setup(self) -> None:
        self.all_fixtures: list[dict[str, Any]] = []
        for filename in ALL_FIXTURE_FILENAMES:
            self.all_fixtures.extend(load_fixtures(filename))
        # Pre-create service in setup so timing only measures validation
        self.service = SemanticValidationService()

    def time_run_all_suites(self) -> None:
        _run_suite(self.all_fixtures)

    def time_service_check_all_sources(self) -> None:
        self.service.cache.clear()
        for fixture in self.all_fixtures:
            self.service.check_file(fixture["source"], fixture["filename"])