cleveragents-core/benchmarks/context_indexing_bench.py

"""ASV benchmarks for repo indexing service (issue #195).

Measures indexing throughput, incremental refresh overhead, and
language detection performance.
"""

from __future__ import annotations

import shutil
import sys
import tempfile
from pathlib import Path

_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

from sqlalchemy import create_engine  # noqa: E402
from sqlalchemy.orm import sessionmaker  # noqa: E402

import cleveragents  # noqa: E402, F401
from cleveragents.application.services.repo_indexing_service import (  # noqa: E402
    RepoIndexingService,
    detect_language,
    estimate_tokens,
)
from cleveragents.infrastructure.database.models import Base  # noqa: E402

_RID = "01KK0D8WNATFNEX2JMG5GKF6FP"


def _make_service() -> RepoIndexingService:
    engine = create_engine("sqlite:///:memory:", echo=False)
    Base.metadata.create_all(engine)
    factory = sessionmaker(bind=engine, expire_on_commit=False)
    return RepoIndexingService(session_factory=factory)


class TimeRepoIndexing:
    """Benchmark repo indexing throughput."""

    timeout = 120

    def setup(self) -> None:
        """Create a temporary project with 100 files."""
        self.svc = _make_service()
        self.tmpdir = tempfile.mkdtemp()
        for i in range(50):
            (Path(self.tmpdir) / f"module_{i:03d}.py").write_text(
                f"# Module {i}\ndef func_{i}():\n    return {i}\n" * 10
            )
        for i in range(30):
            (Path(self.tmpdir) / f"component_{i:03d}.ts").write_text(
                f"export const comp{i} = () => {{}};\n" * 10
            )
        for i in range(20):
            (Path(self.tmpdir) / f"doc_{i:03d}.md").write_text(
                f"# Doc {i}\n\nContent for document {i}.\n" * 5
            )

    def teardown(self) -> None:
        shutil.rmtree(self.tmpdir, ignore_errors=True)

    def time_full_index_100_files(self) -> None:
        """Full index of 100 files."""
        self.svc.index_resource(_RID, self.tmpdir)

    def time_incremental_refresh_no_changes(self) -> None:
        """Incremental refresh when no files changed (100 files)."""
        # Pre-index to ensure an index exists, then measure refresh only.
        self.svc.index_resource(_RID, self.tmpdir)
        self.svc.refresh_index(_RID, self.tmpdir)

    def time_refresh_only_no_changes(self) -> None:
        """Pure refresh measurement (index already exists).

        Note: ASV times the entire method body including the pre-index
        call.  For a purer measurement, use ``setup`` to pre-index and
        only call ``refresh_index`` in the timed body.
        """
        self.svc.index_resource(_RID, self.tmpdir)
        self.svc.refresh_index(_RID, self.tmpdir)

    def time_language_detection_1000(self) -> None:
        """Language detection for 1000 file paths."""
        paths = (
            [f"module_{i}.py" for i in range(250)]
            + [f"component_{i}.ts" for i in range(250)]
            + [f"doc_{i}.md" for i in range(250)]
            + [f"data_{i}.json" for i in range(250)]
        )
        for p in paths:
            detect_language(p)

    def time_token_estimation_1000(self) -> None:
        """Token estimation for 1000 sizes."""
        for size in range(1000):
            estimate_tokens(size * 100)


class TrackRepoIndexing:
    """Track metrics for repo indexing."""

    timeout = 120

    def setup(self) -> None:
        self.svc = _make_service()
        self.tmpdir = tempfile.mkdtemp()
        for i in range(50):
            (Path(self.tmpdir) / f"module_{i:03d}.py").write_text(
                f"def func_{i}():\n    return {i}\n" * 10
            )

    def teardown(self) -> None:
        shutil.rmtree(self.tmpdir, ignore_errors=True)

    def track_file_count(self) -> int:
        """Track number of files indexed."""
        idx = self.svc.index_resource(_RID, self.tmpdir)
        return idx.metadata.file_count

    track_file_count.unit = "files"  # type: ignore[attr-defined]

    def track_token_estimate(self) -> int:
        """Track total token estimate."""
        idx = self.svc.index_resource(_RID, self.tmpdir)
        return idx.metadata.token_estimate

    track_token_estimate.unit = "tokens"  # type: ignore[attr-defined]


class TimeRepoIndexingLarge:
    """Benchmark indexing throughput for 10,000-file repositories."""

    timeout = 1200

    def setup(self) -> None:
        """Create a temporary project with exactly 10,000 Python files."""
        self.svc = _make_service()
        self.tmpdir = tempfile.mkdtemp()
        root = Path(self.tmpdir)
        for i in range(100):
            module_dir = root / f"module_{i:03d}"
            module_dir.mkdir(parents=True, exist_ok=True)
            for j in range(100):
                (module_dir / f"file_{j:03d}.py").write_text(
                    f"def f_{i}_{j}():\n    return {i + j}\n"
                )

    def teardown(self) -> None:
        shutil.rmtree(self.tmpdir, ignore_errors=True)

    def time_full_index_10000_files(self) -> None:
        """Full index for a 10,000-file repository."""
        self.svc.index_resource(_RID, self.tmpdir, timeout_seconds=120.0)