cleveragents-core/benchmarks/cli_init_yes_bench.py

"""ASV benchmarks for ``agents init --yes`` invocation time.

Measures in-process execution time for the init command with the --yes flag,
which should perform non-interactive initialization using defaults.

These benchmarks target bug #522 and are expected to error until the fix
is applied (the --yes flag does not yet exist).
"""

from __future__ import annotations

import shutil
import sys
import tempfile
from pathlib import Path
from unittest.mock import create_autospec, patch

try:
    from cleveragents.cli.main import app
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.cli.main import app

from typer.testing import CliRunner

from cleveragents.application.services.project_service import ProjectService


class _MockProject:
    """Lightweight stand-in for the Project model with spec fields.

    We cannot use ``create_autospec(Project)`` because the spec-required
    output fields (``data_dir``, ``config_path``, ``database_status``,
    ``directories``) do not yet exist on the legacy ``Project`` model.
    """

    name: str
    path: Path
    data_dir: Path
    config_path: Path
    database_status: str
    directories: list[str]


class InitYesFlagSuite:
    """Benchmark ``agents init --yes`` invocation."""

    timeout = 30.0

    def setup(self) -> None:
        self._runner = CliRunner()
        self._tmpdir = tempfile.mkdtemp()
        self._last_exit_code = -1

    def teardown(self) -> None:
        shutil.rmtree(self._tmpdir, ignore_errors=True)

    def _invoke_with_mock(self, args: list[str], project_name: str) -> None:
        """Invoke the CLI with a mocked container and the given *args*.

        Stores the result exit code so benchmark data is only meaningful
        when the command succeeds (exit code 0).  When ``--yes`` is not yet
        implemented the benchmark measures error-path latency; after the
        fix it measures real init latency.
        """
        with patch(
            "cleveragents.application.container.get_container"
        ) as mock_container:
            mock_service = create_autospec(ProjectService, instance=True)
            mock_project = _MockProject()
            mock_project.name = project_name
            mock_project.path = Path(self._tmpdir)
            # Pre-populate spec-required fields (specification.md:1381-1386)
            # so benchmarks measure real output paths once #522 is fixed.
            mock_project.data_dir = Path(self._tmpdir)
            mock_project.config_path = Path(self._tmpdir) / "config.toml"
            mock_project.database_status = "initialized (schema v3)"
            mock_project.directories = ["logs", "cache", "sessions", "contexts"]
            mock_service.initialize_project.return_value = mock_project
            mock_container.return_value.project_service.return_value = mock_service

            result = self._runner.invoke(app, args)
            self._last_exit_code = result.exit_code

    def time_init_yes_flag(self) -> None:
        """Measure end-to-end latency of ``agents init --yes``."""
        self._invoke_with_mock(["init", "--yes"], "bench-project")

    def time_init_short_y_flag(self) -> None:
        """Measure end-to-end latency of ``agents init -y``."""
        self._invoke_with_mock(["init", "-y"], "bench-project")

    def time_init_yes_flag_with_path(self) -> None:
        """Measure latency of ``agents init --yes --path <dir>``.

        NOTE: ``--path`` is an implementation detail of the current ``init``
        command.  The spec (``specification.md:1217``) defines only
        ``agents init [--yes|-y]`` with no ``--path`` option.  When #522
        aligns the command with the spec this benchmark may need to be
        removed or updated.
        """
        self._invoke_with_mock(
            ["init", "--yes", "--path", self._tmpdir], "bench-path-project"
        )

    def track_exit_code(self) -> int:
        """Track the CLI exit code as an ASV metric.

        Returns 0 when ``--yes`` is correctly implemented; non-zero while
        the flag is missing.  This lets ASV detect regressions that
        reintroduce ``NoSuchOption`` without silently reporting error-path
        latency.
        """
        self._invoke_with_mock(["init", "--yes"], "bench-project")
        return self._last_exit_code


InitYesFlagSuite.track_exit_code.unit = "exit_code"