cleveragents-core/benchmarks/cli_extension_tests_bench.py

"""ASV benchmarks for CLI extension test scenario runtime baselines.

Benchmarks the extended test scenarios added in #326:
- Automation profile resolution across all builtin profiles
- Invariant ordering verification with multiple flags
- Actor override validation (valid and error paths)
- Output format rendering (JSON, YAML, table) with extended fields
"""

from __future__ import annotations

import importlib
import sys
from datetime import datetime
from pathlib import Path
from unittest.mock import MagicMock, patch

# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

import cleveragents  # noqa: E402

importlib.reload(cleveragents)

from typer.testing import CliRunner  # noqa: E402

from cleveragents.cli.commands.action import app as action_app  # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app  # noqa: E402
from cleveragents.cli.commands.plan import (  # noqa: E402
    validate_namespaced_actor,
)
from cleveragents.core.exceptions import ValidationError  # noqa: E402
from cleveragents.domain.models.core.action import (  # noqa: E402
    Action,
    ActionState,
)
from cleveragents.domain.models.core.plan import (  # noqa: E402
    AutomationProfileProvenance,
    AutomationProfileRef,
    InvariantSource,
    NamespacedName,
    Plan,
    PlanIdentity,
    PlanInvariant,
    PlanPhase,
    PlanTimestamps,
    ProcessingState,
    ProjectLink,
)

_runner = CliRunner()
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"

_BUILTIN_PROFILE_NAMES = [
    "manual",
    "review",
    "supervised",
    "cautious",
    "trusted",
    "auto",
]


def _mock_action(name: str = "local/bench-action") -> Action:
    return Action(
        namespaced_name=NamespacedName.parse(name),
        description="Benchmark action",
        long_description=None,
        definition_of_done="Benchmarks pass",
        strategy_actor="openai/gpt-4",
        execution_actor="openai/gpt-4",
        estimation_actor="openai/gpt-4",
        invariant_actor="anthropic/claude-3",
        reusable=True,
        read_only=False,
        invariants=["No regressions", "Keep backward compat"],
        inputs_schema={
            "type": "object",
            "properties": {"coverage": {"type": "integer"}},
        },
        state=ActionState.AVAILABLE,
        created_by=None,
        created_at=datetime.now(),
        updated_at=datetime.now(),
    )


def _mock_plan(
    name: str = "local/bench-plan",
    *,
    with_profile: bool = False,
    with_invariants: bool = False,
    invariant_count: int = 2,
) -> Plan:
    now = datetime.now()
    profile = None
    if with_profile:
        profile = AutomationProfileRef(
            profile_name="trusted",
            provenance=AutomationProfileProvenance.PLAN,
        )
    invariants: list[PlanInvariant] = []
    if with_invariants:
        invariants = [
            PlanInvariant(
                text=f"Invariant {i}",
                source=InvariantSource.PLAN,
            )
            for i in range(invariant_count)
        ]
    return Plan(
        identity=PlanIdentity(plan_id=_PLAN_ULID),
        namespaced_name=NamespacedName.parse(name),
        description="Benchmark plan",
        definition_of_done="Benchmarks pass",
        action_name="local/bench-action",
        phase=PlanPhase.STRATEGIZE,
        processing_state=ProcessingState.QUEUED,
        project_links=[ProjectLink(project_name="my-project")],
        arguments={},
        arguments_order=[],
        automation_profile=profile,
        invariants=invariants,
        strategy_actor="openai/gpt-4",
        execution_actor="openai/gpt-4",
        reusable=True,
        read_only=False,
        created_by=None,
        timestamps=PlanTimestamps(created_at=now, updated_at=now),
    )


class ProfileResolutionSuite:
    """Benchmark automation profile resolution across all builtin profiles."""

    def setup(self) -> None:
        self._mock_service = MagicMock()
        self._mock_service.get_action_by_name.return_value = _mock_action()
        self._mock_service.use_action.return_value = _mock_plan(with_profile=True)
        self._patcher = patch(
            "cleveragents.cli.commands.plan._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._patcher.start()

    def teardown(self) -> None:
        self._patcher.stop()

    def time_resolve_all_builtin_profiles(self) -> None:
        """Benchmark resolving each builtin profile in sequence."""
        for profile_name in _BUILTIN_PROFILE_NAMES:
            _runner.invoke(
                plan_app,
                [
                    "use",
                    "local/bench-action",
                    "--automation-profile",
                    profile_name,
                ],
            )

    def time_resolve_single_profile(self) -> None:
        """Benchmark resolving a single profile."""
        _runner.invoke(
            plan_app,
            ["use", "local/bench-action", "--automation-profile", "trusted"],
        )

    def time_reject_invalid_profile(self) -> None:
        """Benchmark rejecting an invalid profile name."""
        _runner.invoke(
            plan_app,
            ["use", "local/bench-action", "--automation-profile", "nonexistent"],
        )


class InvariantOrderingSuite:
    """Benchmark invariant ordering verification with multiple flags."""

    def setup(self) -> None:
        self._mock_service = MagicMock()
        self._mock_service.get_action_by_name.return_value = _mock_action()
        self._mock_service.use_action.return_value = _mock_plan(
            with_invariants=True, invariant_count=5
        )
        self._patcher = patch(
            "cleveragents.cli.commands.plan._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._patcher.start()

    def teardown(self) -> None:
        self._patcher.stop()

    def time_three_invariants(self) -> None:
        """Benchmark plan use with three invariant flags."""
        _runner.invoke(
            plan_app,
            [
                "use",
                "local/bench-action",
                "--invariant",
                "Alpha",
                "--invariant",
                "Beta",
                "--invariant",
                "Gamma",
            ],
        )

    def time_five_invariants(self) -> None:
        """Benchmark plan use with five invariant flags."""
        _runner.invoke(
            plan_app,
            [
                "use",
                "local/bench-action",
                "--invariant",
                "A",
                "--invariant",
                "B",
                "--invariant",
                "C",
                "--invariant",
                "D",
                "--invariant",
                "E",
            ],
        )


class ActorValidationErrorSuite:
    """Benchmark actor override validation for error paths."""

    def setup(self) -> None:
        self._mock_service = MagicMock()
        self._mock_service.get_action_by_name.return_value = _mock_action()
        self._mock_service.use_action.return_value = _mock_plan()
        self._patcher = patch(
            "cleveragents.cli.commands.plan._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._patcher.start()

    def teardown(self) -> None:
        self._patcher.stop()

    def time_valid_actor_validation(self) -> None:
        """Benchmark validating a valid actor name."""
        validate_namespaced_actor("openai/gpt-4", "--strategy-actor")

    def time_invalid_actor_rejection(self) -> None:
        """Benchmark rejecting an invalid actor name."""
        try:
            validate_namespaced_actor("bad-format", "--strategy-actor")
        except ValidationError:
            pass

    def time_plan_use_invalid_actor_cli(self) -> None:
        """Benchmark CLI rejection of invalid actor flag."""
        _runner.invoke(
            plan_app,
            ["use", "local/bench-action", "--strategy-actor", "bad-format"],
        )


class OutputFormatRenderingSuite:
    """Benchmark output rendering in JSON/YAML/table with extended fields."""

    def setup(self) -> None:
        self._mock_service = MagicMock()
        self._mock_service.get_action_by_name.return_value = _mock_action()
        plan = _mock_plan(with_profile=True, with_invariants=True)
        self._mock_service.list_plans.return_value = [plan]
        self._mock_service.get_plan.return_value = plan
        self._plan_patcher = patch(
            "cleveragents.cli.commands.plan._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._action_patcher = patch(
            "cleveragents.cli.commands.action._get_lifecycle_service",
            return_value=self._mock_service,
        )
        self._plan_patcher.start()
        self._action_patcher.start()

    def teardown(self) -> None:
        self._plan_patcher.stop()
        self._action_patcher.stop()

    def time_action_show_json(self) -> None:
        """Benchmark action show in JSON format."""
        _runner.invoke(
            action_app,
            ["show", "local/bench-action", "--format", "json"],
        )

    def time_action_show_yaml(self) -> None:
        """Benchmark action show in YAML format."""
        _runner.invoke(
            action_app,
            ["show", "local/bench-action", "--format", "yaml"],
        )

    def time_action_show_table(self) -> None:
        """Benchmark action show in table format."""
        _runner.invoke(
            action_app,
            ["show", "local/bench-action", "--format", "table"],
        )

    def time_plan_status_json(self) -> None:
        """Benchmark plan status in JSON format."""
        _runner.invoke(plan_app, ["status", "--format", "json"])

    def time_plan_status_yaml(self) -> None:
        """Benchmark plan status in YAML format."""
        _runner.invoke(plan_app, ["status", "--format", "yaml"])

    def time_plan_lifecycle_list_json(self) -> None:
        """Benchmark plan lifecycle-list in JSON format."""
        _runner.invoke(plan_app, ["lifecycle-list", "--format", "json"])