"""ASV benchmarks for CLI extension test scenario runtime baselines. Benchmarks the extended test scenarios added in #326: - Automation profile resolution across all builtin profiles - Invariant ordering verification with multiple flags - Actor override validation (valid and error paths) - Output format rendering (JSON, YAML, table) with extended fields """ from __future__ import annotations import contextlib import importlib import sys from datetime import datetime from pathlib import Path from unittest.mock import MagicMock, patch # Ensure the local *source* tree is importable even when ASV has an # older build of the package installed. _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) import cleveragents # noqa: E402 importlib.reload(cleveragents) from typer.testing import CliRunner # noqa: E402 from cleveragents.cli.commands.action import app as action_app # noqa: E402 from cleveragents.cli.commands.plan import app as plan_app # noqa: E402 from cleveragents.cli.commands.plan import ( # noqa: E402 validate_namespaced_actor, ) from cleveragents.core.exceptions import ValidationError # noqa: E402 from cleveragents.domain.models.core.action import ( # noqa: E402 Action, ActionState, ) from cleveragents.domain.models.core.plan import ( # noqa: E402 AutomationProfileProvenance, AutomationProfileRef, InvariantSource, NamespacedName, Plan, PlanIdentity, PlanInvariant, PlanPhase, PlanTimestamps, ProcessingState, ProjectLink, ) _runner = CliRunner() _PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S" _BUILTIN_PROFILE_NAMES = [ "manual", "review", "supervised", "cautious", "trusted", "auto", ] def _mock_action(name: str = "local/bench-action") -> Action: return Action( namespaced_name=NamespacedName.parse(name), description="Benchmark action", long_description=None, definition_of_done="Benchmarks pass", strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", estimation_actor="openai/gpt-4", invariant_actor="anthropic/claude-3", reusable=True, read_only=False, invariants=["No regressions", "Keep backward compat"], inputs_schema={ "type": "object", "properties": {"coverage": {"type": "integer"}}, }, state=ActionState.AVAILABLE, created_by=None, created_at=datetime.now(), updated_at=datetime.now(), ) def _mock_plan( name: str = "local/bench-plan", *, with_profile: bool = False, with_invariants: bool = False, invariant_count: int = 2, ) -> Plan: now = datetime.now() profile = None if with_profile: profile = AutomationProfileRef( profile_name="trusted", provenance=AutomationProfileProvenance.PLAN, ) invariants: list[PlanInvariant] = [] if with_invariants: invariants = [ PlanInvariant( text=f"Invariant {i}", source=InvariantSource.PLAN, ) for i in range(invariant_count) ] return Plan( identity=PlanIdentity(plan_id=_PLAN_ULID), namespaced_name=NamespacedName.parse(name), description="Benchmark plan", definition_of_done="Benchmarks pass", action_name="local/bench-action", phase=PlanPhase.STRATEGIZE, processing_state=ProcessingState.QUEUED, project_links=[ProjectLink(project_name="my-project")], arguments={}, arguments_order=[], automation_profile=profile, invariants=invariants, strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", reusable=True, read_only=False, created_by=None, timestamps=PlanTimestamps(created_at=now, updated_at=now), ) class ProfileResolutionSuite: """Benchmark automation profile resolution across all builtin profiles.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() self._mock_service.use_action.return_value = _mock_plan(with_profile=True) self._patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_resolve_all_builtin_profiles(self) -> None: """Benchmark resolving each builtin profile in sequence.""" for profile_name in _BUILTIN_PROFILE_NAMES: _runner.invoke( plan_app, [ "use", "local/bench-action", "--automation-profile", profile_name, ], ) def time_resolve_single_profile(self) -> None: """Benchmark resolving a single profile.""" _runner.invoke( plan_app, ["use", "local/bench-action", "--automation-profile", "trusted"], ) def time_reject_invalid_profile(self) -> None: """Benchmark rejecting an invalid profile name.""" _runner.invoke( plan_app, ["use", "local/bench-action", "--automation-profile", "nonexistent"], ) class InvariantOrderingSuite: """Benchmark invariant ordering verification with multiple flags.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() self._mock_service.use_action.return_value = _mock_plan( with_invariants=True, invariant_count=5 ) self._patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_three_invariants(self) -> None: """Benchmark plan use with three invariant flags.""" _runner.invoke( plan_app, [ "use", "local/bench-action", "--invariant", "Alpha", "--invariant", "Beta", "--invariant", "Gamma", ], ) def time_five_invariants(self) -> None: """Benchmark plan use with five invariant flags.""" _runner.invoke( plan_app, [ "use", "local/bench-action", "--invariant", "A", "--invariant", "B", "--invariant", "C", "--invariant", "D", "--invariant", "E", ], ) class ActorValidationErrorSuite: """Benchmark actor override validation for error paths.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() self._mock_service.use_action.return_value = _mock_plan() self._patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_valid_actor_validation(self) -> None: """Benchmark validating a valid actor name.""" validate_namespaced_actor("openai/gpt-4", "--strategy-actor") def time_invalid_actor_rejection(self) -> None: """Benchmark rejecting an invalid actor name.""" with contextlib.suppress(ValidationError): validate_namespaced_actor("bad-format", "--strategy-actor") def time_plan_use_invalid_actor_cli(self) -> None: """Benchmark CLI rejection of invalid actor flag.""" _runner.invoke( plan_app, ["use", "local/bench-action", "--strategy-actor", "bad-format"], ) class OutputFormatRenderingSuite: """Benchmark output rendering in JSON/YAML/table with extended fields.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() plan = _mock_plan(with_profile=True, with_invariants=True) self._mock_service.list_plans.return_value = [plan] self._mock_service.get_plan.return_value = plan self._plan_patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._action_patcher = patch( "cleveragents.cli.commands.action._get_lifecycle_service", return_value=self._mock_service, ) self._plan_patcher.start() self._action_patcher.start() def teardown(self) -> None: self._plan_patcher.stop() self._action_patcher.stop() def time_action_show_json(self) -> None: """Benchmark action show in JSON format.""" _runner.invoke( action_app, ["show", "local/bench-action", "--format", "json"], ) def time_action_show_yaml(self) -> None: """Benchmark action show in YAML format.""" _runner.invoke( action_app, ["show", "local/bench-action", "--format", "yaml"], ) def time_action_show_table(self) -> None: """Benchmark action show in table format.""" _runner.invoke( action_app, ["show", "local/bench-action", "--format", "table"], ) def time_plan_status_json(self) -> None: """Benchmark plan status in JSON format.""" _runner.invoke(plan_app, ["status", "--format", "json"]) def time_plan_status_yaml(self) -> None: """Benchmark plan status in YAML format.""" _runner.invoke(plan_app, ["status", "--format", "yaml"]) def time_plan_list_json(self) -> None: """Benchmark plan list in JSON format.""" _runner.invoke(plan_app, ["list", "--format", "json"])