"""ASV benchmarks for CLI extensions parsing overhead. Measures the performance of: - plan use with automation profile and invariant flags - plan use with actor override flags and validation - plan status / list rendering with extended fields - action show rendering with optional actors/invariants/inputs_schema """ from __future__ import annotations import importlib import sys from datetime import datetime from pathlib import Path from unittest.mock import MagicMock, patch # Ensure the local *source* tree is importable even when ASV has an # older build of the package installed. _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) import cleveragents # noqa: E402 importlib.reload(cleveragents) from typer.testing import CliRunner # noqa: E402 from cleveragents.cli.commands.action import app as action_app # noqa: E402 from cleveragents.cli.commands.plan import app as plan_app # noqa: E402 from cleveragents.domain.models.core.action import ( # noqa: E402 Action, ActionState, ) from cleveragents.domain.models.core.plan import ( # noqa: E402 AutomationProfileProvenance, AutomationProfileRef, InvariantSource, NamespacedName, Plan, PlanIdentity, PlanInvariant, PlanPhase, PlanTimestamps, ProcessingState, ProjectLink, ) _runner = CliRunner() _PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S" def _mock_action(name: str = "local/bench-action") -> Action: return Action( namespaced_name=NamespacedName.parse(name), description="Benchmark action", long_description=None, definition_of_done="Benchmarks pass", strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", estimation_actor="openai/gpt-4", invariant_actor="anthropic/claude-3", reusable=True, read_only=False, invariants=["No regressions", "Keep backward compat"], inputs_schema={ "type": "object", "properties": {"coverage": {"type": "integer"}}, }, state=ActionState.AVAILABLE, created_by=None, created_at=datetime.now(), updated_at=datetime.now(), ) def _mock_plan( name: str = "local/bench-plan", *, with_profile: bool = False, with_invariants: bool = False, ) -> Plan: now = datetime.now() profile = None if with_profile: profile = AutomationProfileRef( profile_name="trusted", provenance=AutomationProfileProvenance.PLAN, ) invariants: list[PlanInvariant] = [] if with_invariants: invariants = [ PlanInvariant(text="No warnings", source=InvariantSource.PLAN), PlanInvariant(text="Keep compat", source=InvariantSource.ACTION), ] return Plan( identity=PlanIdentity(plan_id=_PLAN_ULID), namespaced_name=NamespacedName.parse(name), description="Benchmark plan", definition_of_done="Benchmarks pass", action_name="local/bench-action", phase=PlanPhase.STRATEGIZE, processing_state=ProcessingState.QUEUED, project_links=[ProjectLink(project_name="my-project")], arguments={}, arguments_order=[], automation_profile=profile, invariants=invariants, strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", reusable=True, read_only=False, created_by=None, timestamps=PlanTimestamps(created_at=now, updated_at=now), ) class PlanUseWithProfileSuite: """Benchmark plan use with --automation-profile.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() self._mock_service.use_action.return_value = _mock_plan(with_profile=True) self._patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_plan_use_with_profile(self) -> None: """Benchmark plan use with automation profile.""" _runner.invoke( plan_app, [ "use", "local/bench-action", "--automation-profile", "trusted", ], ) class PlanUseWithInvariantsSuite: """Benchmark plan use with --invariant flags.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() self._mock_service.use_action.return_value = _mock_plan(with_invariants=True) self._patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_plan_use_with_invariants(self) -> None: """Benchmark plan use with invariant flags.""" _runner.invoke( plan_app, [ "use", "local/bench-action", "--invariant", "No warnings", "--invariant", "Keep compat", ], ) class PlanUseActorValidationSuite: """Benchmark plan use with actor override flags + validation.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() self._mock_service.use_action.return_value = _mock_plan() self._patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_plan_use_with_actor_overrides(self) -> None: """Benchmark plan use with all actor overrides.""" _runner.invoke( plan_app, [ "use", "local/bench-action", "--strategy-actor", "openai/gpt-4", "--execution-actor", "anthropic/claude-3", "--estimation-actor", "openai/gpt-4", "--invariant-actor", "anthropic/claude-3", ], ) class PlanStatusWithExtendedFieldsSuite: """Benchmark plan status with automation profile + invariants.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.list_plans.return_value = [ _mock_plan(f"local/plan-{i}", with_profile=True, with_invariants=True) for i in range(20) ] self._patcher = patch( "cleveragents.cli.commands.plan._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_plan_status_table(self) -> None: """Benchmark plan status table rendering with extended fields.""" _runner.invoke(plan_app, ["status"]) def time_plan_list_table(self) -> None: """Benchmark list table rendering with extended fields.""" _runner.invoke(plan_app, ["list"]) class ActionShowExtendedSuite: """Benchmark action show with optional actors/invariants/schema.""" def setup(self) -> None: self._mock_service = MagicMock() self._mock_service.get_action_by_name.return_value = _mock_action() self._patcher = patch( "cleveragents.cli.commands.action._get_lifecycle_service", return_value=self._mock_service, ) self._patcher.start() def teardown(self) -> None: self._patcher.stop() def time_action_show_rich(self) -> None: """Benchmark action show in rich format.""" _runner.invoke(action_app, ["show", "local/bench-action"]) def time_action_show_json(self) -> None: """Benchmark action show in JSON format.""" _runner.invoke(action_app, ["show", "local/bench-action", "--format", "json"])