263 lines
8.1 KiB
Python
263 lines
8.1 KiB
Python
"""ASV benchmarks for CLI extensions parsing overhead.
|
|
|
|
Measures the performance of:
|
|
- plan use with automation profile and invariant flags
|
|
- plan use with actor override flags and validation
|
|
- plan status / lifecycle-list rendering with extended fields
|
|
- action show rendering with optional actors/invariants/inputs_schema
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from typer.testing import CliRunner # noqa: E402
|
|
|
|
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
|
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
|
from cleveragents.domain.models.core.action import ( # noqa: E402
|
|
Action,
|
|
ActionState,
|
|
)
|
|
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
|
AutomationProfileProvenance,
|
|
AutomationProfileRef,
|
|
InvariantSource,
|
|
NamespacedName,
|
|
Plan,
|
|
PlanIdentity,
|
|
PlanInvariant,
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
ProjectLink,
|
|
)
|
|
|
|
_runner = CliRunner()
|
|
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"
|
|
|
|
|
|
def _mock_action(name: str = "local/bench-action") -> Action:
|
|
return Action(
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Benchmark action",
|
|
long_description=None,
|
|
definition_of_done="Benchmarks pass",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
estimation_actor="openai/gpt-4",
|
|
invariant_actor="anthropic/claude-3",
|
|
reusable=True,
|
|
read_only=False,
|
|
invariants=["No regressions", "Keep backward compat"],
|
|
inputs_schema={
|
|
"type": "object",
|
|
"properties": {"coverage": {"type": "integer"}},
|
|
},
|
|
state=ActionState.AVAILABLE,
|
|
created_by=None,
|
|
created_at=datetime.now(),
|
|
updated_at=datetime.now(),
|
|
)
|
|
|
|
|
|
def _mock_plan(
|
|
name: str = "local/bench-plan",
|
|
*,
|
|
with_profile: bool = False,
|
|
with_invariants: bool = False,
|
|
) -> Plan:
|
|
now = datetime.now()
|
|
profile = None
|
|
if with_profile:
|
|
profile = AutomationProfileRef(
|
|
profile_name="trusted",
|
|
provenance=AutomationProfileProvenance.PLAN,
|
|
)
|
|
invariants: list[PlanInvariant] = []
|
|
if with_invariants:
|
|
invariants = [
|
|
PlanInvariant(text="No warnings", source=InvariantSource.PLAN),
|
|
PlanInvariant(text="Keep compat", source=InvariantSource.ACTION),
|
|
]
|
|
return Plan(
|
|
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Benchmark plan",
|
|
definition_of_done="Benchmarks pass",
|
|
action_name="local/bench-action",
|
|
phase=PlanPhase.STRATEGIZE,
|
|
processing_state=ProcessingState.QUEUED,
|
|
project_links=[ProjectLink(project_name="my-project")],
|
|
arguments={},
|
|
arguments_order=[],
|
|
automation_profile=profile,
|
|
invariants=invariants,
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
reusable=True,
|
|
read_only=False,
|
|
created_by=None,
|
|
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
|
)
|
|
|
|
|
|
class PlanUseWithProfileSuite:
|
|
"""Benchmark plan use with --automation-profile."""
|
|
|
|
def setup(self) -> None:
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.get_action_by_name.return_value = _mock_action()
|
|
self._mock_service.use_action.return_value = _mock_plan(with_profile=True)
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
|
|
def time_plan_use_with_profile(self) -> None:
|
|
"""Benchmark plan use with automation profile."""
|
|
_runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/bench-action",
|
|
"--automation-profile",
|
|
"trusted",
|
|
],
|
|
)
|
|
|
|
|
|
class PlanUseWithInvariantsSuite:
|
|
"""Benchmark plan use with --invariant flags."""
|
|
|
|
def setup(self) -> None:
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.get_action_by_name.return_value = _mock_action()
|
|
self._mock_service.use_action.return_value = _mock_plan(with_invariants=True)
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
|
|
def time_plan_use_with_invariants(self) -> None:
|
|
"""Benchmark plan use with invariant flags."""
|
|
_runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/bench-action",
|
|
"--invariant",
|
|
"No warnings",
|
|
"--invariant",
|
|
"Keep compat",
|
|
],
|
|
)
|
|
|
|
|
|
class PlanUseActorValidationSuite:
|
|
"""Benchmark plan use with actor override flags + validation."""
|
|
|
|
def setup(self) -> None:
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.get_action_by_name.return_value = _mock_action()
|
|
self._mock_service.use_action.return_value = _mock_plan()
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
|
|
def time_plan_use_with_actor_overrides(self) -> None:
|
|
"""Benchmark plan use with all actor overrides."""
|
|
_runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/bench-action",
|
|
"--strategy-actor",
|
|
"openai/gpt-4",
|
|
"--execution-actor",
|
|
"anthropic/claude-3",
|
|
"--estimation-actor",
|
|
"openai/gpt-4",
|
|
"--invariant-actor",
|
|
"anthropic/claude-3",
|
|
],
|
|
)
|
|
|
|
|
|
class PlanStatusWithExtendedFieldsSuite:
|
|
"""Benchmark plan status with automation profile + invariants."""
|
|
|
|
def setup(self) -> None:
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.list_plans.return_value = [
|
|
_mock_plan(f"local/plan-{i}", with_profile=True, with_invariants=True)
|
|
for i in range(20)
|
|
]
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
|
|
def time_plan_status_table(self) -> None:
|
|
"""Benchmark plan status table rendering with extended fields."""
|
|
_runner.invoke(plan_app, ["status"])
|
|
|
|
def time_plan_lifecycle_list_table(self) -> None:
|
|
"""Benchmark lifecycle-list table rendering with extended fields."""
|
|
_runner.invoke(plan_app, ["lifecycle-list"])
|
|
|
|
|
|
class ActionShowExtendedSuite:
|
|
"""Benchmark action show with optional actors/invariants/schema."""
|
|
|
|
def setup(self) -> None:
|
|
self._mock_service = MagicMock()
|
|
self._mock_service.get_action_by_name.return_value = _mock_action()
|
|
self._patcher = patch(
|
|
"cleveragents.cli.commands.action._get_lifecycle_service",
|
|
return_value=self._mock_service,
|
|
)
|
|
self._patcher.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._patcher.stop()
|
|
|
|
def time_action_show_rich(self) -> None:
|
|
"""Benchmark action show in rich format."""
|
|
_runner.invoke(action_app, ["show", "local/bench-action"])
|
|
|
|
def time_action_show_json(self) -> None:
|
|
"""Benchmark action show in JSON format."""
|
|
_runner.invoke(action_app, ["show", "local/bench-action", "--format", "json"])
|