freemo
3bd02a7c6e
Add CLI command group `agents automation-profile` with add, remove, list, and show subcommands for managing automation profiles that control plan execution autonomy. Changes: - New `automation_profile.py` CLI module with YAML config input, schema_version guard, namespaced name validation, --update support, and all output formats (json/yaml/plain/table/rich) - Register automation-profile in main CLI app - Add deprecation warnings for --automation-level on plan use and set-automation-level commands - Update CLI reference docs with command examples, built-in profiles list, and deprecation notes - 26 Behave scenarios covering all commands and error paths - 9 Robot Framework integration smoke tests - ASV benchmarks for CLI parsing performance - Mark A6.cli items complete in implementation_plan.md
127 lines
3.4 KiB
Python
127 lines
3.4 KiB
Python
"""ASV benchmarks for Automation Profile CLI command throughput.
|
|
|
|
Measures the performance of:
|
|
- Add profile from YAML config
|
|
- List profiles rendering
|
|
- Show profile rendering
|
|
- Remove profile
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from typer.testing import CliRunner # noqa: E402
|
|
|
|
from cleveragents.cli.commands.automation_profile import ( # noqa: E402
|
|
_InMemoryProfileRepository,
|
|
app as profile_app,
|
|
)
|
|
from cleveragents.domain.models.core.automation_profile import ( # noqa: E402
|
|
AutomationProfile,
|
|
)
|
|
|
|
_VALID_YAML = """\
|
|
name: bench/test-profile
|
|
description: Benchmark test profile
|
|
schema_version: "1.0"
|
|
auto_strategize: 0.5
|
|
auto_execute: 0.5
|
|
auto_apply: 1.0
|
|
auto_decisions_strategize: 0.5
|
|
auto_decisions_execute: 0.5
|
|
auto_validation_fix: 0.5
|
|
auto_strategy_revision: 0.5
|
|
auto_reversion_from_apply: 0.5
|
|
auto_child_plans: 0.5
|
|
auto_retry_transient: 0.0
|
|
auto_checkpoint_restore: 0.5
|
|
require_sandbox: true
|
|
require_checkpoints: true
|
|
allow_unsafe_tools: false
|
|
"""
|
|
|
|
_runner = CliRunner()
|
|
|
|
|
|
def _reset_repo() -> None:
|
|
"""Reset the module-level in-memory repo."""
|
|
import cleveragents.cli.commands.automation_profile as ap_mod
|
|
|
|
ap_mod._repo = _InMemoryProfileRepository()
|
|
|
|
|
|
class AutomationProfileCLIAddSuite:
|
|
"""Benchmark automation-profile add --config throughput."""
|
|
|
|
def setup(self) -> None:
|
|
"""Write a temporary YAML file."""
|
|
_reset_repo()
|
|
fd, self._path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(_VALID_YAML)
|
|
|
|
def teardown(self) -> None:
|
|
"""Clean up."""
|
|
Path(self._path).unlink(missing_ok=True)
|
|
|
|
def time_add_from_config(self) -> None:
|
|
"""Benchmark add --config end-to-end."""
|
|
_reset_repo()
|
|
_runner.invoke(profile_app, ["add", "--config", self._path])
|
|
|
|
|
|
class AutomationProfileCLIListSuite:
|
|
"""Benchmark automation-profile list throughput."""
|
|
|
|
def setup(self) -> None:
|
|
"""Set up with built-in profiles."""
|
|
_reset_repo()
|
|
|
|
def time_list_all(self) -> None:
|
|
"""Benchmark listing all profiles."""
|
|
_runner.invoke(profile_app, ["list"])
|
|
|
|
def time_list_json(self) -> None:
|
|
"""Benchmark listing profiles as JSON."""
|
|
_runner.invoke(profile_app, ["list", "--format", "json"])
|
|
|
|
def time_list_with_regex(self) -> None:
|
|
"""Benchmark listing with regex filter."""
|
|
_runner.invoke(profile_app, ["list", "caut.*"])
|
|
|
|
|
|
class AutomationProfileCLIShowSuite:
|
|
"""Benchmark automation-profile show throughput."""
|
|
|
|
def setup(self) -> None:
|
|
"""Set up."""
|
|
_reset_repo()
|
|
|
|
def time_show_rich(self) -> None:
|
|
"""Benchmark showing a profile in rich format."""
|
|
_runner.invoke(profile_app, ["show", "manual"])
|
|
|
|
def time_show_json(self) -> None:
|
|
"""Benchmark showing a profile in JSON format."""
|
|
_runner.invoke(profile_app, ["show", "manual", "--format", "json"])
|
|
|
|
def time_show_yaml(self) -> None:
|
|
"""Benchmark showing a profile in YAML format."""
|
|
_runner.invoke(profile_app, ["show", "manual", "--format", "yaml"])
|