forked from HAL9000/cleveragents-core
3bd02a7c6e
Add CLI command group `agents automation-profile` with add, remove, list, and show subcommands for managing automation profiles that control plan execution autonomy. Changes: - New `automation_profile.py` CLI module with YAML config input, schema_version guard, namespaced name validation, --update support, and all output formats (json/yaml/plain/table/rich) - Register automation-profile in main CLI app - Add deprecation warnings for --automation-level on plan use and set-automation-level commands - Update CLI reference docs with command examples, built-in profiles list, and deprecation notes - 26 Behave scenarios covering all commands and error paths - 9 Robot Framework integration smoke tests - ASV benchmarks for CLI parsing performance - Mark A6.cli items complete in implementation_plan.md
258 lines
9.0 KiB
Python
258 lines
9.0 KiB
Python
"""Step definitions for Automation Profile CLI coverage boost."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import warnings
|
|
from unittest.mock import patch
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
from typer.testing import CliRunner
|
|
|
|
from cleveragents.cli.commands.automation_profile import (
|
|
_InMemoryProfileRepository,
|
|
_threshold_summary,
|
|
emit_automation_level_deprecation_warning,
|
|
)
|
|
from cleveragents.cli.commands.automation_profile import (
|
|
app as profile_app,
|
|
)
|
|
from cleveragents.core.exceptions import (
|
|
CleverAgentsError,
|
|
NotFoundError,
|
|
ValidationError,
|
|
)
|
|
from cleveragents.domain.models.core.automation_profile import (
|
|
BUILTIN_PROFILES,
|
|
AutomationProfile,
|
|
)
|
|
|
|
|
|
def _make_profile(
|
|
name: str = "acme/strict",
|
|
description: str = "Strict review",
|
|
) -> AutomationProfile:
|
|
return AutomationProfile(
|
|
name=name,
|
|
description=description,
|
|
schema_version="1.0",
|
|
auto_strategize=0.8,
|
|
auto_execute=0.7,
|
|
auto_apply=0.6,
|
|
)
|
|
|
|
|
|
def _write_temp(context: Context, content: str) -> str:
|
|
fd, path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(content)
|
|
if not hasattr(context, "_temp_files"):
|
|
context._temp_files = []
|
|
context._temp_files.append(path)
|
|
return path
|
|
|
|
|
|
@given("an automation profile coverage CLI runner")
|
|
def step_coverage_runner(context: Context) -> None:
|
|
context.runner = CliRunner()
|
|
context.result = None
|
|
import cleveragents.cli.commands.automation_profile as ap_mod
|
|
|
|
ap_mod._repo = _InMemoryProfileRepository()
|
|
|
|
|
|
@when("I delete a non-existent profile from the in-memory repo")
|
|
def step_delete_nonexistent_from_repo(context: Context) -> None:
|
|
repo = _InMemoryProfileRepository()
|
|
context.repo_error = None
|
|
try:
|
|
repo.delete("nonexistent/profile")
|
|
except NotFoundError as exc:
|
|
context.repo_error = exc
|
|
|
|
|
|
@then("a NotFoundError should be raised from the repo")
|
|
def step_assert_not_found_error(context: Context) -> None:
|
|
assert context.repo_error is not None, "Expected NotFoundError but none was raised"
|
|
assert isinstance(context.repo_error, NotFoundError)
|
|
|
|
|
|
@when("I call threshold summary on a built-in profile")
|
|
def step_call_threshold_summary(context: Context) -> None:
|
|
profile = BUILTIN_PROFILES["manual"]
|
|
context.threshold_result = _threshold_summary(profile)
|
|
|
|
|
|
@then("the threshold summary should contain strategize execute and apply values")
|
|
def step_assert_threshold_summary(context: Context) -> None:
|
|
result = context.threshold_result
|
|
assert "strategize=" in result
|
|
assert "execute=" in result
|
|
assert "apply=" in result
|
|
|
|
|
|
@given("a YAML file containing a list instead of a dict")
|
|
def step_non_dict_yaml(context: Context) -> None:
|
|
context.non_dict_yaml_path = _write_temp(context, "- item1\n- item2\n")
|
|
|
|
|
|
@when("I run automation-profile add with that non-dict YAML file")
|
|
def step_run_add_non_dict(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
profile_app, ["add", "--config", context.non_dict_yaml_path]
|
|
)
|
|
|
|
|
|
@then("the automation-profile coverage command should abort")
|
|
def step_coverage_command_abort(context: Context) -> None:
|
|
assert context.result is not None
|
|
assert context.result.exit_code != 0, (
|
|
f"Expected non-zero exit, got {context.result.exit_code}. "
|
|
f"Output: {context.result.output}"
|
|
)
|
|
|
|
|
|
@then('the automation-profile coverage output should contain "{text}"')
|
|
def step_coverage_output_contains(context: Context, text: str) -> None:
|
|
assert context.result is not None
|
|
assert text.lower() in context.result.output.lower(), (
|
|
f"Expected '{text}' in output. Got: {context.result.output}"
|
|
)
|
|
|
|
|
|
@when("I run automation-profile add with a config that triggers FileNotFoundError")
|
|
def step_run_add_file_not_found(context: Context) -> None:
|
|
# Create a file that exists at check time but causes FileNotFoundError on open
|
|
path = _write_temp(
|
|
context,
|
|
"name: acme/test\ndescription: test\nschema_version: '1.0'\n",
|
|
)
|
|
original_open = open
|
|
|
|
def patched_open(p, *args, **kwargs):
|
|
if str(p) == path:
|
|
raise FileNotFoundError(f"File vanished: {p}")
|
|
return original_open(p, *args, **kwargs)
|
|
|
|
with patch("builtins.open", side_effect=patched_open):
|
|
context.result = context.runner.invoke(profile_app, ["add", "--config", path])
|
|
|
|
|
|
@given("a YAML file that triggers a ValidationError from the service")
|
|
def step_yaml_validation_error(context: Context) -> None:
|
|
context.validation_yaml_path = _write_temp(
|
|
context,
|
|
"name: acme/valerr\ndescription: test\nschema_version: '1.0'\n",
|
|
)
|
|
|
|
|
|
@when("I run automation-profile add with that validation-error YAML file")
|
|
def step_run_add_validation_error(context: Context) -> None:
|
|
with patch(
|
|
"cleveragents.cli.commands.automation_profile.AutomationProfileService.create_profile",
|
|
side_effect=ValidationError("Invalid profile data"),
|
|
):
|
|
context.result = context.runner.invoke(
|
|
profile_app, ["add", "--config", context.validation_yaml_path]
|
|
)
|
|
|
|
|
|
@given("a YAML file that triggers a CleverAgentsError from the service")
|
|
def step_yaml_ca_error(context: Context) -> None:
|
|
context.ca_error_yaml_path = _write_temp(
|
|
context,
|
|
"name: acme/caerr\ndescription: test\nschema_version: '1.0'\n",
|
|
)
|
|
|
|
|
|
@when("I run automation-profile add with that cleveragents-error YAML file")
|
|
def step_run_add_ca_error(context: Context) -> None:
|
|
with patch(
|
|
"cleveragents.cli.commands.automation_profile.AutomationProfileService.create_profile",
|
|
side_effect=CleverAgentsError("Something went wrong"),
|
|
):
|
|
context.result = context.runner.invoke(
|
|
profile_app, ["add", "--config", context.ca_error_yaml_path]
|
|
)
|
|
|
|
|
|
@given('a custom coverage profile "{name}" has been added')
|
|
def step_add_custom_coverage_profile(context: Context, name: str) -> None:
|
|
import cleveragents.cli.commands.automation_profile as ap_mod
|
|
|
|
profile = _make_profile(name=name)
|
|
ap_mod._repo.upsert(profile)
|
|
|
|
|
|
@when('I run automation-profile remove "{name}" without --yes and decline')
|
|
def step_remove_no_yes_decline(context: Context, name: str) -> None:
|
|
context.result = context.runner.invoke(profile_app, ["remove", name], input="n\n")
|
|
|
|
|
|
@when('I run automation-profile remove "{name}" without --yes and confirm')
|
|
def step_remove_no_yes_confirm(context: Context, name: str) -> None:
|
|
context.result = context.runner.invoke(profile_app, ["remove", name], input="y\n")
|
|
|
|
|
|
@then("the automation-profile coverage remove should succeed")
|
|
def step_coverage_remove_succeed(context: Context) -> None:
|
|
assert context.result is not None
|
|
assert context.result.exit_code == 0, (
|
|
f"Expected exit 0, got {context.result.exit_code}. "
|
|
f"Output: {context.result.output}"
|
|
)
|
|
|
|
|
|
@when('I run automation-profile remove "{name}" with validation error')
|
|
def step_remove_validation_error(context: Context, name: str) -> None:
|
|
with patch(
|
|
"cleveragents.cli.commands.automation_profile.AutomationProfileService.delete_profile",
|
|
side_effect=ValidationError("Cannot delete"),
|
|
):
|
|
context.result = context.runner.invoke(profile_app, ["remove", name, "--yes"])
|
|
|
|
|
|
@when('I run automation-profile remove "{name}" with CleverAgentsError')
|
|
def step_remove_ca_error(context: Context, name: str) -> None:
|
|
with patch(
|
|
"cleveragents.cli.commands.automation_profile.AutomationProfileService.delete_profile",
|
|
side_effect=CleverAgentsError("Delete failed"),
|
|
):
|
|
context.result = context.runner.invoke(profile_app, ["remove", name, "--yes"])
|
|
|
|
|
|
@when("I run automation-profile list with a CleverAgentsError from service")
|
|
def step_list_ca_error(context: Context) -> None:
|
|
with patch(
|
|
"cleveragents.cli.commands.automation_profile.AutomationProfileService.list_profiles",
|
|
side_effect=CleverAgentsError("List failed"),
|
|
):
|
|
context.result = context.runner.invoke(profile_app, ["list"])
|
|
|
|
|
|
@when("I run automation-profile show with a CleverAgentsError from service")
|
|
def step_show_ca_error(context: Context) -> None:
|
|
with patch(
|
|
"cleveragents.cli.commands.automation_profile.AutomationProfileService.get_profile",
|
|
side_effect=CleverAgentsError("Show failed"),
|
|
):
|
|
context.result = context.runner.invoke(profile_app, ["show", "manual"])
|
|
|
|
|
|
@when("I call emit_automation_level_deprecation_warning")
|
|
def step_call_deprecation_warning(context: Context) -> None:
|
|
with warnings.catch_warnings(record=True) as w:
|
|
warnings.simplefilter("always")
|
|
emit_automation_level_deprecation_warning()
|
|
context.warnings = w
|
|
|
|
|
|
@then("a DeprecationWarning should be emitted")
|
|
def step_assert_deprecation_warning(context: Context) -> None:
|
|
assert len(context.warnings) >= 1, "Expected at least one warning"
|
|
assert any(issubclass(w.category, DeprecationWarning) for w in context.warnings), (
|
|
f"Expected DeprecationWarning, got: {[w.category for w in context.warnings]}"
|
|
)
|