Files
temp/features/steps/automation_profile_cli_coverage_steps.py
T
Luis Mendes 2acb957d83 fix(cli): replace in-memory automation-profile repository with database-backed persistence
The automation-profile CLI commands used an _InMemoryProfileRepository
(a Python dict) that lost all data between CLI process invocations.
Profiles created with "automation-profile add" were invisible to
subsequent "automation-profile show" or "list" calls because each CLI
command is a separate process with a fresh empty dict.

Changes:
- Replaced _InMemoryProfileRepository with the real
  AutomationProfileRepository from the infrastructure layer, wired
  via the DI container following the same pattern as tool.py and
  session.py.
- Added auto_commit support to AutomationProfileRepository (matching
  the existing SessionRepository pattern) so that CLI commands running
  outside a UnitOfWork commit each operation automatically.
- Added safety_json and guards_json Text columns to the
  automation_profiles table (Alembic migration m6_005) for full-fidelity
  round-trip of the AutomationGuard and SafetyProfile sub-models.
  Previously, guards and several safety fields (max_cost_per_plan,
  max_retries_per_step, etc.) were silently dropped on persistence.
- Updated _from_domain, _to_domain, and _update_row to serialize and
  deserialize the full guard and safety sub-models via JSON, with
  backward-compatible fallback to legacy scalar columns.

Refs: #746
2026-03-17 09:53:53 +00:00

283 lines
10 KiB
Python

"""Step definitions for Automation Profile CLI coverage boost."""
from __future__ import annotations
import os
import tempfile
import warnings
from unittest.mock import patch
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from cleveragents.cli.commands.automation_profile import (
_threshold_summary,
emit_automation_level_deprecation_warning,
)
from cleveragents.cli.commands.automation_profile import (
app as profile_app,
)
from cleveragents.core.exceptions import (
CleverAgentsError,
ValidationError,
)
from cleveragents.domain.models.core.automation_profile import (
BUILTIN_PROFILES,
AutomationProfile,
)
from cleveragents.infrastructure.database.repositories import (
AutomationProfileNotFoundError,
)
def _create_in_memory_profile_service():
"""Create an AutomationProfileService backed by an in-memory SQLite DB."""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from cleveragents.application.services.automation_profile_service import (
AutomationProfileService,
)
from cleveragents.infrastructure.database.models import Base
from cleveragents.infrastructure.database.repositories import (
AutomationProfileRepository,
)
engine = create_engine("sqlite:///:memory:", echo=False)
Base.metadata.create_all(engine)
factory = sessionmaker(bind=engine, expire_on_commit=False)
repo = AutomationProfileRepository(session_factory=factory, auto_commit=True)
return AutomationProfileService(repo=repo)
def _make_profile(
name: str = "acme/strict",
description: str = "Strict review",
) -> AutomationProfile:
return AutomationProfile(
name=name,
description=description,
schema_version="1.0",
auto_strategize=0.8,
auto_execute=0.7,
auto_apply=0.6,
)
def _write_temp(context: Context, content: str) -> str:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(content)
if not hasattr(context, "_temp_files"):
context._temp_files = []
context._temp_files.append(path)
return path
@given("an automation profile coverage CLI runner")
def step_coverage_runner(context: Context) -> None:
context.runner = CliRunner()
context.result = None
# Create an in-memory service and patch _get_service for this scenario
context._ap_service = _create_in_memory_profile_service()
context._ap_patcher = patch(
"cleveragents.cli.commands.automation_profile._get_service",
return_value=context._ap_service,
)
context._ap_patcher.start()
if not hasattr(context, "_cleanup_handlers"):
context._cleanup_handlers = []
context._cleanup_handlers.append(context._ap_patcher.stop)
@when("I delete a non-existent profile from the in-memory repo")
def step_delete_nonexistent_from_repo(context: Context) -> None:
context.repo_error = None
try:
context._ap_service._repo.delete("nonexistent/profile")
except AutomationProfileNotFoundError as exc:
context.repo_error = exc
@then("a NotFoundError should be raised from the repo")
def step_assert_not_found_error(context: Context) -> None:
assert context.repo_error is not None, "Expected NotFoundError but none was raised"
assert isinstance(context.repo_error, AutomationProfileNotFoundError)
@when("I call threshold summary on a built-in profile")
def step_call_threshold_summary(context: Context) -> None:
profile = BUILTIN_PROFILES["manual"]
context.threshold_result = _threshold_summary(profile)
@then("the threshold summary should contain strategize execute and apply values")
def step_assert_threshold_summary(context: Context) -> None:
result = context.threshold_result
assert "strategize=" in result
assert "execute=" in result
assert "apply=" in result
@given("a YAML file containing a list instead of a dict")
def step_non_dict_yaml(context: Context) -> None:
context.non_dict_yaml_path = _write_temp(context, "- item1\n- item2\n")
@when("I run automation-profile add with that non-dict YAML file")
def step_run_add_non_dict(context: Context) -> None:
context.result = context.runner.invoke(
profile_app, ["add", "--config", context.non_dict_yaml_path]
)
@then("the automation-profile coverage command should abort")
def step_coverage_command_abort(context: Context) -> None:
assert context.result is not None
assert context.result.exit_code != 0, (
f"Expected non-zero exit, got {context.result.exit_code}. "
f"Output: {context.result.output}"
)
@then('the automation-profile coverage output should contain "{text}"')
def step_coverage_output_contains(context: Context, text: str) -> None:
assert context.result is not None
assert text.lower() in context.result.output.lower(), (
f"Expected '{text}' in output. Got: {context.result.output}"
)
@when("I run automation-profile add with a config that triggers FileNotFoundError")
def step_run_add_file_not_found(context: Context) -> None:
# Create a file that exists at check time but causes FileNotFoundError on open
path = _write_temp(
context,
"name: acme/test\ndescription: test\nschema_version: '1.0'\n",
)
original_open = open
def patched_open(p, *args, **kwargs):
if str(p) == path:
raise FileNotFoundError(f"File vanished: {p}")
return original_open(p, *args, **kwargs)
with patch("builtins.open", side_effect=patched_open):
context.result = context.runner.invoke(profile_app, ["add", "--config", path])
@given("a YAML file that triggers a ValidationError from the service")
def step_yaml_validation_error(context: Context) -> None:
context.validation_yaml_path = _write_temp(
context,
"name: acme/valerr\ndescription: test\nschema_version: '1.0'\n",
)
@when("I run automation-profile add with that validation-error YAML file")
def step_run_add_validation_error(context: Context) -> None:
with patch(
"cleveragents.cli.commands.automation_profile.AutomationProfileService.create_profile",
side_effect=ValidationError("Invalid profile data"),
):
context.result = context.runner.invoke(
profile_app, ["add", "--config", context.validation_yaml_path]
)
@given("a YAML file that triggers a CleverAgentsError from the service")
def step_yaml_ca_error(context: Context) -> None:
context.ca_error_yaml_path = _write_temp(
context,
"name: acme/caerr\ndescription: test\nschema_version: '1.0'\n",
)
@when("I run automation-profile add with that cleveragents-error YAML file")
def step_run_add_ca_error(context: Context) -> None:
with patch(
"cleveragents.cli.commands.automation_profile.AutomationProfileService.create_profile",
side_effect=CleverAgentsError("Something went wrong"),
):
context.result = context.runner.invoke(
profile_app, ["add", "--config", context.ca_error_yaml_path]
)
@given('a custom coverage profile "{name}" has been added')
def step_add_custom_coverage_profile(context: Context, name: str) -> None:
profile = _make_profile(name=name)
context._ap_service._repo.upsert(profile)
@when('I run automation-profile remove "{name}" without --yes and decline')
def step_remove_no_yes_decline(context: Context, name: str) -> None:
context.result = context.runner.invoke(profile_app, ["remove", name], input="n\n")
@when('I run automation-profile remove "{name}" without --yes and confirm')
def step_remove_no_yes_confirm(context: Context, name: str) -> None:
context.result = context.runner.invoke(profile_app, ["remove", name], input="y\n")
@then("the automation-profile coverage remove should succeed")
def step_coverage_remove_succeed(context: Context) -> None:
assert context.result is not None
assert context.result.exit_code == 0, (
f"Expected exit 0, got {context.result.exit_code}. "
f"Output: {context.result.output}"
)
@when('I run automation-profile remove "{name}" with validation error')
def step_remove_validation_error(context: Context, name: str) -> None:
with patch(
"cleveragents.cli.commands.automation_profile.AutomationProfileService.delete_profile",
side_effect=ValidationError("Cannot delete"),
):
context.result = context.runner.invoke(profile_app, ["remove", name, "--yes"])
@when('I run automation-profile remove "{name}" with CleverAgentsError')
def step_remove_ca_error(context: Context, name: str) -> None:
with patch(
"cleveragents.cli.commands.automation_profile.AutomationProfileService.delete_profile",
side_effect=CleverAgentsError("Delete failed"),
):
context.result = context.runner.invoke(profile_app, ["remove", name, "--yes"])
@when("I run automation-profile list with a CleverAgentsError from service")
def step_list_ca_error(context: Context) -> None:
with patch(
"cleveragents.cli.commands.automation_profile.AutomationProfileService.list_profiles",
side_effect=CleverAgentsError("List failed"),
):
context.result = context.runner.invoke(profile_app, ["list"])
@when("I run automation-profile show with a CleverAgentsError from service")
def step_show_ca_error(context: Context) -> None:
with patch(
"cleveragents.cli.commands.automation_profile.AutomationProfileService.get_profile",
side_effect=CleverAgentsError("Show failed"),
):
context.result = context.runner.invoke(profile_app, ["show", "manual"])
@when("I call emit_automation_level_deprecation_warning")
def step_call_deprecation_warning(context: Context) -> None:
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
emit_automation_level_deprecation_warning()
context.warnings = w
@then("a DeprecationWarning should be emitted")
def step_assert_deprecation_warning(context: Context) -> None:
assert len(context.warnings) >= 1, "Expected at least one warning"
assert any(issubclass(w.category, DeprecationWarning) for w in context.warnings), (
f"Expected DeprecationWarning, got: {[w.category for w in context.warnings]}"
)