Files
temp/features/steps/automation_profile_cli_coverage_boost_steps.py
CoreRasurae 007af498b8 refactor(autonomy): rename automation profile task flags to spec names
Renamed all 11 task-type confidence threshold fields in AutomationProfile
from phase-transition semantics to spec-defined task-type semantics.
Updated all 8 built-in profiles, CLI formatting, YAML schema, services,
and all Behave/Robot tests referencing the old field names.

Post-review fixes:
- Fixed 24 stale old field names in M6 fixture files
  (automation_profiles.json, autonomy_guardrails.json)
- Added model_validator(mode='before') to detect legacy field names
  and raise actionable ValueError with rename mapping
- Added semantic bridge comments in PlanLifecycleService mapping
  task-type thresholds to phase-transition gates
- Added threshold_field to structured log messages for observability
- Restored categorised CLI automation-profile show output to match
  spec (Phase Transitions / Decision Automation / Self-Repair /
  Execution Controls) instead of flat list
- Added missing access_network field to spec show output examples
  (Rich, Plain, JSON, YAML variants)
- Aligned ADR-017 profile fields table to all 11 fields with
  descriptions matching spec Automatable Tasks table
- Aligned automation_profiles.md threshold descriptions with spec
- Added spec section references in phase_reversion.md, error_recovery.md,
  and plan_execute.md for field naming context
- Extended repository roundtrip test to assert all 11 threshold fields
- Fixed benchmark _make_profile() passing safety fields as top-level
  kwargs instead of via SafetyProfile sub-model (incompatible with
  extra="forbid")
- Aligned CLI JSON/YAML output structure for automation-profile show
  with the specification grouped format (phase_transitions,
  decision_automation, self_repair, execution_controls)
- Moved safety boolean fields into the Execution Controls section
  of Rich output per spec examples
- Reverted auto profile description to "Fully automatic except apply"
  per specification (line 16703, line 28406)
- Improved bridge comments in test steps with semantic context for
  threshold-to-gate mappings

ISSUES CLOSED: #902
2026-03-30 13:18:07 +01:00

371 lines
13 KiB
Python

"""Step definitions for Automation Profile CLI guards coverage boost.
Targets uncovered lines 91-97 (_guards_dict non-None branch) and
lines 194-204 (_print_profile rich guards display block).
"""
from __future__ import annotations
import os
import tempfile
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from cleveragents.cli.commands.automation_profile import (
_guards_dict,
_profile_spec_dict,
)
from cleveragents.cli.commands.automation_profile import (
app as profile_app,
)
from cleveragents.domain.models.core.automation_guard import AutomationGuard
from cleveragents.domain.models.core.automation_profile import AutomationProfile
def _create_in_memory_profile_service():
"""Create an AutomationProfileService backed by an in-memory SQLite DB."""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from cleveragents.application.services.automation_profile_service import (
AutomationProfileService,
)
from cleveragents.infrastructure.database.models import Base
from cleveragents.infrastructure.database.repositories import (
AutomationProfileRepository,
)
engine = create_engine("sqlite:///:memory:", echo=False)
Base.metadata.create_all(engine)
factory = sessionmaker(bind=engine, expire_on_commit=False)
repo = AutomationProfileRepository(session_factory=factory, auto_commit=True)
return AutomationProfileService(repo=repo)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_guarded_profile(
name: str = "acme/guarded",
description: str = "Profile with guards",
max_tool_calls: int = 5,
max_total_cost: float = 100.0,
tool_allowlist: list[str] | None = None,
tool_denylist: list[str] | None = None,
require_approval_for_writes: bool = True,
require_approval_for_apply: bool = False,
) -> AutomationProfile:
"""Create an AutomationProfile with guards attached."""
guards = AutomationGuard(
max_tool_calls_per_step=max_tool_calls,
max_total_cost=max_total_cost,
tool_allowlist=tool_allowlist,
tool_denylist=tool_denylist,
require_approval_for_writes=require_approval_for_writes,
require_approval_for_apply=require_approval_for_apply,
)
return AutomationProfile(
name=name,
description=description,
schema_version="1.0",
decompose_task=0.5,
create_tool=0.4,
select_tool=0.3,
guards=guards,
)
def _write_temp_yaml(context: Context, content: str) -> str:
"""Write content to a temporary YAML file, tracked for cleanup."""
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(content)
if not hasattr(context, "_temp_files"):
context._temp_files = []
context._temp_files.append(path)
return path
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("a fresh automation profile CLI runner for guards coverage")
def step_fresh_guards_runner(context: Context) -> None:
from unittest.mock import patch
context.runner = CliRunner()
context.result = None
context.guards_dict_result = None
context.profile_spec_result = None
context.guard_obj = None
context.guarded_profile = None
# Create an in-memory service and patch _get_service for this scenario
context._ap_service = _create_in_memory_profile_service()
context._ap_patcher = patch(
"cleveragents.cli.commands.automation_profile._get_service",
return_value=context._ap_service,
)
context._ap_patcher.start()
if not hasattr(context, "_cleanup_handlers"):
context._cleanup_handlers = []
context._cleanup_handlers.append(context._ap_patcher.stop)
# ---------------------------------------------------------------------------
# _guards_dict direct tests (lines 91-97)
# ---------------------------------------------------------------------------
@given(
"an AutomationGuard with max_tool_calls_per_step {calls:d} and max_total_cost {cost:g}"
)
def step_create_guard_with_calls_and_cost(
context: Context,
calls: int,
cost: float,
) -> None:
context.guard_obj = AutomationGuard(
max_tool_calls_per_step=calls,
max_total_cost=cost,
tool_allowlist=None,
tool_denylist=None,
require_approval_for_writes=False,
require_approval_for_apply=False,
)
@given(
'an AutomationGuard with tool_allowlist "{allowlist}" and tool_denylist "{denylist}"'
)
def step_create_guard_with_lists(
context: Context,
allowlist: str,
denylist: str,
) -> None:
context.guard_obj = AutomationGuard(
max_tool_calls_per_step=None,
max_total_cost=None,
tool_allowlist=allowlist.split(","),
tool_denylist=denylist.split(","),
require_approval_for_writes=False,
require_approval_for_apply=False,
)
@when("I call _guards_dict with the guard object")
def step_call_guards_dict(context: Context) -> None:
context.guards_dict_result = _guards_dict(context.guard_obj)
@when("I call _guards_dict with None")
def step_call_guards_dict_none(context: Context) -> None:
context.guards_dict_result = _guards_dict(None)
@then('the guards dict should contain key "{key}" with value {value:d}')
def step_guards_dict_int_value(context: Context, key: str, value: int) -> None:
result = context.guards_dict_result
assert result is not None, "Expected non-None result from _guards_dict"
assert key in result, f"Key '{key}' not found in guards dict: {result}"
assert result[key] == value, f"Expected {key}={value}, got {result[key]}"
@then('the guards dict should contain key "{key}" with value {value:g}')
def step_guards_dict_float_value(context: Context, key: str, value: float) -> None:
result = context.guards_dict_result
assert result is not None, "Expected non-None result from _guards_dict"
assert key in result, f"Key '{key}' not found in guards dict: {result}"
assert result[key] == value, f"Expected {key}={value}, got {result[key]}"
@then('the guards dict should contain key "{key}"')
def step_guards_dict_has_key(context: Context, key: str) -> None:
result = context.guards_dict_result
assert result is not None, "Expected non-None result from _guards_dict"
assert key in result, f"Key '{key}' not found in guards dict: {result}"
@then("the guards dict result should be None")
def step_guards_dict_is_none(context: Context) -> None:
assert context.guards_dict_result is None, (
f"Expected None, got {context.guards_dict_result}"
)
@then('the guards dict tool_allowlist should be "{expected}"')
def step_guards_dict_allowlist(context: Context, expected: str) -> None:
result = context.guards_dict_result
assert result is not None
expected_list = expected.split(",")
assert result["tool_allowlist"] == expected_list, (
f"Expected {expected_list}, got {result['tool_allowlist']}"
)
@then('the guards dict tool_denylist should be "{expected}"')
def step_guards_dict_denylist(context: Context, expected: str) -> None:
result = context.guards_dict_result
assert result is not None
expected_list = expected.split(",")
assert result["tool_denylist"] == expected_list, (
f"Expected {expected_list}, got {result['tool_denylist']}"
)
# ---------------------------------------------------------------------------
# _profile_spec_dict with guards (lines 91-97 indirect)
# ---------------------------------------------------------------------------
@given("an AutomationProfile with guards having require_approval_for_writes true")
def step_create_profile_with_guards(context: Context) -> None:
context.guarded_profile = _make_guarded_profile(
name="acme/spectest",
require_approval_for_writes=True,
)
@when("I call _profile_spec_dict with the guarded profile")
def step_call_profile_spec_dict(context: Context) -> None:
context.profile_spec_result = _profile_spec_dict(context.guarded_profile)
@then("the profile spec dict guards should not be None")
def step_spec_dict_guards_not_none(context: Context) -> None:
result = context.profile_spec_result
assert result is not None
assert result.get("guards") is not None, (
f"Expected guards to be non-None, got: {result.get('guards')}"
)
@then('the profile spec dict guards should have "{key}" as true')
def step_spec_dict_guards_key_true(context: Context, key: str) -> None:
result = context.profile_spec_result
assert result is not None
guards = result["guards"]
assert guards is not None
assert guards[key] is True, f"Expected {key}=True, got {guards[key]}"
# ---------------------------------------------------------------------------
# CLI show with guards - rich format (lines 194-204)
# ---------------------------------------------------------------------------
@given('a custom guarded profile "{name}" is stored in the repo')
def step_store_guarded_profile(context: Context, name: str) -> None:
profile = _make_guarded_profile(
name=name,
max_tool_calls=5,
max_total_cost=100.0,
tool_allowlist=["read_file", "search"],
tool_denylist=["rm_rf"],
require_approval_for_writes=True,
require_approval_for_apply=True,
)
context._ap_service._repo.upsert(profile)
@when('I run automation-profile show "{name}" in rich format')
def step_show_guarded_rich(context: Context, name: str) -> None:
context.result = context.runner.invoke(profile_app, ["show", name])
@when('I run automation-profile show "{name}" with format "{fmt}"')
def step_show_guarded_fmt(context: Context, name: str, fmt: str) -> None:
context.result = context.runner.invoke(
profile_app,
["show", name, "--format", fmt],
)
@then("the guards coverage show command should succeed")
def step_guards_coverage_show_succeed(context: Context) -> None:
assert context.result is not None
assert context.result.exit_code == 0, (
f"Expected exit 0, got {context.result.exit_code}. "
f"Output: {context.result.output}"
)
@then('the guards coverage output should contain "{text}"')
def step_guards_coverage_output_contains(context: Context, text: str) -> None:
assert context.result is not None
assert text in context.result.output, (
f"Expected '{text}' in output. Got:\n{context.result.output}"
)
# ---------------------------------------------------------------------------
# CLI add with guards config (lines 194-204 via add path)
# ---------------------------------------------------------------------------
@given('a YAML config file for a guarded profile "{name}"')
def step_guarded_yaml_config(context: Context, name: str) -> None:
yaml_content = f"""\
name: {name}
description: Guarded profile for testing
schema_version: "1.0"
decompose_task: 0.5
create_tool: 0.4
select_tool: 0.3
guards:
max_tool_calls_per_step: 8
max_total_cost: 200.0
tool_allowlist:
- read_file
- search
tool_denylist:
- rm_rf
require_approval_for_writes: true
require_approval_for_apply: false
"""
context.guarded_yaml_path = _write_temp_yaml(context, yaml_content)
@when("I run automation-profile add with the guarded config file")
def step_add_guarded_config(context: Context) -> None:
context.result = context.runner.invoke(
profile_app,
["add", "--config", context.guarded_yaml_path],
)
@then("the guards coverage add command should succeed")
def step_guards_coverage_add_succeed(context: Context) -> None:
assert context.result is not None
assert context.result.exit_code == 0, (
f"Expected exit 0, got {context.result.exit_code}. "
f"Output: {context.result.output}"
)
# ---------------------------------------------------------------------------
# CLI remove guarded profile with --format json (lines 91-97 via remove)
# ---------------------------------------------------------------------------
@when('I run automation-profile remove "{name}" with --yes and format "{fmt}"')
def step_remove_guarded_fmt(context: Context, name: str, fmt: str) -> None:
context.result = context.runner.invoke(
profile_app,
["remove", name, "--yes", "--format", fmt],
)
@then("the guards coverage remove command should succeed")
def step_guards_coverage_remove_succeed(context: Context) -> None:
assert context.result is not None
assert context.result.exit_code == 0, (
f"Expected exit 0, got {context.result.exit_code}. "
f"Output: {context.result.output}"
)