forked from HAL9000/cleveragents-core
007af498b8
Renamed all 11 task-type confidence threshold fields in AutomationProfile from phase-transition semantics to spec-defined task-type semantics. Updated all 8 built-in profiles, CLI formatting, YAML schema, services, and all Behave/Robot tests referencing the old field names. Post-review fixes: - Fixed 24 stale old field names in M6 fixture files (automation_profiles.json, autonomy_guardrails.json) - Added model_validator(mode='before') to detect legacy field names and raise actionable ValueError with rename mapping - Added semantic bridge comments in PlanLifecycleService mapping task-type thresholds to phase-transition gates - Added threshold_field to structured log messages for observability - Restored categorised CLI automation-profile show output to match spec (Phase Transitions / Decision Automation / Self-Repair / Execution Controls) instead of flat list - Added missing access_network field to spec show output examples (Rich, Plain, JSON, YAML variants) - Aligned ADR-017 profile fields table to all 11 fields with descriptions matching spec Automatable Tasks table - Aligned automation_profiles.md threshold descriptions with spec - Added spec section references in phase_reversion.md, error_recovery.md, and plan_execute.md for field naming context - Extended repository roundtrip test to assert all 11 threshold fields - Fixed benchmark _make_profile() passing safety fields as top-level kwargs instead of via SafetyProfile sub-model (incompatible with extra="forbid") - Aligned CLI JSON/YAML output structure for automation-profile show with the specification grouped format (phase_transitions, decision_automation, self_repair, execution_controls) - Moved safety boolean fields into the Execution Controls section of Rich output per spec examples - Reverted auto profile description to "Fully automatic except apply" per specification (line 16703, line 28406) - Improved bridge comments in test steps with semantic context for threshold-to-gate mappings ISSUES CLOSED: #902
371 lines
13 KiB
Python
371 lines
13 KiB
Python
"""Step definitions for Automation Profile CLI guards coverage boost.
|
|
|
|
Targets uncovered lines 91-97 (_guards_dict non-None branch) and
|
|
lines 194-204 (_print_profile rich guards display block).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
from typer.testing import CliRunner
|
|
|
|
from cleveragents.cli.commands.automation_profile import (
|
|
_guards_dict,
|
|
_profile_spec_dict,
|
|
)
|
|
from cleveragents.cli.commands.automation_profile import (
|
|
app as profile_app,
|
|
)
|
|
from cleveragents.domain.models.core.automation_guard import AutomationGuard
|
|
from cleveragents.domain.models.core.automation_profile import AutomationProfile
|
|
|
|
|
|
def _create_in_memory_profile_service():
|
|
"""Create an AutomationProfileService backed by an in-memory SQLite DB."""
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from cleveragents.application.services.automation_profile_service import (
|
|
AutomationProfileService,
|
|
)
|
|
from cleveragents.infrastructure.database.models import Base
|
|
from cleveragents.infrastructure.database.repositories import (
|
|
AutomationProfileRepository,
|
|
)
|
|
|
|
engine = create_engine("sqlite:///:memory:", echo=False)
|
|
Base.metadata.create_all(engine)
|
|
factory = sessionmaker(bind=engine, expire_on_commit=False)
|
|
repo = AutomationProfileRepository(session_factory=factory, auto_commit=True)
|
|
return AutomationProfileService(repo=repo)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_guarded_profile(
|
|
name: str = "acme/guarded",
|
|
description: str = "Profile with guards",
|
|
max_tool_calls: int = 5,
|
|
max_total_cost: float = 100.0,
|
|
tool_allowlist: list[str] | None = None,
|
|
tool_denylist: list[str] | None = None,
|
|
require_approval_for_writes: bool = True,
|
|
require_approval_for_apply: bool = False,
|
|
) -> AutomationProfile:
|
|
"""Create an AutomationProfile with guards attached."""
|
|
guards = AutomationGuard(
|
|
max_tool_calls_per_step=max_tool_calls,
|
|
max_total_cost=max_total_cost,
|
|
tool_allowlist=tool_allowlist,
|
|
tool_denylist=tool_denylist,
|
|
require_approval_for_writes=require_approval_for_writes,
|
|
require_approval_for_apply=require_approval_for_apply,
|
|
)
|
|
return AutomationProfile(
|
|
name=name,
|
|
description=description,
|
|
schema_version="1.0",
|
|
decompose_task=0.5,
|
|
create_tool=0.4,
|
|
select_tool=0.3,
|
|
guards=guards,
|
|
)
|
|
|
|
|
|
def _write_temp_yaml(context: Context, content: str) -> str:
|
|
"""Write content to a temporary YAML file, tracked for cleanup."""
|
|
fd, path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(content)
|
|
if not hasattr(context, "_temp_files"):
|
|
context._temp_files = []
|
|
context._temp_files.append(path)
|
|
return path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Background
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a fresh automation profile CLI runner for guards coverage")
|
|
def step_fresh_guards_runner(context: Context) -> None:
|
|
from unittest.mock import patch
|
|
|
|
context.runner = CliRunner()
|
|
context.result = None
|
|
context.guards_dict_result = None
|
|
context.profile_spec_result = None
|
|
context.guard_obj = None
|
|
context.guarded_profile = None
|
|
# Create an in-memory service and patch _get_service for this scenario
|
|
context._ap_service = _create_in_memory_profile_service()
|
|
context._ap_patcher = patch(
|
|
"cleveragents.cli.commands.automation_profile._get_service",
|
|
return_value=context._ap_service,
|
|
)
|
|
context._ap_patcher.start()
|
|
if not hasattr(context, "_cleanup_handlers"):
|
|
context._cleanup_handlers = []
|
|
context._cleanup_handlers.append(context._ap_patcher.stop)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _guards_dict direct tests (lines 91-97)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given(
|
|
"an AutomationGuard with max_tool_calls_per_step {calls:d} and max_total_cost {cost:g}"
|
|
)
|
|
def step_create_guard_with_calls_and_cost(
|
|
context: Context,
|
|
calls: int,
|
|
cost: float,
|
|
) -> None:
|
|
context.guard_obj = AutomationGuard(
|
|
max_tool_calls_per_step=calls,
|
|
max_total_cost=cost,
|
|
tool_allowlist=None,
|
|
tool_denylist=None,
|
|
require_approval_for_writes=False,
|
|
require_approval_for_apply=False,
|
|
)
|
|
|
|
|
|
@given(
|
|
'an AutomationGuard with tool_allowlist "{allowlist}" and tool_denylist "{denylist}"'
|
|
)
|
|
def step_create_guard_with_lists(
|
|
context: Context,
|
|
allowlist: str,
|
|
denylist: str,
|
|
) -> None:
|
|
context.guard_obj = AutomationGuard(
|
|
max_tool_calls_per_step=None,
|
|
max_total_cost=None,
|
|
tool_allowlist=allowlist.split(","),
|
|
tool_denylist=denylist.split(","),
|
|
require_approval_for_writes=False,
|
|
require_approval_for_apply=False,
|
|
)
|
|
|
|
|
|
@when("I call _guards_dict with the guard object")
|
|
def step_call_guards_dict(context: Context) -> None:
|
|
context.guards_dict_result = _guards_dict(context.guard_obj)
|
|
|
|
|
|
@when("I call _guards_dict with None")
|
|
def step_call_guards_dict_none(context: Context) -> None:
|
|
context.guards_dict_result = _guards_dict(None)
|
|
|
|
|
|
@then('the guards dict should contain key "{key}" with value {value:d}')
|
|
def step_guards_dict_int_value(context: Context, key: str, value: int) -> None:
|
|
result = context.guards_dict_result
|
|
assert result is not None, "Expected non-None result from _guards_dict"
|
|
assert key in result, f"Key '{key}' not found in guards dict: {result}"
|
|
assert result[key] == value, f"Expected {key}={value}, got {result[key]}"
|
|
|
|
|
|
@then('the guards dict should contain key "{key}" with value {value:g}')
|
|
def step_guards_dict_float_value(context: Context, key: str, value: float) -> None:
|
|
result = context.guards_dict_result
|
|
assert result is not None, "Expected non-None result from _guards_dict"
|
|
assert key in result, f"Key '{key}' not found in guards dict: {result}"
|
|
assert result[key] == value, f"Expected {key}={value}, got {result[key]}"
|
|
|
|
|
|
@then('the guards dict should contain key "{key}"')
|
|
def step_guards_dict_has_key(context: Context, key: str) -> None:
|
|
result = context.guards_dict_result
|
|
assert result is not None, "Expected non-None result from _guards_dict"
|
|
assert key in result, f"Key '{key}' not found in guards dict: {result}"
|
|
|
|
|
|
@then("the guards dict result should be None")
|
|
def step_guards_dict_is_none(context: Context) -> None:
|
|
assert context.guards_dict_result is None, (
|
|
f"Expected None, got {context.guards_dict_result}"
|
|
)
|
|
|
|
|
|
@then('the guards dict tool_allowlist should be "{expected}"')
|
|
def step_guards_dict_allowlist(context: Context, expected: str) -> None:
|
|
result = context.guards_dict_result
|
|
assert result is not None
|
|
expected_list = expected.split(",")
|
|
assert result["tool_allowlist"] == expected_list, (
|
|
f"Expected {expected_list}, got {result['tool_allowlist']}"
|
|
)
|
|
|
|
|
|
@then('the guards dict tool_denylist should be "{expected}"')
|
|
def step_guards_dict_denylist(context: Context, expected: str) -> None:
|
|
result = context.guards_dict_result
|
|
assert result is not None
|
|
expected_list = expected.split(",")
|
|
assert result["tool_denylist"] == expected_list, (
|
|
f"Expected {expected_list}, got {result['tool_denylist']}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _profile_spec_dict with guards (lines 91-97 indirect)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("an AutomationProfile with guards having require_approval_for_writes true")
|
|
def step_create_profile_with_guards(context: Context) -> None:
|
|
context.guarded_profile = _make_guarded_profile(
|
|
name="acme/spectest",
|
|
require_approval_for_writes=True,
|
|
)
|
|
|
|
|
|
@when("I call _profile_spec_dict with the guarded profile")
|
|
def step_call_profile_spec_dict(context: Context) -> None:
|
|
context.profile_spec_result = _profile_spec_dict(context.guarded_profile)
|
|
|
|
|
|
@then("the profile spec dict guards should not be None")
|
|
def step_spec_dict_guards_not_none(context: Context) -> None:
|
|
result = context.profile_spec_result
|
|
assert result is not None
|
|
assert result.get("guards") is not None, (
|
|
f"Expected guards to be non-None, got: {result.get('guards')}"
|
|
)
|
|
|
|
|
|
@then('the profile spec dict guards should have "{key}" as true')
|
|
def step_spec_dict_guards_key_true(context: Context, key: str) -> None:
|
|
result = context.profile_spec_result
|
|
assert result is not None
|
|
guards = result["guards"]
|
|
assert guards is not None
|
|
assert guards[key] is True, f"Expected {key}=True, got {guards[key]}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI show with guards - rich format (lines 194-204)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a custom guarded profile "{name}" is stored in the repo')
|
|
def step_store_guarded_profile(context: Context, name: str) -> None:
|
|
profile = _make_guarded_profile(
|
|
name=name,
|
|
max_tool_calls=5,
|
|
max_total_cost=100.0,
|
|
tool_allowlist=["read_file", "search"],
|
|
tool_denylist=["rm_rf"],
|
|
require_approval_for_writes=True,
|
|
require_approval_for_apply=True,
|
|
)
|
|
context._ap_service._repo.upsert(profile)
|
|
|
|
|
|
@when('I run automation-profile show "{name}" in rich format')
|
|
def step_show_guarded_rich(context: Context, name: str) -> None:
|
|
context.result = context.runner.invoke(profile_app, ["show", name])
|
|
|
|
|
|
@when('I run automation-profile show "{name}" with format "{fmt}"')
|
|
def step_show_guarded_fmt(context: Context, name: str, fmt: str) -> None:
|
|
context.result = context.runner.invoke(
|
|
profile_app,
|
|
["show", name, "--format", fmt],
|
|
)
|
|
|
|
|
|
@then("the guards coverage show command should succeed")
|
|
def step_guards_coverage_show_succeed(context: Context) -> None:
|
|
assert context.result is not None
|
|
assert context.result.exit_code == 0, (
|
|
f"Expected exit 0, got {context.result.exit_code}. "
|
|
f"Output: {context.result.output}"
|
|
)
|
|
|
|
|
|
@then('the guards coverage output should contain "{text}"')
|
|
def step_guards_coverage_output_contains(context: Context, text: str) -> None:
|
|
assert context.result is not None
|
|
assert text in context.result.output, (
|
|
f"Expected '{text}' in output. Got:\n{context.result.output}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI add with guards config (lines 194-204 via add path)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a YAML config file for a guarded profile "{name}"')
|
|
def step_guarded_yaml_config(context: Context, name: str) -> None:
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: Guarded profile for testing
|
|
schema_version: "1.0"
|
|
decompose_task: 0.5
|
|
create_tool: 0.4
|
|
select_tool: 0.3
|
|
guards:
|
|
max_tool_calls_per_step: 8
|
|
max_total_cost: 200.0
|
|
tool_allowlist:
|
|
- read_file
|
|
- search
|
|
tool_denylist:
|
|
- rm_rf
|
|
require_approval_for_writes: true
|
|
require_approval_for_apply: false
|
|
"""
|
|
context.guarded_yaml_path = _write_temp_yaml(context, yaml_content)
|
|
|
|
|
|
@when("I run automation-profile add with the guarded config file")
|
|
def step_add_guarded_config(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
profile_app,
|
|
["add", "--config", context.guarded_yaml_path],
|
|
)
|
|
|
|
|
|
@then("the guards coverage add command should succeed")
|
|
def step_guards_coverage_add_succeed(context: Context) -> None:
|
|
assert context.result is not None
|
|
assert context.result.exit_code == 0, (
|
|
f"Expected exit 0, got {context.result.exit_code}. "
|
|
f"Output: {context.result.output}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI remove guarded profile with --format json (lines 91-97 via remove)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run automation-profile remove "{name}" with --yes and format "{fmt}"')
|
|
def step_remove_guarded_fmt(context: Context, name: str, fmt: str) -> None:
|
|
context.result = context.runner.invoke(
|
|
profile_app,
|
|
["remove", name, "--yes", "--format", fmt],
|
|
)
|
|
|
|
|
|
@then("the guards coverage remove command should succeed")
|
|
def step_guards_coverage_remove_succeed(context: Context) -> None:
|
|
assert context.result is not None
|
|
assert context.result.exit_code == 0, (
|
|
f"Expected exit 0, got {context.result.exit_code}. "
|
|
f"Output: {context.result.output}"
|
|
)
|