diff --git a/features/automation_profile_cli.feature b/features/automation_profile_cli.feature index a28883860..b2c491937 100644 --- a/features/automation_profile_cli.feature +++ b/features/automation_profile_cli.feature @@ -13,6 +13,29 @@ Feature: Automation Profile CLI commands Then the automation-profile add should succeed And the automation-profile output should contain "acme/strict" + @tdd_issue @tdd_issue_6345 + Scenario: Add profile JSON output uses flat schema + Given a valid automation profile config YAML file + When I run automation-profile add with --config pointing to the YAML file and --format json + Then the automation-profile add should succeed + And the automation-profile add json output matches the flat schema + + @tdd_issue @tdd_issue_6345 + Scenario: Add profile YAML output uses flat schema + Given a valid automation profile config YAML file + When I run automation-profile add with --config pointing to the YAML file and --format yaml + Then the automation-profile add should succeed + And the automation-profile add yaml output matches the flat schema + + @tdd_issue @tdd_issue_6345 + Scenario: Add profile plain output includes created timestamp + Given a valid automation profile config YAML file + When I run automation-profile add with --config pointing to the YAML file and --format plain + Then the automation-profile add should succeed + And the automation-profile output should contain "Profile Registered" + And the automation-profile output should contain "Created:" + And the automation-profile output should contain "require_checkpoints: true" + Scenario: Add profile with --update for existing custom profile Given a valid automation profile config YAML file And the custom profile "acme/strict" already exists diff --git a/features/steps/automation_profile_cli_steps.py b/features/steps/automation_profile_cli_steps.py index 7197d96d5..7925b002e 100644 --- a/features/steps/automation_profile_cli_steps.py +++ b/features/steps/automation_profile_cli_steps.py @@ -6,32 +6,31 @@ import json import os import re import tempfile +from datetime import datetime from unittest.mock import MagicMock, patch +import yaml from behave import given, then, when from behave.runner import Context +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from typer.testing import CliRunner -from cleveragents.cli.commands.automation_profile import ( - app as profile_app, +from cleveragents.application.services.automation_profile_service import ( + AutomationProfileService, ) +from cleveragents.cli.commands.automation_profile import app as profile_app +from cleveragents.cli.commands.plan import app as plan_app from cleveragents.domain.models.core.automation_profile import AutomationProfile from cleveragents.domain.models.core.safety_profile import SafetyProfile +from cleveragents.infrastructure.database.models import Base +from cleveragents.infrastructure.database.repositories import ( + AutomationProfileRepository, +) def _create_in_memory_profile_service(): """Create an AutomationProfileService backed by an in-memory SQLite DB.""" - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - - from cleveragents.application.services.automation_profile_service import ( - AutomationProfileService, - ) - from cleveragents.infrastructure.database.models import Base - from cleveragents.infrastructure.database.repositories import ( - AutomationProfileRepository, - ) - engine = create_engine("sqlite:///:memory:", echo=False) Base.metadata.create_all(engine) factory = sessionmaker(bind=engine, expire_on_commit=False) @@ -262,6 +261,39 @@ def step_run_add_config(context: Context) -> None: ) +@when( + "I run automation-profile add with --config pointing to the YAML file and --format json" +) +def step_run_add_config_format_json(context: Context) -> None: + """Run the add command with --config producing JSON output.""" + context.result = context.runner.invoke( + profile_app, + ["add", "--config", context.yaml_path, "--format", "json"], + ) + + +@when( + "I run automation-profile add with --config pointing to the YAML file and --format yaml" +) +def step_run_add_config_format_yaml(context: Context) -> None: + """Run the add command with --config producing YAML output.""" + context.result = context.runner.invoke( + profile_app, + ["add", "--config", context.yaml_path, "--format", "yaml"], + ) + + +@when( + "I run automation-profile add with --config pointing to the YAML file and --format plain" +) +def step_run_add_config_format_plain(context: Context) -> None: + """Run the add command with --config producing plain output.""" + context.result = context.runner.invoke( + profile_app, + ["add", "--config", context.yaml_path, "--format", "plain"], + ) + + @when("I run automation-profile add with --config and --update") def step_run_add_config_update(context: Context) -> None: """Run the add command with --config and --update.""" @@ -434,8 +466,6 @@ def step_run_remove_yes_json(context: Context, name: str) -> None: @when('I invoke plan use with --automation-level "{level}"') def step_invoke_plan_use_automation_level(context: Context, level: str) -> None: """Invoke plan use with the deprecated --automation-level flag.""" - from cleveragents.cli.commands.plan import app as plan_app - # Mock the lifecycle service so we don't need real infra mock_service = MagicMock() mock_plan = MagicMock() @@ -510,6 +540,168 @@ def step_add_should_succeed(context: Context) -> None: ) +@then("the automation-profile add json output matches the flat schema") +def step_add_json_matches_flat_schema(context: Context) -> None: + """Validate JSON output uses the spec-defined flat schema.""" + assert context.result is not None + output = context.result.output + parsed: dict[str, object] | None = None + for idx, ch in enumerate(output): + if ch == "{": + try: + parsed = json.loads(output[idx:]) + break + except json.JSONDecodeError: + continue + assert parsed is not None, f"No valid JSON found in output: {output[:200]}" + assert isinstance(parsed, dict), f"Expected dict at top level, got {type(parsed)}" + + assert parsed.get("command") == "automation-profile add", ( + "JSON envelope command field must be 'automation-profile add'" + ) + + data = parsed.get("data") + assert isinstance(data, dict), f"Envelope data must be dict, got {type(data)}" + + created = data.get("created") + assert isinstance(created, str), "created must be an ISO-8601 string" + try: + datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError as exc: # pragma: no cover - defensive + raise AssertionError(f"created is not ISO-8601: {created}") from exc + + expected_thresholds = { + "decompose_task", + "create_tool", + "select_tool", + "edit_code", + "execute_command", + "create_file", + "delete_content", + "access_network", + "install_dependency", + "modify_config", + "approve_plan", + } + thresholds = data.get("thresholds") + assert isinstance(thresholds, dict), "thresholds must be a dict" + assert set(thresholds.keys()) == expected_thresholds, ( + f"threshold keys mismatch: {set(thresholds.keys())}" + ) + for key in expected_thresholds: + assert isinstance(thresholds[key], (int, float)), ( + f"threshold {key} must be numeric" + ) + + expected_flags = { + "require_sandbox", + "require_checkpoints", + "allow_unsafe_tools", + } + flags = data.get("flags") + assert isinstance(flags, dict), "flags must be a dict" + assert set(flags.keys()) == expected_flags, ( + f"flag keys mismatch: {set(flags.keys())}" + ) + for key in expected_flags: + assert isinstance(flags[key], bool), f"flag {key} must be boolean" + + forbidden_keys = { + "phase_transitions", + "decision_automation", + "self_repair", + "execution_controls", + "schema_version", + "guards", + } + for key in forbidden_keys: + assert key not in data, f"Unexpected grouped schema field present: {key}" + + messages = parsed.get("messages") + assert isinstance(messages, list), "messages must be a list" + assert messages, "messages list must not be empty" + first_message = messages[0] + assert isinstance(first_message, dict), "messages entries must be mappings" + assert first_message.get("level") == "ok", "message level must be 'ok'" + assert first_message.get("text") == "Profile registered", ( + "message text must indicate profile registration" + ) + + +@then("the automation-profile add yaml output matches the flat schema") +def step_add_yaml_matches_flat_schema(context: Context) -> None: + """Validate YAML output uses the spec-defined flat schema.""" + assert context.result is not None + parsed = yaml.safe_load(context.result.output) + assert isinstance(parsed, dict), f"Expected dict at top level, got {type(parsed)}" + assert parsed.get("command") == "automation-profile add", ( + "YAML envelope command field must be 'automation-profile add'" + ) + + data = parsed.get("data") + assert isinstance(data, dict), "Envelope data must be a mapping" + + created = data.get("created") + assert isinstance(created, str), "created must be a string" + try: + datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError as exc: # pragma: no cover - defensive + raise AssertionError(f"created is not ISO-8601: {created}") from exc + + expected_thresholds = { + "decompose_task", + "create_tool", + "select_tool", + "edit_code", + "execute_command", + "create_file", + "delete_content", + "access_network", + "install_dependency", + "modify_config", + "approve_plan", + } + thresholds = data.get("thresholds") + assert isinstance(thresholds, dict), "thresholds must be a mapping" + assert set(thresholds.keys()) == expected_thresholds, ( + f"threshold keys mismatch: {set(thresholds.keys())}" + ) + + expected_flags = { + "require_sandbox", + "require_checkpoints", + "allow_unsafe_tools", + } + flags = data.get("flags") + assert isinstance(flags, dict), "flags must be a mapping" + assert set(flags.keys()) == expected_flags, ( + f"flag keys mismatch: {set(flags.keys())}" + ) + for key in expected_flags: + assert isinstance(flags[key], bool), f"flag {key} must be boolean" + + forbidden_keys = { + "phase_transitions", + "decision_automation", + "self_repair", + "execution_controls", + "schema_version", + "guards", + } + for key in forbidden_keys: + assert key not in data, f"Unexpected grouped schema field present: {key}" + + messages = parsed.get("messages") + assert isinstance(messages, list), "messages must be a list" + assert messages, "messages list must not be empty" + first_message = messages[0] + assert isinstance(first_message, dict), "messages entries must be mappings" + assert first_message.get("level") == "ok", "message level must be 'ok'" + assert first_message.get("text") == "Profile registered", ( + "message text must indicate profile registration" + ) + + @then('the automation-profile output should contain "{text}"') def step_output_contains(context: Context, text: str) -> None: """Assert the command output contains specific text.""" diff --git a/src/cleveragents/cli/commands/automation_profile.py b/src/cleveragents/cli/commands/automation_profile.py index 2fc2c99d6..d318276a5 100644 --- a/src/cleveragents/cli/commands/automation_profile.py +++ b/src/cleveragents/cli/commands/automation_profile.py @@ -11,6 +11,9 @@ from __future__ import annotations import contextlib import re import warnings +from collections.abc import Callable +from datetime import UTC, datetime +from functools import partial from pathlib import Path from typing import Annotated, Any @@ -30,9 +33,9 @@ from cleveragents.core.exceptions import ( NotFoundError, ValidationError, ) +from cleveragents.domain.models.core.automation_guard import AutomationGuard from cleveragents.domain.models.core.automation_profile import ( BUILTIN_PROFILES, - AutomationGuard, AutomationProfile, ) @@ -110,6 +113,165 @@ def _profile_spec_dict(profile: AutomationProfile) -> dict[str, object]: return result +def _profile_add_dict( + profile: AutomationProfile, + *, + created_iso: str, +) -> dict[str, object]: + """Return automation-profile add data using the spec-defined schema.""" + + thresholds: dict[str, float] = { + "decompose_task": profile.decompose_task, + "create_tool": profile.create_tool, + "select_tool": profile.select_tool, + "edit_code": profile.edit_code, + "execute_command": profile.execute_command, + "create_file": profile.create_file, + "delete_content": profile.delete_content, + "access_network": profile.access_network, + "install_dependency": profile.install_dependency, + "modify_config": profile.modify_config, + "approve_plan": profile.approve_plan, + } + + flags: dict[str, bool] = { + "require_sandbox": profile.safety.require_sandbox, + "require_checkpoints": profile.safety.require_checkpoints, + "allow_unsafe_tools": profile.safety.allow_unsafe_tools, + } + + return { + "name": profile.name, + "description": profile.description, + "created": created_iso, + "thresholds": thresholds, + "flags": flags, + } + + +def _current_utc_iso() -> str: + """Return current UTC timestamp without microseconds and with Z suffix.""" + + return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _render_profile_add_rich( + profile: AutomationProfile, + *, + created_iso: str, + heading: str, + success_message: str, +) -> None: + """Render automation-profile add rich output using the spec layout.""" + + header_lines = ( + f"[cyan]Name:[/cyan] {profile.name}\n" + f"[blue]Description:[/blue] {profile.description}\n" + f"[green]Created:[/green] {created_iso}" + ) + console.print(Panel.fit(header_lines, title=heading)) + + threshold_table = Table.grid(padding=(0, 1)) + threshold_table.add_column(justify="right", style="cyan", no_wrap=True) + threshold_table.add_column(justify="left", style="white") + + threshold_entries: list[tuple[str, object]] = [ + ("decompose_task", profile.decompose_task), + ("create_tool", profile.create_tool), + ("select_tool", profile.select_tool), + ("edit_code", profile.edit_code), + ("execute_command", profile.execute_command), + ("create_file", profile.create_file), + ("delete_content", profile.delete_content), + ("access_network", profile.access_network), + ("install_dependency", profile.install_dependency), + ("modify_config", profile.modify_config), + ("approve_plan", profile.approve_plan), + ("require_sandbox", profile.safety.require_sandbox), + ("require_checkpoints", profile.safety.require_checkpoints), + ("allow_unsafe_tools", profile.safety.allow_unsafe_tools), + ] + + for key, value in threshold_entries: + if isinstance(value, bool): + rendered = "true" if value else "false" + elif isinstance(value, (int, float)): + rendered = f"{float(value):.1f}" + else: + rendered = str(value) + threshold_table.add_row(f"{key}:", rendered) + + console.print(Panel(threshold_table, title="Confidence Thresholds")) + + if profile.guards is not None: + g = profile.guards + guards_table = Table.grid(padding=(0, 1)) + guards_table.add_column(justify="right", style="cyan", no_wrap=True) + guards_table.add_column(justify="left", style="white") + guards_table.add_row("max_tool_calls_per_step:", str(g.max_tool_calls_per_step)) + guards_table.add_row("max_total_cost:", str(g.max_total_cost)) + guards_table.add_row("tool_allowlist:", str(g.tool_allowlist)) + guards_table.add_row("tool_denylist:", str(g.tool_denylist)) + guards_table.add_row( + "require_approval_for_writes:", str(g.require_approval_for_writes) + ) + guards_table.add_row( + "require_approval_for_apply:", str(g.require_approval_for_apply) + ) + console.print(Panel(guards_table, title="Guards")) + + console.print(f"[green]\u2713 OK[/green] {success_message}") + + +def _render_profile_add_plain( + profile: AutomationProfile, + *, + created_iso: str, + heading: str, + success_message: str, +) -> None: + """Render automation-profile add plain output using the spec layout.""" + + lines = [ + heading, + f" Name: {profile.name}", + f" Description: {profile.description}", + f" Created: {created_iso}", + "", + "Confidence Thresholds", + ] + + threshold_entries: list[tuple[str, object]] = [ + ("decompose_task", profile.decompose_task), + ("create_tool", profile.create_tool), + ("select_tool", profile.select_tool), + ("edit_code", profile.edit_code), + ("execute_command", profile.execute_command), + ("create_file", profile.create_file), + ("delete_content", profile.delete_content), + ("access_network", profile.access_network), + ("install_dependency", profile.install_dependency), + ("modify_config", profile.modify_config), + ("approve_plan", profile.approve_plan), + ("require_sandbox", profile.safety.require_sandbox), + ("require_checkpoints", profile.safety.require_checkpoints), + ("allow_unsafe_tools", profile.safety.allow_unsafe_tools), + ] + + for key, value in threshold_entries: + if isinstance(value, bool): + rendered = "true" if value else "false" + elif isinstance(value, (int, float)): + rendered = f"{float(value):.1f}" + else: + rendered = str(value) + lines.append(f" {key}: {rendered}") + + lines.extend(["", f"[OK] {success_message}"]) + + console.print("\n".join(lines)) + + def _threshold_summary(profile: AutomationProfile) -> str: """Return a compact summary of key thresholds.""" return ( @@ -123,11 +285,26 @@ def _print_profile( profile: AutomationProfile, title: str = "Automation Profile", fmt: str = OutputFormat.RICH.value, + *, + payload_factory: Callable[[AutomationProfile], dict[str, object]] | None = None, + command_name: str = "automation-profile show", + rich_renderer: Callable[[AutomationProfile], None] | None = None, + plain_renderer: Callable[[AutomationProfile], None] | None = None, + messages: list[dict[str, str]] | None = None, ) -> None: """Print profile details in the requested format.""" + if fmt == OutputFormat.PLAIN.value and plain_renderer is not None: + plain_renderer(profile) + return + if fmt != OutputFormat.RICH.value: - data = _profile_spec_dict(profile) - console.print(format_output(data, fmt)) + factory = payload_factory or _profile_spec_dict + data = factory(profile) + console.print(format_output(data, fmt, command=command_name, messages=messages)) + return + + if rich_renderer is not None: + rich_renderer(profile) return source = "built-in" if profile.name in BUILTIN_PROFILES else "custom" @@ -247,9 +424,33 @@ def add_profile( title = "Profile Updated" else: profile = service.create_profile(config_data) - title = "Profile Added" + title = "Profile Added" if update else "Profile Registered" - _print_profile(profile, title=title, fmt=fmt) + created_iso = _current_utc_iso() + success_message = ( + "Profile updated" if update and existing else "Profile registered" + ) + messages = [{"level": "ok", "text": success_message}] + _print_profile( + profile, + title=title, + fmt=fmt, + payload_factory=partial(_profile_add_dict, created_iso=created_iso), + command_name="automation-profile add", + rich_renderer=partial( + _render_profile_add_rich, + created_iso=created_iso, + heading=title, + success_message=success_message, + ), + plain_renderer=partial( + _render_profile_add_plain, + created_iso=created_iso, + heading=title, + success_message=success_message, + ), + messages=messages, + ) except FileNotFoundError as exc: console.print(f"[red]Config file error:[/red] {exc}")