fix(cli): fix automation-profile add JSON/YAML output format #6598

Merged
HAL9000 merged 2 commits from fix/issue-6345-automation-profile-add-output into master 2026-05-31 17:29:08 +00:00
3 changed files with 436 additions and 20 deletions
+23
View File
@@ -13,6 +13,29 @@ Feature: Automation Profile CLI commands
Then the automation-profile add should succeed
And the automation-profile output should contain "acme/strict"
@tdd_issue @tdd_issue_6345
Scenario: Add profile JSON output uses flat schema
Given a valid automation profile config YAML file
When I run automation-profile add with --config pointing to the YAML file and --format json
Then the automation-profile add should succeed
And the automation-profile add json output matches the flat schema
@tdd_issue @tdd_issue_6345
Scenario: Add profile YAML output uses flat schema
Given a valid automation profile config YAML file
When I run automation-profile add with --config pointing to the YAML file and --format yaml
Then the automation-profile add should succeed
And the automation-profile add yaml output matches the flat schema
@tdd_issue @tdd_issue_6345
Scenario: Add profile plain output includes created timestamp
Given a valid automation profile config YAML file
When I run automation-profile add with --config pointing to the YAML file and --format plain
Then the automation-profile add should succeed
And the automation-profile output should contain "Profile Registered"
And the automation-profile output should contain "Created:"
And the automation-profile output should contain "require_checkpoints: true"
Scenario: Add profile with --update for existing custom profile
Given a valid automation profile config YAML file
And the custom profile "acme/strict" already exists
+207 -15
View File
@@ -6,32 +6,31 @@ import json
import os
import re
import tempfile
from datetime import datetime
from unittest.mock import MagicMock, patch
import yaml
from behave import given, then, when
from behave.runner import Context
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from typer.testing import CliRunner
from cleveragents.cli.commands.automation_profile import (
app as profile_app,
from cleveragents.application.services.automation_profile_service import (
AutomationProfileService,
)
from cleveragents.cli.commands.automation_profile import app as profile_app
from cleveragents.cli.commands.plan import app as plan_app
from cleveragents.domain.models.core.automation_profile import AutomationProfile
from cleveragents.domain.models.core.safety_profile import SafetyProfile
from cleveragents.infrastructure.database.models import Base
from cleveragents.infrastructure.database.repositories import (
AutomationProfileRepository,
)
def _create_in_memory_profile_service():
"""Create an AutomationProfileService backed by an in-memory SQLite DB."""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from cleveragents.application.services.automation_profile_service import (
AutomationProfileService,
)
from cleveragents.infrastructure.database.models import Base
from cleveragents.infrastructure.database.repositories import (
AutomationProfileRepository,
)
engine = create_engine("sqlite:///:memory:", echo=False)
Base.metadata.create_all(engine)
factory = sessionmaker(bind=engine, expire_on_commit=False)
@@ -262,6 +261,39 @@ def step_run_add_config(context: Context) -> None:
)
@when(
"I run automation-profile add with --config pointing to the YAML file and --format json"
)
def step_run_add_config_format_json(context: Context) -> None:
"""Run the add command with --config producing JSON output."""
context.result = context.runner.invoke(
profile_app,
["add", "--config", context.yaml_path, "--format", "json"],
)
@when(
"I run automation-profile add with --config pointing to the YAML file and --format yaml"
)
def step_run_add_config_format_yaml(context: Context) -> None:
"""Run the add command with --config producing YAML output."""
context.result = context.runner.invoke(
profile_app,
["add", "--config", context.yaml_path, "--format", "yaml"],
)
@when(
"I run automation-profile add with --config pointing to the YAML file and --format plain"
)
def step_run_add_config_format_plain(context: Context) -> None:
"""Run the add command with --config producing plain output."""
context.result = context.runner.invoke(
profile_app,
["add", "--config", context.yaml_path, "--format", "plain"],
)
@when("I run automation-profile add with --config and --update")
def step_run_add_config_update(context: Context) -> None:
"""Run the add command with --config and --update."""
@@ -434,8 +466,6 @@ def step_run_remove_yes_json(context: Context, name: str) -> None:
@when('I invoke plan use with --automation-level "{level}"')
def step_invoke_plan_use_automation_level(context: Context, level: str) -> None:
"""Invoke plan use with the deprecated --automation-level flag."""
from cleveragents.cli.commands.plan import app as plan_app
# Mock the lifecycle service so we don't need real infra
mock_service = MagicMock()
mock_plan = MagicMock()
@@ -510,6 +540,168 @@ def step_add_should_succeed(context: Context) -> None:
)
@then("the automation-profile add json output matches the flat schema")
def step_add_json_matches_flat_schema(context: Context) -> None:
"""Validate JSON output uses the spec-defined flat schema."""
assert context.result is not None
output = context.result.output
parsed: dict[str, object] | None = None
for idx, ch in enumerate(output):
if ch == "{":
try:
parsed = json.loads(output[idx:])
break
except json.JSONDecodeError:
continue
assert parsed is not None, f"No valid JSON found in output: {output[:200]}"
assert isinstance(parsed, dict), f"Expected dict at top level, got {type(parsed)}"
assert parsed.get("command") == "automation-profile add", (
"JSON envelope command field must be 'automation-profile add'"
)
data = parsed.get("data")
assert isinstance(data, dict), f"Envelope data must be dict, got {type(data)}"
created = data.get("created")
assert isinstance(created, str), "created must be an ISO-8601 string"
try:
datetime.fromisoformat(created.replace("Z", "+00:00"))
except ValueError as exc: # pragma: no cover - defensive
raise AssertionError(f"created is not ISO-8601: {created}") from exc
expected_thresholds = {
"decompose_task",
"create_tool",
"select_tool",
"edit_code",
"execute_command",
"create_file",
"delete_content",
"access_network",
"install_dependency",
"modify_config",
"approve_plan",
}
thresholds = data.get("thresholds")
assert isinstance(thresholds, dict), "thresholds must be a dict"
assert set(thresholds.keys()) == expected_thresholds, (
f"threshold keys mismatch: {set(thresholds.keys())}"
)
for key in expected_thresholds:
assert isinstance(thresholds[key], (int, float)), (
f"threshold {key} must be numeric"
)
expected_flags = {
"require_sandbox",
"require_checkpoints",
"allow_unsafe_tools",
}
flags = data.get("flags")
assert isinstance(flags, dict), "flags must be a dict"
assert set(flags.keys()) == expected_flags, (
f"flag keys mismatch: {set(flags.keys())}"
)
for key in expected_flags:
assert isinstance(flags[key], bool), f"flag {key} must be boolean"
forbidden_keys = {
"phase_transitions",
"decision_automation",
"self_repair",
"execution_controls",
"schema_version",
"guards",
}
for key in forbidden_keys:
assert key not in data, f"Unexpected grouped schema field present: {key}"
messages = parsed.get("messages")
assert isinstance(messages, list), "messages must be a list"
assert messages, "messages list must not be empty"
first_message = messages[0]
assert isinstance(first_message, dict), "messages entries must be mappings"
assert first_message.get("level") == "ok", "message level must be 'ok'"
assert first_message.get("text") == "Profile registered", (
"message text must indicate profile registration"
)
@then("the automation-profile add yaml output matches the flat schema")
def step_add_yaml_matches_flat_schema(context: Context) -> None:
"""Validate YAML output uses the spec-defined flat schema."""
assert context.result is not None
parsed = yaml.safe_load(context.result.output)
assert isinstance(parsed, dict), f"Expected dict at top level, got {type(parsed)}"
assert parsed.get("command") == "automation-profile add", (
"YAML envelope command field must be 'automation-profile add'"
)
data = parsed.get("data")
assert isinstance(data, dict), "Envelope data must be a mapping"
created = data.get("created")
assert isinstance(created, str), "created must be a string"
try:
datetime.fromisoformat(created.replace("Z", "+00:00"))
except ValueError as exc: # pragma: no cover - defensive
raise AssertionError(f"created is not ISO-8601: {created}") from exc
expected_thresholds = {
"decompose_task",
"create_tool",
"select_tool",
"edit_code",
"execute_command",
"create_file",
"delete_content",
"access_network",
"install_dependency",
"modify_config",
"approve_plan",
}
thresholds = data.get("thresholds")
assert isinstance(thresholds, dict), "thresholds must be a mapping"
assert set(thresholds.keys()) == expected_thresholds, (
f"threshold keys mismatch: {set(thresholds.keys())}"
)
expected_flags = {
"require_sandbox",
"require_checkpoints",
"allow_unsafe_tools",
}
flags = data.get("flags")
assert isinstance(flags, dict), "flags must be a mapping"
assert set(flags.keys()) == expected_flags, (
f"flag keys mismatch: {set(flags.keys())}"
)
for key in expected_flags:
assert isinstance(flags[key], bool), f"flag {key} must be boolean"
forbidden_keys = {
"phase_transitions",
"decision_automation",
"self_repair",
"execution_controls",
"schema_version",
"guards",
}
for key in forbidden_keys:
assert key not in data, f"Unexpected grouped schema field present: {key}"
messages = parsed.get("messages")
assert isinstance(messages, list), "messages must be a list"
assert messages, "messages list must not be empty"
first_message = messages[0]
assert isinstance(first_message, dict), "messages entries must be mappings"
assert first_message.get("level") == "ok", "message level must be 'ok'"
assert first_message.get("text") == "Profile registered", (
"message text must indicate profile registration"
)
@then('the automation-profile output should contain "{text}"')
def step_output_contains(context: Context, text: str) -> None:
"""Assert the command output contains specific text."""
@@ -11,6 +11,9 @@ from __future__ import annotations
import contextlib
import re
import warnings
from collections.abc import Callable
from datetime import UTC, datetime
from functools import partial
from pathlib import Path
from typing import Annotated, Any
@@ -30,9 +33,9 @@ from cleveragents.core.exceptions import (
NotFoundError,
ValidationError,
)
from cleveragents.domain.models.core.automation_guard import AutomationGuard
from cleveragents.domain.models.core.automation_profile import (
BUILTIN_PROFILES,
AutomationGuard,
AutomationProfile,
)
@@ -110,6 +113,165 @@ def _profile_spec_dict(profile: AutomationProfile) -> dict[str, object]:
return result
def _profile_add_dict(
profile: AutomationProfile,
*,
created_iso: str,
) -> dict[str, object]:
"""Return automation-profile add data using the spec-defined schema."""
thresholds: dict[str, float] = {
"decompose_task": profile.decompose_task,
"create_tool": profile.create_tool,
"select_tool": profile.select_tool,
"edit_code": profile.edit_code,
"execute_command": profile.execute_command,
"create_file": profile.create_file,
"delete_content": profile.delete_content,
"access_network": profile.access_network,
"install_dependency": profile.install_dependency,
"modify_config": profile.modify_config,
"approve_plan": profile.approve_plan,
}
flags: dict[str, bool] = {
"require_sandbox": profile.safety.require_sandbox,
"require_checkpoints": profile.safety.require_checkpoints,
"allow_unsafe_tools": profile.safety.allow_unsafe_tools,
}
return {
"name": profile.name,
"description": profile.description,
"created": created_iso,
"thresholds": thresholds,
"flags": flags,
}
def _current_utc_iso() -> str:
"""Return current UTC timestamp without microseconds and with Z suffix."""
return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def _render_profile_add_rich(
profile: AutomationProfile,
*,
created_iso: str,
heading: str,
success_message: str,
) -> None:
"""Render automation-profile add rich output using the spec layout."""
header_lines = (
f"[cyan]Name:[/cyan] {profile.name}\n"
f"[blue]Description:[/blue] {profile.description}\n"
f"[green]Created:[/green] {created_iso}"
)
console.print(Panel.fit(header_lines, title=heading))
threshold_table = Table.grid(padding=(0, 1))
threshold_table.add_column(justify="right", style="cyan", no_wrap=True)
threshold_table.add_column(justify="left", style="white")
threshold_entries: list[tuple[str, object]] = [
("decompose_task", profile.decompose_task),
("create_tool", profile.create_tool),
("select_tool", profile.select_tool),
("edit_code", profile.edit_code),
("execute_command", profile.execute_command),
("create_file", profile.create_file),
("delete_content", profile.delete_content),
("access_network", profile.access_network),
("install_dependency", profile.install_dependency),
("modify_config", profile.modify_config),
("approve_plan", profile.approve_plan),
("require_sandbox", profile.safety.require_sandbox),
("require_checkpoints", profile.safety.require_checkpoints),
("allow_unsafe_tools", profile.safety.allow_unsafe_tools),
]
for key, value in threshold_entries:
if isinstance(value, bool):
rendered = "true" if value else "false"
elif isinstance(value, (int, float)):
rendered = f"{float(value):.1f}"
else:
rendered = str(value)
threshold_table.add_row(f"{key}:", rendered)
console.print(Panel(threshold_table, title="Confidence Thresholds"))
if profile.guards is not None:
g = profile.guards
guards_table = Table.grid(padding=(0, 1))
guards_table.add_column(justify="right", style="cyan", no_wrap=True)
guards_table.add_column(justify="left", style="white")
guards_table.add_row("max_tool_calls_per_step:", str(g.max_tool_calls_per_step))
guards_table.add_row("max_total_cost:", str(g.max_total_cost))
guards_table.add_row("tool_allowlist:", str(g.tool_allowlist))
guards_table.add_row("tool_denylist:", str(g.tool_denylist))
guards_table.add_row(
"require_approval_for_writes:", str(g.require_approval_for_writes)
)
guards_table.add_row(
"require_approval_for_apply:", str(g.require_approval_for_apply)
)
console.print(Panel(guards_table, title="Guards"))
console.print(f"[green]\u2713 OK[/green] {success_message}")
def _render_profile_add_plain(
profile: AutomationProfile,
*,
created_iso: str,
heading: str,
success_message: str,
) -> None:
"""Render automation-profile add plain output using the spec layout."""
lines = [
heading,
f" Name: {profile.name}",
f" Description: {profile.description}",
f" Created: {created_iso}",
"",
"Confidence Thresholds",
]
threshold_entries: list[tuple[str, object]] = [
("decompose_task", profile.decompose_task),
("create_tool", profile.create_tool),
("select_tool", profile.select_tool),
("edit_code", profile.edit_code),
("execute_command", profile.execute_command),
("create_file", profile.create_file),
("delete_content", profile.delete_content),
("access_network", profile.access_network),
("install_dependency", profile.install_dependency),
("modify_config", profile.modify_config),
("approve_plan", profile.approve_plan),
("require_sandbox", profile.safety.require_sandbox),
("require_checkpoints", profile.safety.require_checkpoints),
("allow_unsafe_tools", profile.safety.allow_unsafe_tools),
]
for key, value in threshold_entries:
if isinstance(value, bool):
rendered = "true" if value else "false"
elif isinstance(value, (int, float)):
rendered = f"{float(value):.1f}"
else:
rendered = str(value)
lines.append(f" {key}: {rendered}")
lines.extend(["", f"[OK] {success_message}"])
console.print("\n".join(lines))
def _threshold_summary(profile: AutomationProfile) -> str:
"""Return a compact summary of key thresholds."""
return (
@@ -123,11 +285,26 @@ def _print_profile(
profile: AutomationProfile,
title: str = "Automation Profile",
fmt: str = OutputFormat.RICH.value,
*,
payload_factory: Callable[[AutomationProfile], dict[str, object]] | None = None,
command_name: str = "automation-profile show",
rich_renderer: Callable[[AutomationProfile], None] | None = None,
plain_renderer: Callable[[AutomationProfile], None] | None = None,
messages: list[dict[str, str]] | None = None,
) -> None:
"""Print profile details in the requested format."""
if fmt == OutputFormat.PLAIN.value and plain_renderer is not None:
plain_renderer(profile)
return
if fmt != OutputFormat.RICH.value:
data = _profile_spec_dict(profile)
console.print(format_output(data, fmt))
factory = payload_factory or _profile_spec_dict
data = factory(profile)
console.print(format_output(data, fmt, command=command_name, messages=messages))
return
if rich_renderer is not None:
rich_renderer(profile)
return
source = "built-in" if profile.name in BUILTIN_PROFILES else "custom"
@@ -247,9 +424,33 @@ def add_profile(
title = "Profile Updated"
else:
profile = service.create_profile(config_data)
title = "Profile Added"
title = "Profile Added" if update else "Profile Registered"
_print_profile(profile, title=title, fmt=fmt)
created_iso = _current_utc_iso()
success_message = (
"Profile updated" if update and existing else "Profile registered"
)
messages = [{"level": "ok", "text": success_message}]
_print_profile(
profile,
title=title,
fmt=fmt,
payload_factory=partial(_profile_add_dict, created_iso=created_iso),
command_name="automation-profile add",
rich_renderer=partial(
_render_profile_add_rich,
created_iso=created_iso,
heading=title,
success_message=success_message,
),
plain_renderer=partial(
_render_profile_add_plain,
created_iso=created_iso,
heading=title,
success_message=success_message,
),
messages=messages,
)
except FileNotFoundError as exc:
console.print(f"[red]Config file error:[/red] {exc}")