fix(cli): fix automation-profile add JSON/YAML output format #6598
@@ -13,6 +13,29 @@ Feature: Automation Profile CLI commands
|
||||
Then the automation-profile add should succeed
|
||||
And the automation-profile output should contain "acme/strict"
|
||||
|
||||
@tdd_issue @tdd_issue_6345
|
||||
Scenario: Add profile JSON output uses flat schema
|
||||
Given a valid automation profile config YAML file
|
||||
When I run automation-profile add with --config pointing to the YAML file and --format json
|
||||
Then the automation-profile add should succeed
|
||||
And the automation-profile add json output matches the flat schema
|
||||
|
||||
@tdd_issue @tdd_issue_6345
|
||||
Scenario: Add profile YAML output uses flat schema
|
||||
Given a valid automation profile config YAML file
|
||||
When I run automation-profile add with --config pointing to the YAML file and --format yaml
|
||||
Then the automation-profile add should succeed
|
||||
And the automation-profile add yaml output matches the flat schema
|
||||
|
||||
@tdd_issue @tdd_issue_6345
|
||||
Scenario: Add profile plain output includes created timestamp
|
||||
Given a valid automation profile config YAML file
|
||||
When I run automation-profile add with --config pointing to the YAML file and --format plain
|
||||
Then the automation-profile add should succeed
|
||||
And the automation-profile output should contain "Profile Registered"
|
||||
And the automation-profile output should contain "Created:"
|
||||
And the automation-profile output should contain "require_checkpoints: true"
|
||||
|
||||
Scenario: Add profile with --update for existing custom profile
|
||||
Given a valid automation profile config YAML file
|
||||
And the custom profile "acme/strict" already exists
|
||||
|
||||
@@ -6,32 +6,31 @@ import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import yaml
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.commands.automation_profile import (
|
||||
app as profile_app,
|
||||
from cleveragents.application.services.automation_profile_service import (
|
||||
AutomationProfileService,
|
||||
)
|
||||
from cleveragents.cli.commands.automation_profile import app as profile_app
|
||||
from cleveragents.cli.commands.plan import app as plan_app
|
||||
from cleveragents.domain.models.core.automation_profile import AutomationProfile
|
||||
from cleveragents.domain.models.core.safety_profile import SafetyProfile
|
||||
from cleveragents.infrastructure.database.models import Base
|
||||
from cleveragents.infrastructure.database.repositories import (
|
||||
AutomationProfileRepository,
|
||||
)
|
||||
|
||||
|
||||
def _create_in_memory_profile_service():
|
||||
"""Create an AutomationProfileService backed by an in-memory SQLite DB."""
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from cleveragents.application.services.automation_profile_service import (
|
||||
AutomationProfileService,
|
||||
)
|
||||
from cleveragents.infrastructure.database.models import Base
|
||||
from cleveragents.infrastructure.database.repositories import (
|
||||
AutomationProfileRepository,
|
||||
)
|
||||
|
||||
engine = create_engine("sqlite:///:memory:", echo=False)
|
||||
Base.metadata.create_all(engine)
|
||||
factory = sessionmaker(bind=engine, expire_on_commit=False)
|
||||
@@ -262,6 +261,39 @@ def step_run_add_config(context: Context) -> None:
|
||||
)
|
||||
|
||||
|
||||
@when(
|
||||
"I run automation-profile add with --config pointing to the YAML file and --format json"
|
||||
)
|
||||
def step_run_add_config_format_json(context: Context) -> None:
|
||||
"""Run the add command with --config producing JSON output."""
|
||||
context.result = context.runner.invoke(
|
||||
profile_app,
|
||||
["add", "--config", context.yaml_path, "--format", "json"],
|
||||
)
|
||||
|
||||
|
||||
@when(
|
||||
"I run automation-profile add with --config pointing to the YAML file and --format yaml"
|
||||
)
|
||||
def step_run_add_config_format_yaml(context: Context) -> None:
|
||||
"""Run the add command with --config producing YAML output."""
|
||||
context.result = context.runner.invoke(
|
||||
profile_app,
|
||||
["add", "--config", context.yaml_path, "--format", "yaml"],
|
||||
)
|
||||
|
||||
|
||||
@when(
|
||||
"I run automation-profile add with --config pointing to the YAML file and --format plain"
|
||||
)
|
||||
def step_run_add_config_format_plain(context: Context) -> None:
|
||||
"""Run the add command with --config producing plain output."""
|
||||
context.result = context.runner.invoke(
|
||||
profile_app,
|
||||
["add", "--config", context.yaml_path, "--format", "plain"],
|
||||
)
|
||||
|
||||
|
||||
@when("I run automation-profile add with --config and --update")
|
||||
def step_run_add_config_update(context: Context) -> None:
|
||||
"""Run the add command with --config and --update."""
|
||||
@@ -434,8 +466,6 @@ def step_run_remove_yes_json(context: Context, name: str) -> None:
|
||||
@when('I invoke plan use with --automation-level "{level}"')
|
||||
def step_invoke_plan_use_automation_level(context: Context, level: str) -> None:
|
||||
"""Invoke plan use with the deprecated --automation-level flag."""
|
||||
from cleveragents.cli.commands.plan import app as plan_app
|
||||
|
||||
# Mock the lifecycle service so we don't need real infra
|
||||
mock_service = MagicMock()
|
||||
mock_plan = MagicMock()
|
||||
@@ -510,6 +540,168 @@ def step_add_should_succeed(context: Context) -> None:
|
||||
)
|
||||
|
||||
|
||||
@then("the automation-profile add json output matches the flat schema")
|
||||
def step_add_json_matches_flat_schema(context: Context) -> None:
|
||||
"""Validate JSON output uses the spec-defined flat schema."""
|
||||
assert context.result is not None
|
||||
output = context.result.output
|
||||
parsed: dict[str, object] | None = None
|
||||
for idx, ch in enumerate(output):
|
||||
if ch == "{":
|
||||
try:
|
||||
parsed = json.loads(output[idx:])
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
assert parsed is not None, f"No valid JSON found in output: {output[:200]}"
|
||||
assert isinstance(parsed, dict), f"Expected dict at top level, got {type(parsed)}"
|
||||
|
||||
assert parsed.get("command") == "automation-profile add", (
|
||||
"JSON envelope command field must be 'automation-profile add'"
|
||||
)
|
||||
|
||||
data = parsed.get("data")
|
||||
assert isinstance(data, dict), f"Envelope data must be dict, got {type(data)}"
|
||||
|
||||
created = data.get("created")
|
||||
assert isinstance(created, str), "created must be an ISO-8601 string"
|
||||
try:
|
||||
datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
except ValueError as exc: # pragma: no cover - defensive
|
||||
raise AssertionError(f"created is not ISO-8601: {created}") from exc
|
||||
|
||||
expected_thresholds = {
|
||||
"decompose_task",
|
||||
"create_tool",
|
||||
"select_tool",
|
||||
"edit_code",
|
||||
"execute_command",
|
||||
"create_file",
|
||||
"delete_content",
|
||||
"access_network",
|
||||
"install_dependency",
|
||||
"modify_config",
|
||||
"approve_plan",
|
||||
}
|
||||
thresholds = data.get("thresholds")
|
||||
assert isinstance(thresholds, dict), "thresholds must be a dict"
|
||||
assert set(thresholds.keys()) == expected_thresholds, (
|
||||
f"threshold keys mismatch: {set(thresholds.keys())}"
|
||||
)
|
||||
for key in expected_thresholds:
|
||||
assert isinstance(thresholds[key], (int, float)), (
|
||||
f"threshold {key} must be numeric"
|
||||
)
|
||||
|
||||
expected_flags = {
|
||||
"require_sandbox",
|
||||
"require_checkpoints",
|
||||
"allow_unsafe_tools",
|
||||
}
|
||||
flags = data.get("flags")
|
||||
assert isinstance(flags, dict), "flags must be a dict"
|
||||
assert set(flags.keys()) == expected_flags, (
|
||||
f"flag keys mismatch: {set(flags.keys())}"
|
||||
)
|
||||
for key in expected_flags:
|
||||
assert isinstance(flags[key], bool), f"flag {key} must be boolean"
|
||||
|
||||
forbidden_keys = {
|
||||
"phase_transitions",
|
||||
"decision_automation",
|
||||
"self_repair",
|
||||
"execution_controls",
|
||||
"schema_version",
|
||||
"guards",
|
||||
}
|
||||
for key in forbidden_keys:
|
||||
assert key not in data, f"Unexpected grouped schema field present: {key}"
|
||||
|
||||
messages = parsed.get("messages")
|
||||
assert isinstance(messages, list), "messages must be a list"
|
||||
assert messages, "messages list must not be empty"
|
||||
first_message = messages[0]
|
||||
assert isinstance(first_message, dict), "messages entries must be mappings"
|
||||
assert first_message.get("level") == "ok", "message level must be 'ok'"
|
||||
assert first_message.get("text") == "Profile registered", (
|
||||
"message text must indicate profile registration"
|
||||
)
|
||||
|
||||
|
||||
@then("the automation-profile add yaml output matches the flat schema")
|
||||
def step_add_yaml_matches_flat_schema(context: Context) -> None:
|
||||
"""Validate YAML output uses the spec-defined flat schema."""
|
||||
assert context.result is not None
|
||||
parsed = yaml.safe_load(context.result.output)
|
||||
assert isinstance(parsed, dict), f"Expected dict at top level, got {type(parsed)}"
|
||||
assert parsed.get("command") == "automation-profile add", (
|
||||
"YAML envelope command field must be 'automation-profile add'"
|
||||
)
|
||||
|
||||
data = parsed.get("data")
|
||||
assert isinstance(data, dict), "Envelope data must be a mapping"
|
||||
|
||||
created = data.get("created")
|
||||
assert isinstance(created, str), "created must be a string"
|
||||
try:
|
||||
datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
except ValueError as exc: # pragma: no cover - defensive
|
||||
raise AssertionError(f"created is not ISO-8601: {created}") from exc
|
||||
|
||||
expected_thresholds = {
|
||||
"decompose_task",
|
||||
"create_tool",
|
||||
"select_tool",
|
||||
"edit_code",
|
||||
"execute_command",
|
||||
"create_file",
|
||||
"delete_content",
|
||||
"access_network",
|
||||
"install_dependency",
|
||||
"modify_config",
|
||||
"approve_plan",
|
||||
}
|
||||
thresholds = data.get("thresholds")
|
||||
assert isinstance(thresholds, dict), "thresholds must be a mapping"
|
||||
assert set(thresholds.keys()) == expected_thresholds, (
|
||||
f"threshold keys mismatch: {set(thresholds.keys())}"
|
||||
)
|
||||
|
||||
expected_flags = {
|
||||
"require_sandbox",
|
||||
"require_checkpoints",
|
||||
"allow_unsafe_tools",
|
||||
}
|
||||
flags = data.get("flags")
|
||||
assert isinstance(flags, dict), "flags must be a mapping"
|
||||
assert set(flags.keys()) == expected_flags, (
|
||||
f"flag keys mismatch: {set(flags.keys())}"
|
||||
)
|
||||
for key in expected_flags:
|
||||
assert isinstance(flags[key], bool), f"flag {key} must be boolean"
|
||||
|
||||
forbidden_keys = {
|
||||
"phase_transitions",
|
||||
"decision_automation",
|
||||
"self_repair",
|
||||
"execution_controls",
|
||||
"schema_version",
|
||||
"guards",
|
||||
}
|
||||
for key in forbidden_keys:
|
||||
assert key not in data, f"Unexpected grouped schema field present: {key}"
|
||||
|
||||
messages = parsed.get("messages")
|
||||
assert isinstance(messages, list), "messages must be a list"
|
||||
assert messages, "messages list must not be empty"
|
||||
first_message = messages[0]
|
||||
assert isinstance(first_message, dict), "messages entries must be mappings"
|
||||
assert first_message.get("level") == "ok", "message level must be 'ok'"
|
||||
assert first_message.get("text") == "Profile registered", (
|
||||
"message text must indicate profile registration"
|
||||
)
|
||||
|
||||
|
||||
@then('the automation-profile output should contain "{text}"')
|
||||
def step_output_contains(context: Context, text: str) -> None:
|
||||
"""Assert the command output contains specific text."""
|
||||
|
||||
@@ -11,6 +11,9 @@ from __future__ import annotations
|
||||
import contextlib
|
||||
import re
|
||||
import warnings
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any
|
||||
|
||||
@@ -30,9 +33,9 @@ from cleveragents.core.exceptions import (
|
||||
NotFoundError,
|
||||
ValidationError,
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_guard import AutomationGuard
|
||||
from cleveragents.domain.models.core.automation_profile import (
|
||||
BUILTIN_PROFILES,
|
||||
AutomationGuard,
|
||||
AutomationProfile,
|
||||
)
|
||||
|
||||
@@ -110,6 +113,165 @@ def _profile_spec_dict(profile: AutomationProfile) -> dict[str, object]:
|
||||
return result
|
||||
|
||||
|
||||
def _profile_add_dict(
|
||||
profile: AutomationProfile,
|
||||
*,
|
||||
created_iso: str,
|
||||
) -> dict[str, object]:
|
||||
"""Return automation-profile add data using the spec-defined schema."""
|
||||
|
||||
thresholds: dict[str, float] = {
|
||||
"decompose_task": profile.decompose_task,
|
||||
"create_tool": profile.create_tool,
|
||||
"select_tool": profile.select_tool,
|
||||
"edit_code": profile.edit_code,
|
||||
"execute_command": profile.execute_command,
|
||||
"create_file": profile.create_file,
|
||||
"delete_content": profile.delete_content,
|
||||
"access_network": profile.access_network,
|
||||
"install_dependency": profile.install_dependency,
|
||||
"modify_config": profile.modify_config,
|
||||
"approve_plan": profile.approve_plan,
|
||||
}
|
||||
|
||||
flags: dict[str, bool] = {
|
||||
"require_sandbox": profile.safety.require_sandbox,
|
||||
"require_checkpoints": profile.safety.require_checkpoints,
|
||||
"allow_unsafe_tools": profile.safety.allow_unsafe_tools,
|
||||
}
|
||||
|
||||
return {
|
||||
"name": profile.name,
|
||||
"description": profile.description,
|
||||
"created": created_iso,
|
||||
"thresholds": thresholds,
|
||||
"flags": flags,
|
||||
}
|
||||
|
||||
|
||||
def _current_utc_iso() -> str:
|
||||
"""Return current UTC timestamp without microseconds and with Z suffix."""
|
||||
|
||||
return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _render_profile_add_rich(
|
||||
profile: AutomationProfile,
|
||||
*,
|
||||
created_iso: str,
|
||||
heading: str,
|
||||
success_message: str,
|
||||
) -> None:
|
||||
"""Render automation-profile add rich output using the spec layout."""
|
||||
|
||||
header_lines = (
|
||||
f"[cyan]Name:[/cyan] {profile.name}\n"
|
||||
f"[blue]Description:[/blue] {profile.description}\n"
|
||||
f"[green]Created:[/green] {created_iso}"
|
||||
)
|
||||
console.print(Panel.fit(header_lines, title=heading))
|
||||
|
||||
threshold_table = Table.grid(padding=(0, 1))
|
||||
threshold_table.add_column(justify="right", style="cyan", no_wrap=True)
|
||||
threshold_table.add_column(justify="left", style="white")
|
||||
|
||||
threshold_entries: list[tuple[str, object]] = [
|
||||
("decompose_task", profile.decompose_task),
|
||||
("create_tool", profile.create_tool),
|
||||
("select_tool", profile.select_tool),
|
||||
("edit_code", profile.edit_code),
|
||||
("execute_command", profile.execute_command),
|
||||
("create_file", profile.create_file),
|
||||
("delete_content", profile.delete_content),
|
||||
("access_network", profile.access_network),
|
||||
("install_dependency", profile.install_dependency),
|
||||
("modify_config", profile.modify_config),
|
||||
("approve_plan", profile.approve_plan),
|
||||
("require_sandbox", profile.safety.require_sandbox),
|
||||
("require_checkpoints", profile.safety.require_checkpoints),
|
||||
("allow_unsafe_tools", profile.safety.allow_unsafe_tools),
|
||||
]
|
||||
|
||||
for key, value in threshold_entries:
|
||||
if isinstance(value, bool):
|
||||
rendered = "true" if value else "false"
|
||||
elif isinstance(value, (int, float)):
|
||||
rendered = f"{float(value):.1f}"
|
||||
else:
|
||||
rendered = str(value)
|
||||
threshold_table.add_row(f"{key}:", rendered)
|
||||
|
||||
console.print(Panel(threshold_table, title="Confidence Thresholds"))
|
||||
|
||||
if profile.guards is not None:
|
||||
g = profile.guards
|
||||
guards_table = Table.grid(padding=(0, 1))
|
||||
guards_table.add_column(justify="right", style="cyan", no_wrap=True)
|
||||
guards_table.add_column(justify="left", style="white")
|
||||
guards_table.add_row("max_tool_calls_per_step:", str(g.max_tool_calls_per_step))
|
||||
guards_table.add_row("max_total_cost:", str(g.max_total_cost))
|
||||
guards_table.add_row("tool_allowlist:", str(g.tool_allowlist))
|
||||
guards_table.add_row("tool_denylist:", str(g.tool_denylist))
|
||||
guards_table.add_row(
|
||||
"require_approval_for_writes:", str(g.require_approval_for_writes)
|
||||
)
|
||||
guards_table.add_row(
|
||||
"require_approval_for_apply:", str(g.require_approval_for_apply)
|
||||
)
|
||||
console.print(Panel(guards_table, title="Guards"))
|
||||
|
||||
console.print(f"[green]\u2713 OK[/green] {success_message}")
|
||||
|
||||
|
||||
def _render_profile_add_plain(
|
||||
profile: AutomationProfile,
|
||||
*,
|
||||
created_iso: str,
|
||||
heading: str,
|
||||
success_message: str,
|
||||
) -> None:
|
||||
"""Render automation-profile add plain output using the spec layout."""
|
||||
|
||||
lines = [
|
||||
heading,
|
||||
f" Name: {profile.name}",
|
||||
f" Description: {profile.description}",
|
||||
f" Created: {created_iso}",
|
||||
"",
|
||||
"Confidence Thresholds",
|
||||
]
|
||||
|
||||
threshold_entries: list[tuple[str, object]] = [
|
||||
("decompose_task", profile.decompose_task),
|
||||
("create_tool", profile.create_tool),
|
||||
("select_tool", profile.select_tool),
|
||||
("edit_code", profile.edit_code),
|
||||
("execute_command", profile.execute_command),
|
||||
("create_file", profile.create_file),
|
||||
("delete_content", profile.delete_content),
|
||||
("access_network", profile.access_network),
|
||||
("install_dependency", profile.install_dependency),
|
||||
("modify_config", profile.modify_config),
|
||||
("approve_plan", profile.approve_plan),
|
||||
("require_sandbox", profile.safety.require_sandbox),
|
||||
("require_checkpoints", profile.safety.require_checkpoints),
|
||||
("allow_unsafe_tools", profile.safety.allow_unsafe_tools),
|
||||
]
|
||||
|
||||
for key, value in threshold_entries:
|
||||
if isinstance(value, bool):
|
||||
rendered = "true" if value else "false"
|
||||
elif isinstance(value, (int, float)):
|
||||
rendered = f"{float(value):.1f}"
|
||||
else:
|
||||
rendered = str(value)
|
||||
lines.append(f" {key}: {rendered}")
|
||||
|
||||
lines.extend(["", f"[OK] {success_message}"])
|
||||
|
||||
console.print("\n".join(lines))
|
||||
|
||||
|
||||
def _threshold_summary(profile: AutomationProfile) -> str:
|
||||
"""Return a compact summary of key thresholds."""
|
||||
return (
|
||||
@@ -123,11 +285,26 @@ def _print_profile(
|
||||
profile: AutomationProfile,
|
||||
title: str = "Automation Profile",
|
||||
fmt: str = OutputFormat.RICH.value,
|
||||
*,
|
||||
payload_factory: Callable[[AutomationProfile], dict[str, object]] | None = None,
|
||||
command_name: str = "automation-profile show",
|
||||
rich_renderer: Callable[[AutomationProfile], None] | None = None,
|
||||
plain_renderer: Callable[[AutomationProfile], None] | None = None,
|
||||
messages: list[dict[str, str]] | None = None,
|
||||
) -> None:
|
||||
"""Print profile details in the requested format."""
|
||||
if fmt == OutputFormat.PLAIN.value and plain_renderer is not None:
|
||||
plain_renderer(profile)
|
||||
return
|
||||
|
||||
if fmt != OutputFormat.RICH.value:
|
||||
data = _profile_spec_dict(profile)
|
||||
console.print(format_output(data, fmt))
|
||||
factory = payload_factory or _profile_spec_dict
|
||||
data = factory(profile)
|
||||
console.print(format_output(data, fmt, command=command_name, messages=messages))
|
||||
return
|
||||
|
||||
if rich_renderer is not None:
|
||||
rich_renderer(profile)
|
||||
return
|
||||
|
||||
source = "built-in" if profile.name in BUILTIN_PROFILES else "custom"
|
||||
@@ -247,9 +424,33 @@ def add_profile(
|
||||
title = "Profile Updated"
|
||||
else:
|
||||
profile = service.create_profile(config_data)
|
||||
title = "Profile Added"
|
||||
title = "Profile Added" if update else "Profile Registered"
|
||||
|
||||
_print_profile(profile, title=title, fmt=fmt)
|
||||
created_iso = _current_utc_iso()
|
||||
success_message = (
|
||||
"Profile updated" if update and existing else "Profile registered"
|
||||
)
|
||||
messages = [{"level": "ok", "text": success_message}]
|
||||
_print_profile(
|
||||
profile,
|
||||
title=title,
|
||||
fmt=fmt,
|
||||
payload_factory=partial(_profile_add_dict, created_iso=created_iso),
|
||||
command_name="automation-profile add",
|
||||
rich_renderer=partial(
|
||||
_render_profile_add_rich,
|
||||
created_iso=created_iso,
|
||||
heading=title,
|
||||
success_message=success_message,
|
||||
),
|
||||
plain_renderer=partial(
|
||||
_render_profile_add_plain,
|
||||
created_iso=created_iso,
|
||||
heading=title,
|
||||
success_message=success_message,
|
||||
),
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
except FileNotFoundError as exc:
|
||||
console.print(f"[red]Config file error:[/red] {exc}")
|
||||
|
||||
Reference in New Issue
Block a user