diff --git a/features/automation_profile_cli.feature b/features/automation_profile_cli.feature index deff7751c..91601de6b 100644 --- a/features/automation_profile_cli.feature +++ b/features/automation_profile_cli.feature @@ -89,8 +89,34 @@ Feature: Automation Profile CLI commands Scenario: List profiles with --format json output snapshot When I run automation-profile list with --format json Then the automation-profile list should succeed - And the automation-profile json output should contain "name" - And the automation-profile json output should contain "decompose_task" + And the automation-profile json output should contain "profiles" + And the automation-profile json output should contain "summary" + And the automation-profile json output should contain "built_in" + And the automation-profile json output should contain "select_tool" + + Scenario: List profiles JSON output has spec-required profiles wrapper with summary + When I run automation-profile list with --format json + Then the automation-profile list should succeed + And the automation-profile list json output has profiles wrapper with summary + + Scenario: List profiles YAML output has spec-required profiles wrapper + When I run automation-profile list with --format yaml + Then the automation-profile list should succeed + And the automation-profile yaml output should contain "profiles:" + And the automation-profile yaml output should contain "summary:" + And the automation-profile yaml output should contain "built_in:" + And the automation-profile yaml output should contain "select_tool:" + + Scenario: List profiles rich output has Auto-Apply column header + When I run automation-profile list + Then the automation-profile list should succeed + And the automation-profile output should contain "Auto-Ap" + + Scenario: List profiles rich output has Summary panel + When I run automation-profile list + Then the automation-profile list should succeed + And the automation-profile output should contain "Summary" + And the automation-profile output should contain "Built-in:" Scenario: List profiles with no matches When I run automation-profile list with namespace filter "nonexistent" diff --git a/features/steps/automation_profile_cli_steps.py b/features/steps/automation_profile_cli_steps.py index c715e4292..7197d96d5 100644 --- a/features/steps/automation_profile_cli_steps.py +++ b/features/steps/automation_profile_cli_steps.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import re import tempfile @@ -372,6 +373,12 @@ def step_run_list_format_json(context: Context) -> None: context.result = context.runner.invoke(profile_app, ["list", "--format", "json"]) +@when("I run automation-profile list with --format yaml") +def step_run_list_format_yaml(context: Context) -> None: + """Run the list command with --format yaml.""" + context.result = context.runner.invoke(profile_app, ["list", "--format", "yaml"]) + + @when('I run automation-profile show "{name}"') def step_run_show(context: Context, name: str) -> None: """Run the show command.""" @@ -586,3 +593,78 @@ def step_plan_output_should_contain(context: Context, text: str) -> None: assert context.result is not None output = _normalize_cli_output(context.result.output) assert text in output, f"Expected '{text}' in output. Got: {context.result.output}" + + +@then("the automation-profile list json output has profiles wrapper with summary") +def step_list_json_has_profiles_wrapper_with_summary(context: Context) -> None: + """Assert the list JSON output has the spec-required profiles wrapper and summary. + + Per docs/specification.md lines 16998-17017, the JSON output must be a dict + with a ``profiles`` key (list of simplified profile entries) and a ``summary`` + key with ``built_in``, ``custom``, and ``total`` counts. Each profile entry + must contain only: name, source, select_tool, sandbox, description. + """ + assert context.result is not None + assert context.result.exit_code == 0, ( + f"Expected exit code 0, got {context.result.exit_code}. " + f"Output: {context.result.output}" + ) + # Extract JSON from output (may have leading log lines) + output = context.result.output + parsed: object = None + for i, ch in enumerate(output): + if ch == "{": + try: + parsed = json.loads(output[i:]) + break + except json.JSONDecodeError: + continue + assert parsed is not None, f"No valid JSON found in output:\n{output[:500]}" + assert isinstance(parsed, dict), f"Expected dict at top level, got {type(parsed)}" + + # Validate profiles key + assert "profiles" in parsed, ( + f"Missing 'profiles' key in JSON output. Keys: {list(parsed.keys())}" + ) + profiles = parsed["profiles"] + assert isinstance(profiles, list), ( + f"Expected 'profiles' to be a list, got {type(profiles)}" + ) + assert len(profiles) >= 8, ( + f"Expected at least 8 built-in profiles, got {len(profiles)}" + ) + + # Validate each profile entry has only the spec-required fields + required_fields = {"name", "source", "select_tool", "sandbox", "description"} + for entry in profiles: + assert isinstance(entry, dict), f"Profile entry is not a dict: {entry}" + assert required_fields.issubset(entry.keys()), ( + f"Profile entry missing required fields. " + f"Expected {required_fields}, got {set(entry.keys())}" + ) + # Ensure no full profile dict fields leak through + assert "phase_transitions" not in entry, ( + "Profile entry must not contain 'phase_transitions' (full profile dict leaked)" + ) + assert "decompose_task" not in entry, ( + "Profile entry must not contain 'decompose_task' (full profile dict leaked)" + ) + + # Validate summary key + assert "summary" in parsed, ( + f"Missing 'summary' key in JSON output. Keys: {list(parsed.keys())}" + ) + summary = parsed["summary"] + assert isinstance(summary, dict), ( + f"Expected 'summary' to be a dict, got {type(summary)}" + ) + assert "built_in" in summary, f"Missing 'built_in' in summary: {summary}" + assert "custom" in summary, f"Missing 'custom' in summary: {summary}" + assert "total" in summary, f"Missing 'total' in summary: {summary}" + assert summary["built_in"] >= 8, ( + f"Expected at least 8 built-in profiles in summary, got {summary['built_in']}" + ) + assert summary["total"] == summary["built_in"] + summary["custom"], ( + f"summary.total ({summary['total']}) != built_in ({summary['built_in']}) " + f"+ custom ({summary['custom']})" + ) diff --git a/pyproject.toml b/pyproject.toml index bc29cd5f4..dd9b4e961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,7 +113,7 @@ include = ["src/cleveragents/py.typed"] line-length = 88 target-version = "py313" # Target Python 3.13 src = ["src", "tests", "benchmarks"] -extend-exclude = ["docs/reference/contracts/stubs/*.py", "scripts/*.sh"] +extend-exclude = ["docs/reference/contracts/stubs/*.py", "scripts/*.sh", "**/*.feature"] [tool.ruff.lint] select = ["E", "F", "W", "B", "UP", "I", "SIM", "RUF"] diff --git a/robot/helper_automation_profile_cli.py b/robot/helper_automation_profile_cli.py index bc9e7e9e8..b0c4ec8b3 100644 --- a/robot/helper_automation_profile_cli.py +++ b/robot/helper_automation_profile_cli.py @@ -172,13 +172,47 @@ def test_list_profiles() -> None: def test_list_json() -> None: - """Test listing profiles in JSON format.""" + """Test listing profiles in JSON format. + + Per docs/specification.md lines 16998-17017, the output must be a dict + with a ``profiles`` key (list of simplified entries) and a ``summary`` + key with built_in/custom/total counts. + """ _reset_service() result = _invoke(["list", "--format", "json"]) assert result.exit_code == 0, f"list json failed: {result.output}" parsed = _extract_json(result.output) - assert isinstance(parsed, list) - assert len(parsed) >= 8 # At least 8 built-in profiles + assert isinstance(parsed, dict), ( + f"Expected dict at top level, got {type(parsed).__name__}. " + f"Output: {result.output[:300]}" + ) + # Validate profiles wrapper + assert "profiles" in parsed, ( + f"Missing 'profiles' key. Keys: {list(parsed.keys())}" + ) + profiles = parsed["profiles"] + assert isinstance(profiles, list), ( + f"'profiles' must be a list, got {type(profiles)}" + ) + assert len(profiles) >= 8, f"Expected >= 8 built-in profiles, got {len(profiles)}" + # Validate each entry has only spec-required fields + for entry in profiles: + assert "name" in entry, f"Profile entry missing 'name': {entry}" + assert "select_tool" in entry, f"Profile entry missing 'select_tool': {entry}" + assert "sandbox" in entry, f"Profile entry missing 'sandbox': {entry}" + assert "phase_transitions" not in entry, ( + "Full profile dict leaked into list output (found 'phase_transitions')" + ) + # Validate summary + assert "summary" in parsed, ( + f"Missing 'summary' key. Keys: {list(parsed.keys())}" + ) + summary = parsed["summary"] + assert "built_in" in summary, f"Missing 'built_in' in summary: {summary}" + assert "total" in summary, f"Missing 'total' in summary: {summary}" + assert summary["built_in"] >= 8, ( + f"Expected >= 8 built-in profiles in summary, got {summary['built_in']}" + ) print("list-json-ok") diff --git a/src/cleveragents/cli/commands/automation_profile.py b/src/cleveragents/cli/commands/automation_profile.py index 10745a015..da6803375 100644 --- a/src/cleveragents/cli/commands/automation_profile.py +++ b/src/cleveragents/cli/commands/automation_profile.py @@ -381,20 +381,42 @@ def list_profiles( console.print("[yellow]No profiles found.[/yellow]") return - # Non-rich formats + # Compute summary counts + built_in_count = sum(1 for p in profiles if p.name in BUILTIN_PROFILES) + custom_count = len(profiles) - built_in_count + total_count = len(profiles) + + # Non-rich formats: spec-required structured envelope if fmt != OutputFormat.RICH.value: - data = [_profile_spec_dict(p) for p in profiles] + profile_entries = [ + { + "name": p.name, + "source": "built-in" if p.name in BUILTIN_PROFILES else "custom", + "select_tool": p.select_tool, + "sandbox": p.safety.require_sandbox, + "description": p.description, + } + for p in profiles + ] + data = { + "profiles": profile_entries, + "summary": { + "built_in": built_in_count, + "custom": custom_count, + "total": total_count, + }, + } console.print(format_output(data, fmt)) return # Rich table display - table = Table(title=f"Automation Profiles ({len(profiles)} total)") + table = Table(title="Automation Profiles") table.add_column("Name", style="cyan") table.add_column("Source", style="blue") table.add_column("Description", style="dim") table.add_column("Decompose", justify="right") table.add_column("Create Tool", justify="right") - table.add_column("Select Tool", justify="right") + table.add_column("Auto-Apply", justify="right") table.add_column("Sandbox", justify="center") for profile in profiles: @@ -412,6 +434,14 @@ def list_profiles( console.print(table) + # Summary panel + summary_text = ( + f"[bold blue]Built-in:[/bold blue] {built_in_count}\n" + f"[bold blue]Custom:[/bold blue] {custom_count}\n" + f"[bold blue]Total:[/bold blue] {total_count}" + ) + console.print(Panel(summary_text, title="Summary", expand=False)) + except CleverAgentsError as exc: console.print(f"[red]Error:[/red] {exc.message}") raise typer.Abort() from exc