diff --git a/features/cli_json_envelope.feature b/features/cli_json_envelope.feature new file mode 100644 index 000000000..f1b56ece5 --- /dev/null +++ b/features/cli_json_envelope.feature @@ -0,0 +1,85 @@ +Feature: CLI JSON/YAML output envelope structure + As a programmatic consumer of the CleverAgents CLI + I want all --format json and --format yaml outputs to include the spec-required envelope + So that I can reliably parse command, status, exit_code, data, timing, and messages fields + + Background: + Given a CLI output format test runner + And a mocked lifecycle service for format tests + + # ── Envelope field presence ────────────────────────────────────────────── + + Scenario: JSON output includes all required envelope fields + Given there are actions for format testing + When I run action list with --format json + Then the JSON envelope should contain field "command" + And the JSON envelope should contain field "status" + And the JSON envelope should contain field "exit_code" + And the JSON envelope should contain field "data" + And the JSON envelope should contain field "timing" + And the JSON envelope should contain field "messages" + + Scenario: YAML output includes all required envelope fields + Given there are actions for format testing + When I run action list with --format yaml + Then the YAML envelope should contain field "command" + And the YAML envelope should contain field "status" + And the YAML envelope should contain field "exit_code" + And the YAML envelope should contain field "data" + And the YAML envelope should contain field "timing" + And the YAML envelope should contain field "messages" + + # ── Envelope field values ──────────────────────────────────────────────── + + Scenario: JSON envelope status is "ok" for successful commands + Given there are actions for format testing + When I run action list with --format json + Then the JSON envelope status should be "ok" + + Scenario: JSON envelope exit_code is 0 for successful commands + Given there are actions for format testing + When I run action list with --format json + Then the JSON envelope exit_code should be 0 + + Scenario: JSON envelope timing contains duration_ms + Given there are actions for format testing + When I run action list with --format json + Then the JSON envelope timing should contain "duration_ms" + + Scenario: JSON envelope messages is a non-empty list + Given there are actions for format testing + When I run action list with --format json + Then the JSON envelope messages should be a non-empty list + + # ── Data field contains actual payload ────────────────────────────────── + + Scenario: JSON envelope data field contains actor list payload + Given there are actions for format testing + When I run action list with --format json + Then the JSON envelope data should contain actor records + + Scenario: YAML envelope data field contains actor list payload + Given there are actions for format testing + When I run action list with --format yaml + Then the YAML envelope data should contain actor records + + # ── format_output() direct call envelope ──────────────────────────────── + + Scenario: Direct format_output call with json wraps data in envelope + When I call format_output with a dict and format json + Then the format result should be valid JSON dict + And the JSON result data field should contain original keys + + Scenario: Direct format_output call with yaml wraps data in envelope + When I call format_output with a dict and format yaml + Then the format result should be valid YAML dict + And the YAML result data field should contain original keys + + Scenario: Direct format_output call with command name sets envelope command + When I call format_output with command name "agents actor list" and format json + Then the JSON envelope command should be "agents actor list" + + Scenario: Plain format does not wrap in envelope + When I call format_output with a dict and format plain + Then the format result should contain plain key-value pairs + And the plain result should not contain envelope keys diff --git a/features/steps/actor_add_rich_output_steps.py b/features/steps/actor_add_rich_output_steps.py index 0cebb76e0..8965e59b8 100644 --- a/features/steps/actor_add_rich_output_steps.py +++ b/features/steps/actor_add_rich_output_steps.py @@ -14,6 +14,15 @@ from typer.testing import CliRunner from cleveragents.cli.commands.actor import app as actor_app from cleveragents.domain.models.core.actor import Actor +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + def _make_actor( *, @@ -263,7 +272,8 @@ def step_impl(context: Any) -> None: f"exit_code={context.result.exit_code}, output={context.result.output}" ) output = context.result.output.strip() - data = json.loads(output) + parsed = json.loads(output) + data = _unwrap_envelope(parsed) assert isinstance(data, dict), f"Expected JSON object, got: {type(data)}" assert "name" in data, f"Expected 'name' field in JSON output: {data}" assert "Config" not in output, ( diff --git a/features/steps/actor_cli_yaml_steps.py b/features/steps/actor_cli_yaml_steps.py index 37c7e42a0..4a0006a5d 100644 --- a/features/steps/actor_cli_yaml_steps.py +++ b/features/steps/actor_cli_yaml_steps.py @@ -12,6 +12,15 @@ from behave import then, when from cleveragents.cli.commands.actor import app as actor_app from cleveragents.domain.models.core.actor import Actor +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + def _make_actor( *, @@ -92,7 +101,8 @@ def step_list_plain(context: Any) -> None: @then("the actor list output should be valid JSON") def step_list_json_valid(context: Any) -> None: assert context.result.exit_code == 0 - data = json.loads(context.result.output.strip()) + parsed = json.loads(context.result.output.strip()) + data = _unwrap_envelope(parsed) assert isinstance(data, list) assert len(data) == 2 assert data[0]["name"] == "local/first" @@ -164,7 +174,8 @@ def step_show_plain(context: Any) -> None: @then("the actor show output should be valid JSON with actor fields") def step_show_json_valid(context: Any) -> None: assert context.result.exit_code == 0 - data = json.loads(context.result.output.strip()) + parsed = json.loads(context.result.output.strip()) + data = _unwrap_envelope(parsed) assert isinstance(data, dict) assert data["name"] == "local/show-json" assert data["provider"] == "openai" @@ -277,7 +288,8 @@ def step_add_format_yaml(context: Any) -> None: @then("the actor add output should contain JSON data") def step_add_json_valid(context: Any) -> None: assert context.result.exit_code == 0 - data = json.loads(context.result.output.strip()) + parsed = json.loads(context.result.output.strip()) + data = _unwrap_envelope(parsed) assert isinstance(data, dict) assert "name" in data @@ -352,6 +364,7 @@ def step_update_format_json(context: Any) -> None: @then("the actor update output should contain JSON data") def step_update_json_valid(context: Any) -> None: assert context.result.exit_code == 0 - data = json.loads(context.result.output.strip()) + parsed = json.loads(context.result.output.strip()) + data = _unwrap_envelope(parsed) assert isinstance(data, dict) assert data["name"] == "local/update-json" diff --git a/features/steps/actor_context_cmds_steps.py b/features/steps/actor_context_cmds_steps.py index abf94c9ab..cd831971a 100644 --- a/features/steps/actor_context_cmds_steps.py +++ b/features/steps/actor_context_cmds_steps.py @@ -7,6 +7,7 @@ import json import shutil import tempfile from pathlib import Path +from typing import Any from behave import given, then, when from typer.testing import CliRunner @@ -14,6 +15,16 @@ from typer.testing import CliRunner from cleveragents.cli.commands.actor_context import app as actor_context_app from cleveragents.reactive.context_manager import ContextManager +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + + # --------------------------------------------------------------------------- # Background # --------------------------------------------------------------------------- @@ -406,7 +417,8 @@ def step_context_exists_check(context, name): @then('the output should contain valid JSON with key "{key}"') def step_output_json_key(context, key): output = context.result.output - data = json.loads(output) + parsed = json.loads(output) + data = _unwrap_envelope(parsed) assert key in data, f"Key '{key}' not found in JSON output: {data.keys()}" diff --git a/features/steps/cli_json_envelope_steps.py b/features/steps/cli_json_envelope_steps.py new file mode 100644 index 000000000..90f785dd5 --- /dev/null +++ b/features/steps/cli_json_envelope_steps.py @@ -0,0 +1,167 @@ +"""Step definitions for CLI JSON/YAML envelope structure tests.""" + +from __future__ import annotations + +import json +from contextlib import redirect_stdout +from io import StringIO + +import yaml +from behave import then, when +from behave.runner import Context + +from cleveragents.cli.formatting import format_output + +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _capture_format_output(data, fmt, **kwargs): + """Call format_output capturing stdout (machine-readable formats write there).""" + buf = StringIO() + with redirect_stdout(buf): + result = format_output(data, fmt, **kwargs) + return result or buf.getvalue().rstrip("\n") + + +def _parse_json_envelope(output: str) -> dict: + parsed = json.loads(output.strip()) + assert isinstance(parsed, dict), f"Expected dict envelope, got {type(parsed)}" + return parsed + + +def _parse_yaml_envelope(output: str) -> dict: + parsed = yaml.safe_load(output.strip()) + assert isinstance(parsed, dict), f"Expected dict envelope, got {type(parsed)}" + return parsed + + +# ── Envelope field presence ────────────────────────────────────────────── + + +@then('the JSON envelope should contain field "{field}"') +def step_json_envelope_has_field(context: Context, field: str) -> None: + envelope = _parse_json_envelope(context.result.output) + assert field in envelope, ( + f"Envelope field '{field}' missing. Keys present: {list(envelope.keys())}" + ) + + +@then('the YAML envelope should contain field "{field}"') +def step_yaml_envelope_has_field(context: Context, field: str) -> None: + envelope = _parse_yaml_envelope(context.result.output) + assert field in envelope, ( + f"Envelope field '{field}' missing. Keys present: {list(envelope.keys())}" + ) + + +# ── Envelope field values ──────────────────────────────────────────────── + + +@then('the JSON envelope status should be "{expected_status}"') +def step_json_envelope_status(context: Context, expected_status: str) -> None: + envelope = _parse_json_envelope(context.result.output) + assert envelope["status"] == expected_status, ( + f"Expected status '{expected_status}', got '{envelope['status']}'" + ) + + +@then("the JSON envelope exit_code should be {expected_code:d}") +def step_json_envelope_exit_code(context: Context, expected_code: int) -> None: + envelope = _parse_json_envelope(context.result.output) + assert envelope["exit_code"] == expected_code, ( + f"Expected exit_code {expected_code}, got {envelope['exit_code']}" + ) + + +@then('the JSON envelope timing should contain "{key}"') +def step_json_envelope_timing_key(context: Context, key: str) -> None: + envelope = _parse_json_envelope(context.result.output) + timing = envelope.get("timing") + assert isinstance(timing, dict), f"Expected timing dict, got {type(timing)}" + assert key in timing, f"Key '{key}' missing from timing: {list(timing.keys())}" + assert isinstance(timing[key], (int, float)), ( + f"timing.{key} should be numeric, got {type(timing[key])}" + ) + + +@then("the JSON envelope messages should be a non-empty list") +def step_json_envelope_messages_list(context: Context) -> None: + envelope = _parse_json_envelope(context.result.output) + messages = envelope.get("messages") + assert isinstance(messages, list), f"Expected messages list, got {type(messages)}" + assert len(messages) > 0, "messages list should not be empty" + # Each message should have level and text + for msg in messages: + assert "level" in msg, f"Message missing 'level': {msg}" + assert "text" in msg, f"Message missing 'text': {msg}" + + +# ── Data field contains actual payload ────────────────────────────────── + + +@then("the JSON envelope data should contain actor records") +def step_json_envelope_data_actors(context: Context) -> None: + envelope = _parse_json_envelope(context.result.output) + data = envelope["data"] + assert isinstance(data, list), f"Expected data list, got {type(data)}" + assert len(data) > 0, "data list should not be empty" + # Each actor record should have namespaced_name + assert "namespaced_name" in data[0], ( + f"Actor record missing 'namespaced_name': {list(data[0].keys())}" + ) + + +@then("the YAML envelope data should contain actor records") +def step_yaml_envelope_data_actors(context: Context) -> None: + envelope = _parse_yaml_envelope(context.result.output) + data = envelope["data"] + assert isinstance(data, list), f"Expected data list, got {type(data)}" + assert len(data) > 0, "data list should not be empty" + assert "namespaced_name" in data[0], ( + f"Actor record missing 'namespaced_name': {list(data[0].keys())}" + ) + + +# ── format_output() direct call envelope ──────────────────────────────── + + +@then("the JSON result data field should contain original keys") +def step_json_result_data_original_keys(context: Context) -> None: + parsed = json.loads(context.format_result) + data = parsed["data"] + assert isinstance(data, dict), f"Expected data dict, got {type(data)}" + # The original dict had "name" and "count" keys + assert "name" in data, f"'name' missing from data: {list(data.keys())}" + assert "count" in data, f"'count' missing from data: {list(data.keys())}" + + +@then("the YAML result data field should contain original keys") +def step_yaml_result_data_original_keys(context: Context) -> None: + parsed = yaml.safe_load(context.format_result) + data = parsed["data"] + assert isinstance(data, dict), f"Expected data dict, got {type(data)}" + assert "name" in data, f"'name' missing from data: {list(data.keys())}" + assert "count" in data, f"'count' missing from data: {list(data.keys())}" + + +@when('I call format_output with command name "{command_name}" and format json') +def step_call_format_output_with_command(context: Context, command_name: str) -> None: + data = {"name": "test", "count": 42} + context.format_result = _capture_format_output(data, "json", command=command_name) + + +@then('the JSON envelope command should be "{expected_command}"') +def step_json_envelope_command(context: Context, expected_command: str) -> None: + parsed = json.loads(context.format_result) + assert parsed["command"] == expected_command, ( + f"Expected command '{expected_command}', got '{parsed['command']}'" + ) + + +@then("the plain result should not contain envelope keys") +def step_plain_no_envelope(context: Context) -> None: + result = context.format_result + # Plain format should not have JSON envelope structure + assert '"command"' not in result, "Plain output should not contain JSON envelope" + assert '"status"' not in result, "Plain output should not contain JSON envelope" + assert '"exit_code"' not in result, "Plain output should not contain JSON envelope" diff --git a/features/steps/cli_output_formats_steps.py b/features/steps/cli_output_formats_steps.py index 2565a1413..c6bcc3d2b 100644 --- a/features/steps/cli_output_formats_steps.py +++ b/features/steps/cli_output_formats_steps.py @@ -4,6 +4,7 @@ from __future__ import annotations import json from datetime import datetime +from typing import cast from unittest.mock import MagicMock, patch import yaml @@ -201,16 +202,34 @@ def step_plan_status_plain(context: Context) -> None: ) +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: dict | list) -> dict | list: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return cast("dict | list", parsed["data"]) + return parsed + + @when("I save the output keys as json_keys") def step_save_json_keys(context: Context) -> None: - data = json.loads(context.result.output.strip()) - context.saved_keys["json_keys"] = set(data.keys()) + parsed = json.loads(context.result.output.strip()) + data = _unwrap_envelope(parsed) + if isinstance(data, list): + context.saved_keys["json_keys"] = set(data[0].keys()) if data else set() + else: + context.saved_keys["json_keys"] = set(data.keys()) @when("I save the output keys as yaml_keys") def step_save_yaml_keys(context: Context) -> None: - data = yaml.safe_load(context.result.output.strip()) - context.saved_keys["yaml_keys"] = set(data.keys()) + parsed = yaml.safe_load(context.result.output.strip()) + data = _unwrap_envelope(parsed) + if isinstance(data, list): + context.saved_keys["yaml_keys"] = set(data[0].keys()) if data else set() + else: + context.saved_keys["yaml_keys"] = set(data.keys()) # ------- Then steps ------- @@ -223,6 +242,10 @@ def step_output_valid_json(context: Context) -> None: ) parsed = json.loads(context.result.output.strip()) assert parsed is not None + # Verify spec-required envelope fields are present + assert isinstance(parsed, dict), f"Expected envelope dict, got {type(parsed)}" + for field in ("command", "status", "exit_code", "data", "timing", "messages"): + assert field in parsed, f"Envelope field '{field}' missing from JSON output" @then("the output should be valid YAML") @@ -232,26 +255,36 @@ def step_output_valid_yaml(context: Context) -> None: ) parsed = yaml.safe_load(context.result.output.strip()) assert parsed is not None + # Verify spec-required envelope fields are present + assert isinstance(parsed, dict), f"Expected envelope dict, got {type(parsed)}" + for field in ("command", "status", "exit_code", "data", "timing", "messages"): + assert field in parsed, f"Envelope field '{field}' missing from YAML output" @then('the JSON should contain key "{key}"') def step_json_contains_key(context: Context, key: str) -> None: output = context.result.output.strip() parsed = json.loads(output) - if isinstance(parsed, list): - assert key in parsed[0], f"Key '{key}' not in {list(parsed[0].keys())}" + # Unwrap envelope: look inside "data" field + data = _unwrap_envelope(parsed) + if isinstance(data, list): + assert data, f"data array is empty, cannot check key '{key}'" + assert key in data[0], f"Key '{key}' not in {list(data[0].keys())}" else: - assert key in parsed, f"Key '{key}' not in {list(parsed.keys())}" + assert key in data, f"Key '{key}' not in {list(data.keys())}" @then('the YAML should contain key "{key}"') def step_yaml_contains_key(context: Context, key: str) -> None: output = context.result.output.strip() parsed = yaml.safe_load(output) - if isinstance(parsed, list): - assert key in parsed[0], f"Key '{key}' not in {list(parsed[0].keys())}" + # Unwrap envelope: look inside "data" field + data = _unwrap_envelope(parsed) + if isinstance(data, list): + assert data, f"data array is empty, cannot check key '{key}'" + assert key in data[0], f"Key '{key}' not in {list(data[0].keys())}" else: - assert key in parsed, f"Key '{key}' not in {list(parsed.keys())}" + assert key in data, f"Key '{key}' not in {list(data.keys())}" @then('the format output should contain "{text}"') @@ -436,12 +469,22 @@ def step_call_serialize_with_enum(context: Context) -> None: def step_format_result_json_dict(context: Context) -> None: parsed = json.loads(context.format_result) assert isinstance(parsed, dict) + # Verify spec-required envelope fields are present + for field in ("command", "status", "exit_code", "data", "timing", "messages"): + assert field in parsed, f"Envelope field '{field}' missing from JSON result" + # Verify data field contains the original dict + assert isinstance(parsed["data"], dict) @then("the format result should be valid YAML dict") def step_format_result_yaml_dict(context: Context) -> None: parsed = yaml.safe_load(context.format_result) assert isinstance(parsed, dict) + # Verify spec-required envelope fields are present + for field in ("command", "status", "exit_code", "data", "timing", "messages"): + assert field in parsed, f"Envelope field '{field}' missing from YAML result" + # Verify data field contains the original dict + assert isinstance(parsed["data"], dict) @then("the format result should contain plain key-value pairs") diff --git a/features/steps/config_cli_coverage_boost_steps.py b/features/steps/config_cli_coverage_boost_steps.py index c9567d678..ec2a65d1c 100644 --- a/features/steps/config_cli_coverage_boost_steps.py +++ b/features/steps/config_cli_coverage_boost_steps.py @@ -50,6 +50,15 @@ from cleveragents.cli.commands.config import app as config_app _runner = CliRunner() +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + # --------------------------------------------------------------------------- # Helpers — isolated temp directory @@ -275,7 +284,8 @@ def step_cfg_boost_get_verbose_path(context: Context, key: str, fmt: str) -> Non @then("the cfg-boost JSON resolution chain entries should have string values") def step_cfg_boost_json_chain_strings(context: Context) -> None: - data = json.loads(context.cfg_boost_result.output) + parsed = json.loads(context.cfg_boost_result.output) + data = _unwrap_envelope(parsed) chain = data.get("resolution_chain", []) assert len(chain) > 0, "Expected non-empty resolution chain" for entry in chain: diff --git a/features/steps/config_cli_safety_net_coverage_steps.py b/features/steps/config_cli_safety_net_coverage_steps.py index cf3f4ff80..9bdc40816 100644 --- a/features/steps/config_cli_safety_net_coverage_steps.py +++ b/features/steps/config_cli_safety_net_coverage_steps.py @@ -42,6 +42,15 @@ from cleveragents.cli.commands.config import app as config_app _runner = CliRunner() +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + # --------------------------------------------------------------------------- # Helpers - isolated temp directory for config file operations @@ -424,7 +433,7 @@ def step_sn_set_valid_json(context: Context) -> None: assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) assert isinstance(parsed, dict), f"Expected dict, got {type(parsed)}" - context._sn_set_json = parsed + context._sn_set_json = _unwrap_envelope(parsed) @then('the safety-net set JSON field "value" should be boolean true') @@ -530,10 +539,11 @@ def step_sn_get_json_type(context: Context) -> None: result = context._sn_result assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) - assert "type" in parsed, f"'type' not in JSON keys: {list(parsed.keys())}" - assert "key" in parsed, f"'key' not in JSON keys: {list(parsed.keys())}" - assert "resolution_chain" in parsed, ( - f"'resolution_chain' not in JSON: {list(parsed.keys())}" + data = _unwrap_envelope(parsed) + assert "type" in data, f"'type' not in JSON keys: {list(data.keys())}" + assert "key" in data, f"'key' not in JSON keys: {list(data.keys())}" + assert "resolution_chain" in data, ( + f"'resolution_chain' not in JSON: {list(data.keys())}" ) @@ -633,8 +643,9 @@ def step_sn_list_masked(context: Context) -> None: result = context._sn_result assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) - api_entries = [e for e in parsed if e.get("key") == "api_key"] - assert len(api_entries) > 0, f"No api_key entry found in: {parsed}" + data = _unwrap_envelope(parsed) + api_entries = [e for e in data if e.get("key") == "api_key"] + assert len(api_entries) > 0, f"No api_key entry found in: {data}" assert api_entries[0]["value"] == "****", ( f"Expected masked ****, got {api_entries[0]['value']!r}" ) @@ -645,8 +656,9 @@ def step_sn_list_unmasked(context: Context) -> None: result = context._sn_result assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) - api_entries = [e for e in parsed if e.get("key") == "api_key"] - assert len(api_entries) > 0, f"No api_key entry found in: {parsed}" + data = _unwrap_envelope(parsed) + api_entries = [e for e in data if e.get("key") == "api_key"] + assert len(api_entries) > 0, f"No api_key entry found in: {data}" expected = context._sn_actual_secret assert api_entries[0]["value"] == expected, ( f"Expected '{expected}', got {api_entries[0]['value']!r}" @@ -658,7 +670,8 @@ def step_sn_list_masked_by_key(context: Context, key: str) -> None: result = context._sn_result assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) - entries = [e for e in parsed if e.get("key") == key] + data = _unwrap_envelope(parsed) + entries = [e for e in data if e.get("key") == key] assert len(entries) > 0, f"No '{key}' entry found in list output" assert entries[0]["value"] == "****", ( f"Expected masked ****, got {entries[0]['value']!r}" @@ -670,7 +683,8 @@ def step_sn_list_value_by_key(context: Context, value: str, key: str) -> None: result = context._sn_result assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) - entries = [e for e in parsed if e.get("key") == key] + data = _unwrap_envelope(parsed) + entries = [e for e in data if e.get("key") == key] assert len(entries) > 0, f"No '{key}' entry found in list output" assert entries[0]["value"] == value, ( f"Expected '{value}', got {entries[0]['value']!r}" @@ -687,10 +701,9 @@ def step_sn_list_modified(context: Context) -> None: result = context._sn_result assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) - modified_entries = [e for e in parsed if e.get("modified") is True] - assert len(modified_entries) > 0, ( - f"No entries with modified=True found in: {parsed}" - ) + data = _unwrap_envelope(parsed) + modified_entries = [e for e in data if e.get("modified") is True] + assert len(modified_entries) > 0, f"No entries with modified=True found in: {data}" # =================================================================== diff --git a/features/steps/config_cli_uncovered_branches_steps.py b/features/steps/config_cli_uncovered_branches_steps.py index 519c199bd..4dcf4220f 100644 --- a/features/steps/config_cli_uncovered_branches_steps.py +++ b/features/steps/config_cli_uncovered_branches_steps.py @@ -45,6 +45,15 @@ from cleveragents.cli.commands.config import ( _runner = CliRunner() +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + # --------------------------------------------------------------------------- # Helpers - isolated temp dir patching @@ -271,7 +280,7 @@ def step_set_result_json(context: Context) -> None: assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) assert isinstance(parsed, dict), f"Expected dict, got {type(parsed)}" - context._cfg_branch_set_json = parsed + context._cfg_branch_set_json = _unwrap_envelope(parsed) @then('the config cli branch set JSON should contain key "{key}"') @@ -390,7 +399,7 @@ def step_get_result_json(context: Context) -> None: assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) assert isinstance(parsed, dict), f"Expected dict, got {type(parsed)}" - context._cfg_branch_get_json = parsed + context._cfg_branch_get_json = _unwrap_envelope(parsed) @then("the config cli branch get JSON value should be a string not a Path") @@ -500,8 +509,9 @@ def step_list_result_json(context: Context) -> None: result = context._cfg_branch_result assert result.exit_code == 0, f"Exit {result.exit_code}: {result.output}" parsed = json.loads(result.output.strip()) - assert isinstance(parsed, list), f"Expected list, got {type(parsed)}" - context._cfg_branch_list_json = parsed + data = _unwrap_envelope(parsed) + assert isinstance(data, list), f"Expected list, got {type(data)}" + context._cfg_branch_list_json = data @then("the config cli branch list JSON should mask api-key values") diff --git a/features/steps/plan_explain_cli_coverage_steps.py b/features/steps/plan_explain_cli_coverage_steps.py index e9d507460..30392c209 100644 --- a/features/steps/plan_explain_cli_coverage_steps.py +++ b/features/steps/plan_explain_cli_coverage_steps.py @@ -8,6 +8,7 @@ using the Typer CliRunner with mocked services. from __future__ import annotations import json +from typing import Any from unittest.mock import MagicMock, patch from behave import given, then, when @@ -33,6 +34,16 @@ from cleveragents.domain.models.core.decision import ( runner = CliRunner() +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + + _PATCH_CONTAINER = "cleveragents.application.container.get_container" _PATCH_LIFECYCLE = "cleveragents.cli.commands.plan._get_lifecycle_service" @@ -785,13 +796,15 @@ def step_pec_output_not_contains(context: Context, text: str) -> None: @then("pec the output should be valid json") def step_pec_output_valid_json(context: Context) -> None: parsed = json.loads(context.pec_result.output.strip()) - assert isinstance(parsed, dict), "Expected JSON object" + data = _unwrap_envelope(parsed) + assert isinstance(data, dict), "Expected JSON object" @then("pec the output should be valid json list") def step_pec_output_valid_json_list(context: Context) -> None: parsed = json.loads(context.pec_result.output.strip()) - assert isinstance(parsed, list), "Expected JSON array" + data = _unwrap_envelope(parsed) + assert isinstance(data, list), "Expected JSON array" @then("pec the tree should exclude the superseded grandchild") diff --git a/features/steps/plan_use_env_priority_steps.py b/features/steps/plan_use_env_priority_steps.py index ab4f55d00..155ec4e30 100644 --- a/features/steps/plan_use_env_priority_steps.py +++ b/features/steps/plan_use_env_priority_steps.py @@ -5,6 +5,7 @@ from __future__ import annotations import json from datetime import datetime from enum import StrEnum +from typing import Any from unittest.mock import MagicMock, patch from behave import given, then, when @@ -26,6 +27,15 @@ from cleveragents.domain.models.core.plan import ( _PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S" +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + def _make_plan() -> Plan: """Create a Plan instance for env priority tests.""" @@ -232,7 +242,8 @@ def step_env_priority_output_contains(context: Context, text: str) -> None: @then('the plan env priority json output should include "{key}" as "{value}"') def step_env_priority_json_output(context: Context, key: str, value: str) -> None: """Assert the JSON output includes the expected key/value.""" - data = json.loads(context.result.output) + parsed = json.loads(context.result.output) + data = _unwrap_envelope(parsed) assert key in data, ( f"Expected JSON to contain key {key!r}. Keys: {list(data.keys())}" ) diff --git a/features/steps/session_cli_steps.py b/features/steps/session_cli_steps.py index 923549d21..c76746c17 100644 --- a/features/steps/session_cli_steps.py +++ b/features/steps/session_cli_steps.py @@ -6,6 +6,7 @@ import json import os import tempfile from datetime import datetime +from typing import Any from unittest.mock import MagicMock from behave import given, then, when @@ -27,6 +28,15 @@ from cleveragents.domain.models.core.session import ( _SESSION_ID = str(ULID()) _SESSION_ID_2 = str(ULID()) +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + def _make_session( *, @@ -492,7 +502,8 @@ def step_output_valid_json(context: Context) -> None: @then('the session CLI JSON should contain "{key}"') def step_json_contains_key(context: Context, key: str) -> None: - data = json.loads(context.result.output) + parsed = json.loads(context.result.output) + data = _unwrap_envelope(parsed) assert key in data, f"Key '{key}' not found in JSON: {data}" diff --git a/features/steps/session_list_error_steps.py b/features/steps/session_list_error_steps.py index d1eba41d4..402f805ed 100644 --- a/features/steps/session_list_error_steps.py +++ b/features/steps/session_list_error_steps.py @@ -28,6 +28,7 @@ import logging import os import shutil import tempfile +from typing import Any import yaml from behave import given, then, when @@ -51,6 +52,15 @@ from cleveragents.infrastructure.database.repositories import ( runner = CliRunner() +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + # --------------------------------------------------------------------------- # Helpers @@ -230,9 +240,10 @@ def step_output_json_key(context: Context, key: str) -> None: result = context.sle_result assert result is not None, "No command was invoked" try: - data = json.loads(result.output) + parsed = json.loads(result.output) except json.JSONDecodeError as exc: raise AssertionError(f"Output is not valid JSON:\n{result.output}") from exc + data = _unwrap_envelope(parsed) assert isinstance(data, dict), f"Expected JSON object, got {type(data)}: {data}" assert key in data, f"Key '{key}' not in JSON: {data}" assert isinstance(data[key], list), ( diff --git a/features/steps/skill_cli_coverage_boost_steps.py b/features/steps/skill_cli_coverage_boost_steps.py index 9c6b2dfcb..15e7fd1c0 100644 --- a/features/steps/skill_cli_coverage_boost_steps.py +++ b/features/steps/skill_cli_coverage_boost_steps.py @@ -33,6 +33,16 @@ from cleveragents.domain.models.core.skill import ( SkillAgentSource, ) +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + + # ── helpers ───────────────────────────────────────────────── @@ -252,14 +262,20 @@ def step_boost_exit_code_0(context: Context) -> None: @then("boost- the JSON output should be valid") def step_boost_json_valid(context: Context) -> None: - """Assert the CLI output parses as valid JSON.""" + """Assert the CLI output parses as valid JSON. + + Stores the unwrapped ``data`` field in ``context.boost_parsed_json`` so + that downstream key-check steps work against the actual payload rather + than the spec envelope wrapper. + """ assert context.boost_result is not None try: - context.boost_parsed_json = json.loads(context.boost_result.output) + parsed = json.loads(context.boost_result.output) except json.JSONDecodeError as e: raise AssertionError( f"Output is not valid JSON: {e}\nOutput: {context.boost_result.output}" ) from e + context.boost_parsed_json = _unwrap_envelope(parsed) @then('boost- the JSON tools list should contain an entry with source "{source}"') diff --git a/features/steps/skill_cli_coverage_r2_steps.py b/features/steps/skill_cli_coverage_r2_steps.py index aa72cf370..03f0c21cf 100644 --- a/features/steps/skill_cli_coverage_r2_steps.py +++ b/features/steps/skill_cli_coverage_r2_steps.py @@ -388,16 +388,32 @@ def step_r2_exit_code_not_0(context: Context) -> None: ) +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_r2_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + + @then("r2skill- the output should be valid JSON") def step_r2_output_valid_json(context: Context) -> None: - """Assert the CLI output parses as valid JSON.""" + """Assert the CLI output parses as valid JSON. + + Stores the unwrapped ``data`` field in ``context.r2_parsed_json`` so + that downstream key-check steps work against the actual payload rather + than the spec envelope wrapper. + """ assert context.r2_result is not None try: - context.r2_parsed_json = json.loads(context.r2_result.output) + parsed = json.loads(context.r2_result.output) except json.JSONDecodeError as e: raise AssertionError( f"Output is not valid JSON: {e}\nOutput: {context.r2_result.output}" ) from e + context.r2_parsed_json = _unwrap_r2_envelope(parsed) @then('r2skill- the JSON output should not contain key "{key}"') diff --git a/features/steps/skill_cli_coverage_r3_steps.py b/features/steps/skill_cli_coverage_r3_steps.py index 0b7e1ed0d..449983310 100644 --- a/features/steps/skill_cli_coverage_r3_steps.py +++ b/features/steps/skill_cli_coverage_r3_steps.py @@ -520,16 +520,32 @@ def step_r3_output_not_contains(context: Context, text: str) -> None: ) +_ENVELOPE_KEYS_R3 = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_r3_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS_R3.issubset(parsed.keys()): + return parsed["data"] + return parsed + + @then("r3skill- the output should be valid JSON") def step_r3_output_valid_json(context: Context) -> None: - """Assert the CLI output parses as valid JSON.""" + """Assert the CLI output parses as valid JSON. + + Stores the unwrapped ``data`` field in ``context.r3_parsed_json`` so + that downstream key-check steps work against the actual payload rather + than the spec envelope wrapper. + """ assert context.r3_result is not None try: - context.r3_parsed_json = json.loads(context.r3_result.output) + parsed = json.loads(context.r3_result.output) except json.JSONDecodeError as e: raise AssertionError( f"Output is not valid JSON: {e}\nOutput: {context.r3_result.output}" ) from e + context.r3_parsed_json = _unwrap_r3_envelope(parsed) @then('r3skill- the JSON output should have key "{key}"') diff --git a/src/cleveragents/cli/formatting.py b/src/cleveragents/cli/formatting.py index dbba448cb..cd0be666a 100644 --- a/src/cleveragents/cli/formatting.py +++ b/src/cleveragents/cli/formatting.py @@ -14,6 +14,7 @@ Based on v3_spec.md implementation plan Stage A4b / G5b.render. from __future__ import annotations import json +import time from datetime import datetime from enum import Enum, StrEnum from io import StringIO @@ -159,9 +160,88 @@ def _redact_data( return redact_dict(data) +_VALID_STATUSES: frozenset[str] = frozenset({"ok", "warn", "error"}) + + +def _build_envelope( + data: dict[str, Any] | list[dict[str, Any]], + command: str, + status: str, + exit_code: int, + duration_ms: int, + messages: list[dict[str, Any]] | None, +) -> dict[str, Any]: + """Build the spec-required JSON/YAML output envelope. + + The envelope wraps command-specific payload in the structure required + by the specification (§Output Rendering Framework): + + .. code-block:: json + + { + "command": "", + "status": "ok", + "exit_code": 0, + "data": { ... }, + "timing": { "duration_ms": 123 }, + "messages": [{ "level": "ok", "text": "..." }] + } + + Parameters + ---------- + data: + The command-specific payload to wrap. + command: + The CLI command string that produced this output. + status: + Envelope status: must be one of ``"ok"``, ``"warn"``, or ``"error"``. + exit_code: + Envelope exit code; must be a non-negative integer. + duration_ms: + Elapsed time in milliseconds. + messages: + Optional list of message dicts; each must have ``"level"`` and + ``"text"`` keys. If *None*, a default single-entry list is generated. + + Raises + ------ + ValueError + If *status* is not one of the allowed values, *exit_code* is negative, + or any entry in *messages* is missing the required ``"level"`` or + ``"text"`` keys. + """ + if status not in _VALID_STATUSES: + raise ValueError(f"status must be one of {_VALID_STATUSES!r}, got {status!r}") + if exit_code < 0: + raise ValueError(f"exit_code must be non-negative, got {exit_code!r}") + if messages is not None: + for msg in messages: + if "level" not in msg or "text" not in msg: + raise ValueError( + f"Each message must have 'level' and 'text' keys, got {msg!r}" + ) + if messages is None: + messages = [ + {"level": status, "text": f"{command} completed" if command else "ok"} + ] + return { + "command": command, + "status": status, + "exit_code": exit_code, + "data": data, + "timing": {"duration_ms": duration_ms}, + "messages": messages, + } + + def format_output( data: dict[str, Any] | list[dict[str, Any]], format_type: str, + *, + command: str = "", + status: str = "ok", + exit_code: int = 0, + messages: list[dict[str, Any]] | None = None, ) -> str: """Format *data* according to *format_type*. @@ -169,6 +249,11 @@ def format_output( rendering so that sensitive values are masked unless the global ``show_secrets`` flag is enabled. + For machine-readable formats (json, yaml) the output is wrapped in + the spec-required envelope (``command``, ``status``, ``exit_code``, + ``data``, ``timing``, ``messages``) before serialisation. Plain + format renders the raw data without an envelope. + For machine-readable formats (json, yaml, plain) the output is written directly to ``sys.stdout`` to avoid Rich ``console.print`` line-wrapping which can introduce literal newlines inside JSON @@ -181,6 +266,16 @@ def format_output( A single dict or a list of dicts (from ``model.as_cli_dict()``). format_type: One of ``json``, ``yaml``, ``plain``, ``table``, ``rich``, ``color``. + command: + The CLI command string that produced this output (e.g. + ``"agents actor list"``). Used in the JSON/YAML envelope. + status: + Envelope status field: ``"ok"``, ``"warn"``, or ``"error"``. + exit_code: + Envelope exit_code field (integer, typically 0 for success). + messages: + Envelope messages array. If *None*, a default single-entry list + is generated from *command* and *status*. Returns ------- @@ -190,6 +285,7 @@ def format_output( """ import sys + t_start = time.monotonic() safe_data = _redact_data(data) fmt = format_type.lower() @@ -201,9 +297,17 @@ def format_output( OutputFormat.PLAIN.value, ): if fmt == OutputFormat.JSON.value: - rendered = _format_json(safe_data) + duration_ms = int((time.monotonic() - t_start) * 1000) + envelope = _build_envelope( + safe_data, command, status, exit_code, duration_ms, messages + ) + rendered = _format_json(envelope) elif fmt == OutputFormat.YAML.value: - rendered = _format_yaml(safe_data) + duration_ms = int((time.monotonic() - t_start) * 1000) + envelope = _build_envelope( + safe_data, command, status, exit_code, duration_ms, messages + ) + rendered = _format_yaml(envelope) else: rendered = _format_plain(safe_data) sys.stdout.write(rendered + "\n")