Files
placeholder/features/steps/cli_output_formats_steps.py
freemo a0df5a4cd0 fix(cli): wrap format_output() in spec-required JSON/YAML envelope across all CLI commands
Implements the spec-required JSON/YAML output envelope for all CLI commands
that use format_output(). The envelope structure is:

  {
    "command": "<command that was run>",
    "status": "ok" | "warn" | "error",
    "exit_code": 0,
    "data": { ... command-specific payload ... },
    "timing": { "duration_ms": 123 },
    "messages": [{ "level": "ok", "text": "..." }]
  }

Changes:
- Add _build_envelope() helper to construct the spec-required envelope
- Add optional command, status, exit_code, messages parameters to format_output()
- Wrap json/yaml output in the envelope; plain/table/rich/color unchanged
- Add timing measurement (duration_ms) to all json/yaml outputs
- Add new BDD feature file (cli_json_envelope.feature) with 14 scenarios
  testing envelope field presence, values, and data payload
- Update 14 existing step files to unwrap the envelope when checking
  specific data keys (backward-compatible via _unwrap_envelope() helper)

Closes #3431
2026-04-05 19:48:40 +00:00

514 lines
18 KiB
Python

"""Step definitions for CLI output format parity tests."""
from __future__ import annotations
import json
from datetime import datetime
from typing import cast
from unittest.mock import MagicMock, patch
import yaml
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
import cleveragents.cli.commands.action as _action_mod
import cleveragents.cli.commands.plan as _plan_mod
from cleveragents.cli.commands.action import app as action_app
from cleveragents.cli.commands.plan import app as plan_app
from cleveragents.cli.formatting import _serialize_value, format_output
from cleveragents.domain.models.core.action import Action, ActionState
from cleveragents.domain.models.core.plan import (
NamespacedName,
Plan,
PlanIdentity,
PlanPhase,
PlanTimestamps,
ProcessingState,
ProjectLink,
)
_ULID = "01ARZ3NDEKTSV4RRFFQ69G5FAV"
def _make_action(name: str = "local/fmt-action") -> Action:
return Action(
namespaced_name=NamespacedName.parse(name),
description="Format test action",
long_description=None,
definition_of_done="Tests pass",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
reusable=True,
read_only=False,
state=ActionState.AVAILABLE,
created_by=None,
created_at=datetime(2025, 1, 15, 10, 0, 0),
updated_at=datetime(2025, 1, 15, 10, 0, 0),
)
def _make_plan(
name: str = "local/fmt-plan",
plan_id: str = _ULID,
) -> Plan:
now = datetime(2025, 1, 15, 10, 0, 0)
return Plan(
identity=PlanIdentity(plan_id=plan_id),
namespaced_name=NamespacedName.parse(name),
description="Format test plan",
definition_of_done="Tests pass",
action_name="local/fmt-action",
phase=PlanPhase.STRATEGIZE,
processing_state=ProcessingState.QUEUED,
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
project_links=[ProjectLink(project_name="proj-a")],
timestamps=PlanTimestamps(created_at=now, updated_at=now),
created_by=None,
reusable=True,
read_only=False,
)
@given("a CLI output format test runner")
def step_format_test_runner(context: Context) -> None:
context.runner = CliRunner()
context.saved_keys = {}
@given("a mocked lifecycle service for format tests")
def step_mocked_lifecycle_for_formats(context: Context) -> None:
context.mock_action_service = MagicMock()
context.mock_plan_service = MagicMock()
context.action_patcher = patch.object(
_action_mod,
"_get_lifecycle_service",
return_value=context.mock_action_service,
)
context.plan_patcher = patch.object(
_plan_mod,
"_get_lifecycle_service",
return_value=context.mock_plan_service,
)
context.action_patcher.start()
context.plan_patcher.start()
# Patch module-level Rich Console objects so they never inject ANSI
# escape codes into structured output (JSON/YAML/plain). Without this,
# console.print(json_string) adds syntax-highlighting escapes when Rich
# detects a real terminal, which makes json.loads() / yaml.safe_load()
# fail on the user's machine even though the tests pass inside a
# headless container where Console auto-disables colour.
from rich.console import Console as _Console
_plain_console = _Console(no_color=True, highlight=False)
context.action_console_patcher = patch.object(
_action_mod, "console", _plain_console
)
context.plan_console_patcher = patch.object(_plan_mod, "console", _plain_console)
context.action_console_patcher.start()
context.plan_console_patcher.start()
# Patch _notify_facade so that the A2A facade bootstrap does not
# trigger real DI container construction, alembic migrations, or
# structlog output that would pollute captured CLI stdout and break
# json.loads() / yaml.safe_load() assertions.
context.plan_facade_patcher = patch.object(
_plan_mod, "_notify_facade", lambda *_a, **_kw: None
)
context.plan_facade_patcher.start()
if not hasattr(context, "_cleanup_handlers"):
context._cleanup_handlers = []
context._cleanup_handlers.append(context.action_patcher.stop)
context._cleanup_handlers.append(context.plan_patcher.stop)
context._cleanup_handlers.append(context.action_console_patcher.stop)
context._cleanup_handlers.append(context.plan_console_patcher.stop)
context._cleanup_handlers.append(context.plan_facade_patcher.stop)
# ------- Given steps -------
@given("there are actions for format testing")
def step_actions_for_format(context: Context) -> None:
context.mock_action_service.list_actions.return_value = [
_make_action("local/action-a"),
_make_action("local/action-b"),
]
@given("there is a single action for format testing")
def step_single_action_for_format(context: Context) -> None:
context.format_action = _make_action()
context.mock_action_service.get_action_by_name.return_value = context.format_action
@given("there are plans for format testing")
def step_plans_for_format(context: Context) -> None:
context.mock_plan_service.list_plans.return_value = [
_make_plan("local/plan-a", "01ARZ3NDEKTSV4RRFFQ69G5FAV"),
_make_plan("local/plan-b", "01ARZ3NDEKTSV4RRFFQ69G5FAW"),
]
@given("there is a single plan for format testing")
def step_single_plan_for_format(context: Context) -> None:
context.format_plan = _make_plan()
context.mock_plan_service.get_plan.return_value = context.format_plan
# ------- When steps -------
@when("I run action list with --format json")
def step_action_list_json(context: Context) -> None:
context.result = context.runner.invoke(action_app, ["list", "--format", "json"])
@when("I run action show with --format yaml")
def step_action_show_yaml(context: Context) -> None:
context.result = context.runner.invoke(
action_app,
["show", str(context.format_action.namespaced_name), "--format", "yaml"],
)
@when("I run action show with --format json")
def step_action_show_json(context: Context) -> None:
context.result = context.runner.invoke(
action_app,
["show", str(context.format_action.namespaced_name), "--format", "json"],
)
@when("I run plan list with --format json")
def step_plan_list_json(context: Context) -> None:
context.result = context.runner.invoke(plan_app, ["list", "--format", "json"])
@when("I run plan status with plan id and --format json")
def step_plan_status_json(context: Context) -> None:
context.result = context.runner.invoke(
plan_app, ["status", _ULID, "--format", "json"]
)
@when("I run plan status with plan id and --format plain")
def step_plan_status_plain(context: Context) -> None:
context.result = context.runner.invoke(
plan_app, ["status", _ULID, "--format", "plain"]
)
_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"}
def _unwrap_envelope(parsed: dict | list) -> dict | list:
"""Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is."""
if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()):
return cast("dict | list", parsed["data"])
return parsed
@when("I save the output keys as json_keys")
def step_save_json_keys(context: Context) -> None:
parsed = json.loads(context.result.output.strip())
data = _unwrap_envelope(parsed)
if isinstance(data, list):
context.saved_keys["json_keys"] = set(data[0].keys()) if data else set()
else:
context.saved_keys["json_keys"] = set(data.keys())
@when("I save the output keys as yaml_keys")
def step_save_yaml_keys(context: Context) -> None:
parsed = yaml.safe_load(context.result.output.strip())
data = _unwrap_envelope(parsed)
if isinstance(data, list):
context.saved_keys["yaml_keys"] = set(data[0].keys()) if data else set()
else:
context.saved_keys["yaml_keys"] = set(data.keys())
# ------- Then steps -------
@then("the output should be valid JSON")
def step_output_valid_json(context: Context) -> None:
assert context.result.exit_code == 0, (
f"CLI exited with {context.result.exit_code}: {context.result.output}"
)
parsed = json.loads(context.result.output.strip())
assert parsed is not None
# Verify spec-required envelope fields are present
assert isinstance(parsed, dict), f"Expected envelope dict, got {type(parsed)}"
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
assert field in parsed, f"Envelope field '{field}' missing from JSON output"
@then("the output should be valid YAML")
def step_output_valid_yaml(context: Context) -> None:
assert context.result.exit_code == 0, (
f"CLI exited with {context.result.exit_code}: {context.result.output}"
)
parsed = yaml.safe_load(context.result.output.strip())
assert parsed is not None
# Verify spec-required envelope fields are present
assert isinstance(parsed, dict), f"Expected envelope dict, got {type(parsed)}"
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
assert field in parsed, f"Envelope field '{field}' missing from YAML output"
@then('the JSON should contain key "{key}"')
def step_json_contains_key(context: Context, key: str) -> None:
output = context.result.output.strip()
parsed = json.loads(output)
# Unwrap envelope: look inside "data" field
data = _unwrap_envelope(parsed)
if isinstance(data, list):
assert data, f"data array is empty, cannot check key '{key}'"
assert key in data[0], f"Key '{key}' not in {list(data[0].keys())}"
else:
assert key in data, f"Key '{key}' not in {list(data.keys())}"
@then('the YAML should contain key "{key}"')
def step_yaml_contains_key(context: Context, key: str) -> None:
output = context.result.output.strip()
parsed = yaml.safe_load(output)
# Unwrap envelope: look inside "data" field
data = _unwrap_envelope(parsed)
if isinstance(data, list):
assert data, f"data array is empty, cannot check key '{key}'"
assert key in data[0], f"Key '{key}' not in {list(data[0].keys())}"
else:
assert key in data, f"Key '{key}' not in {list(data.keys())}"
@then('the format output should contain "{text}"')
def step_format_output_contains_text(context: Context, text: str) -> None:
assert context.result.exit_code == 0, (
f"CLI exited with {context.result.exit_code}: {context.result.output}"
)
assert text in context.result.output, (
f"'{text}' not found in output: {context.result.output[:300]}"
)
@then("json_keys and yaml_keys should match")
def step_keys_match(context: Context) -> None:
jk = context.saved_keys["json_keys"]
yk = context.saved_keys["yaml_keys"]
assert jk == yk, f"Key mismatch: json={jk}, yaml={yk}"
# ------- Additional Given steps -------
@given("there is an archivable action for format testing")
def step_archivable_action(context: Context) -> None:
action = _make_action()
context.format_action = action
context.mock_action_service.get_action_by_name.return_value = action
context.mock_action_service.archive_action.return_value = action
@given("there are plans for format status listing")
def step_plans_for_status_listing(context: Context) -> None:
context.mock_plan_service.list_plans.return_value = [
_make_plan("local/plan-a", "01ARZ3NDEKTSV4RRFFQ69G5FAV"),
_make_plan("local/plan-b", "01ARZ3NDEKTSV4RRFFQ69G5FAW"),
]
@given("there is an action for plan use format test")
def step_action_for_plan_use(context: Context) -> None:
action = _make_action()
plan = _make_plan()
context.mock_plan_service.get_action_by_name.return_value = action
context.mock_plan_service.use_action.return_value = plan
@given("there is a plan for cancel format test")
def step_plan_for_cancel(context: Context) -> None:
context.mock_plan_service.cancel_plan.return_value = _make_plan()
# ------- Additional When steps -------
@when("I run action list with --format table")
def step_action_list_table(context: Context) -> None:
context.result = context.runner.invoke(action_app, ["list", "--format", "table"])
@when("I run action list with --format plain")
def step_action_list_plain(context: Context) -> None:
context.result = context.runner.invoke(action_app, ["list", "--format", "plain"])
@when("I run action archive with --format json")
def step_action_archive_json(context: Context) -> None:
context.result = context.runner.invoke(
action_app,
[
"archive",
str(context.format_action.namespaced_name),
"--format",
"json",
],
)
@when("I run plan status without id and --format json")
def step_plan_status_no_id_json(context: Context) -> None:
context.result = context.runner.invoke(plan_app, ["status", "--format", "json"])
@when("I run plan use with --format json")
def step_plan_use_json(context: Context) -> None:
context.result = context.runner.invoke(
plan_app,
[
"use",
"local/fmt-action",
"proj-a",
"--format",
"json",
],
)
@when("I run plan cancel with --format json")
def step_plan_cancel_json(context: Context) -> None:
context.result = context.runner.invoke(
plan_app,
["cancel", _ULID, "--format", "json"],
)
# ------- Direct formatting module tests -------
def _capture_format_output(data, fmt):
"""Call format_output capturing stdout (machine-readable formats write there)."""
from contextlib import redirect_stdout
from io import StringIO
buf = StringIO()
with redirect_stdout(buf):
result = format_output(data, fmt)
return result or buf.getvalue().rstrip("\n")
@when("I call format_output with a dict and format json")
def step_call_format_json(context: Context) -> None:
data = {
"name": "test",
"count": 42,
"created_at": datetime(2025, 1, 15).isoformat(),
}
context.format_result = _capture_format_output(data, "json")
@when("I call format_output with a dict and format yaml")
def step_call_format_yaml(context: Context) -> None:
data = {"name": "test", "count": 42}
context.format_result = _capture_format_output(data, "yaml")
@when("I call format_output with a dict and format plain")
def step_call_format_plain(context: Context) -> None:
data = {
"name": "test",
"nested": {"a": 1, "b": 2},
"items": ["x", "y"],
}
context.format_result = _capture_format_output(data, "plain")
@when("I call format_output with a dict and format table")
def step_call_format_table(context: Context) -> None:
data = {"name": "test", "count": 42}
context.format_result = _capture_format_output(data, "table")
@when("I call format_output with a dict and format rich")
def step_call_format_rich(context: Context) -> None:
data = {"name": "test", "count": 42}
context.format_result = _capture_format_output(data, "rich")
@when("I call format_output with a list and format plain")
def step_call_format_list_plain(context: Context) -> None:
data = [
{"name": "a", "value": 1},
{"name": "b", "value": 2},
]
context.format_result = _capture_format_output(data, "plain")
@when("I call serialize_value with enum and nested data")
def step_call_serialize_with_enum(context: Context) -> None:
context.serialized = _serialize_value(
{
"state": ActionState.AVAILABLE,
"nested": {"key": "val"},
"items": [ActionState.ARCHIVED, datetime(2025, 1, 1)],
"ts": datetime(2025, 6, 1, 12, 0),
}
)
# ------- Additional Then steps -------
@then("the format result should be valid JSON dict")
def step_format_result_json_dict(context: Context) -> None:
parsed = json.loads(context.format_result)
assert isinstance(parsed, dict)
# Verify spec-required envelope fields are present
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
assert field in parsed, f"Envelope field '{field}' missing from JSON result"
# Verify data field contains the original dict
assert isinstance(parsed["data"], dict)
@then("the format result should be valid YAML dict")
def step_format_result_yaml_dict(context: Context) -> None:
parsed = yaml.safe_load(context.format_result)
assert isinstance(parsed, dict)
# Verify spec-required envelope fields are present
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
assert field in parsed, f"Envelope field '{field}' missing from YAML result"
# Verify data field contains the original dict
assert isinstance(parsed["data"], dict)
@then("the format result should contain plain key-value pairs")
def step_format_result_plain(context: Context) -> None:
assert "name: test" in context.format_result
assert "nested:" in context.format_result
assert "items:" in context.format_result
@then("the format result should contain table output")
def step_format_result_table(context: Context) -> None:
assert "name" in context.format_result
assert "count" in context.format_result
@then("the format result should contain separator lines")
def step_format_result_separator(context: Context) -> None:
assert "---" in context.format_result
@then("the serialized result should have string enum values")
def step_serialized_enum_values(context: Context) -> None:
assert context.serialized["state"] == "available"
assert isinstance(context.serialized["nested"], dict)
assert context.serialized["items"][0] == "archived"
assert isinstance(context.serialized["ts"], str)