forked from HAL9000/cleveragents-core
a0df5a4cd0
Implements the spec-required JSON/YAML output envelope for all CLI commands
that use format_output(). The envelope structure is:
{
"command": "<command that was run>",
"status": "ok" | "warn" | "error",
"exit_code": 0,
"data": { ... command-specific payload ... },
"timing": { "duration_ms": 123 },
"messages": [{ "level": "ok", "text": "..." }]
}
Changes:
- Add _build_envelope() helper to construct the spec-required envelope
- Add optional command, status, exit_code, messages parameters to format_output()
- Wrap json/yaml output in the envelope; plain/table/rich/color unchanged
- Add timing measurement (duration_ms) to all json/yaml outputs
- Add new BDD feature file (cli_json_envelope.feature) with 14 scenarios
testing envelope field presence, values, and data payload
- Update 14 existing step files to unwrap the envelope when checking
specific data keys (backward-compatible via _unwrap_envelope() helper)
Closes #3431
514 lines
18 KiB
Python
514 lines
18 KiB
Python
"""Step definitions for CLI output format parity tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
from typing import cast
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import yaml
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
from typer.testing import CliRunner
|
|
|
|
import cleveragents.cli.commands.action as _action_mod
|
|
import cleveragents.cli.commands.plan as _plan_mod
|
|
from cleveragents.cli.commands.action import app as action_app
|
|
from cleveragents.cli.commands.plan import app as plan_app
|
|
from cleveragents.cli.formatting import _serialize_value, format_output
|
|
from cleveragents.domain.models.core.action import Action, ActionState
|
|
from cleveragents.domain.models.core.plan import (
|
|
NamespacedName,
|
|
Plan,
|
|
PlanIdentity,
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
ProjectLink,
|
|
)
|
|
|
|
_ULID = "01ARZ3NDEKTSV4RRFFQ69G5FAV"
|
|
|
|
|
|
def _make_action(name: str = "local/fmt-action") -> Action:
|
|
return Action(
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Format test action",
|
|
long_description=None,
|
|
definition_of_done="Tests pass",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
reusable=True,
|
|
read_only=False,
|
|
state=ActionState.AVAILABLE,
|
|
created_by=None,
|
|
created_at=datetime(2025, 1, 15, 10, 0, 0),
|
|
updated_at=datetime(2025, 1, 15, 10, 0, 0),
|
|
)
|
|
|
|
|
|
def _make_plan(
|
|
name: str = "local/fmt-plan",
|
|
plan_id: str = _ULID,
|
|
) -> Plan:
|
|
now = datetime(2025, 1, 15, 10, 0, 0)
|
|
return Plan(
|
|
identity=PlanIdentity(plan_id=plan_id),
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Format test plan",
|
|
definition_of_done="Tests pass",
|
|
action_name="local/fmt-action",
|
|
phase=PlanPhase.STRATEGIZE,
|
|
processing_state=ProcessingState.QUEUED,
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
project_links=[ProjectLink(project_name="proj-a")],
|
|
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
|
created_by=None,
|
|
reusable=True,
|
|
read_only=False,
|
|
)
|
|
|
|
|
|
@given("a CLI output format test runner")
|
|
def step_format_test_runner(context: Context) -> None:
|
|
context.runner = CliRunner()
|
|
context.saved_keys = {}
|
|
|
|
|
|
@given("a mocked lifecycle service for format tests")
|
|
def step_mocked_lifecycle_for_formats(context: Context) -> None:
|
|
context.mock_action_service = MagicMock()
|
|
context.mock_plan_service = MagicMock()
|
|
context.action_patcher = patch.object(
|
|
_action_mod,
|
|
"_get_lifecycle_service",
|
|
return_value=context.mock_action_service,
|
|
)
|
|
context.plan_patcher = patch.object(
|
|
_plan_mod,
|
|
"_get_lifecycle_service",
|
|
return_value=context.mock_plan_service,
|
|
)
|
|
context.action_patcher.start()
|
|
context.plan_patcher.start()
|
|
|
|
# Patch module-level Rich Console objects so they never inject ANSI
|
|
# escape codes into structured output (JSON/YAML/plain). Without this,
|
|
# console.print(json_string) adds syntax-highlighting escapes when Rich
|
|
# detects a real terminal, which makes json.loads() / yaml.safe_load()
|
|
# fail on the user's machine even though the tests pass inside a
|
|
# headless container where Console auto-disables colour.
|
|
from rich.console import Console as _Console
|
|
|
|
_plain_console = _Console(no_color=True, highlight=False)
|
|
context.action_console_patcher = patch.object(
|
|
_action_mod, "console", _plain_console
|
|
)
|
|
context.plan_console_patcher = patch.object(_plan_mod, "console", _plain_console)
|
|
context.action_console_patcher.start()
|
|
context.plan_console_patcher.start()
|
|
|
|
# Patch _notify_facade so that the A2A facade bootstrap does not
|
|
# trigger real DI container construction, alembic migrations, or
|
|
# structlog output that would pollute captured CLI stdout and break
|
|
# json.loads() / yaml.safe_load() assertions.
|
|
context.plan_facade_patcher = patch.object(
|
|
_plan_mod, "_notify_facade", lambda *_a, **_kw: None
|
|
)
|
|
context.plan_facade_patcher.start()
|
|
|
|
if not hasattr(context, "_cleanup_handlers"):
|
|
context._cleanup_handlers = []
|
|
context._cleanup_handlers.append(context.action_patcher.stop)
|
|
context._cleanup_handlers.append(context.plan_patcher.stop)
|
|
context._cleanup_handlers.append(context.action_console_patcher.stop)
|
|
context._cleanup_handlers.append(context.plan_console_patcher.stop)
|
|
context._cleanup_handlers.append(context.plan_facade_patcher.stop)
|
|
|
|
|
|
# ------- Given steps -------
|
|
|
|
|
|
@given("there are actions for format testing")
|
|
def step_actions_for_format(context: Context) -> None:
|
|
context.mock_action_service.list_actions.return_value = [
|
|
_make_action("local/action-a"),
|
|
_make_action("local/action-b"),
|
|
]
|
|
|
|
|
|
@given("there is a single action for format testing")
|
|
def step_single_action_for_format(context: Context) -> None:
|
|
context.format_action = _make_action()
|
|
context.mock_action_service.get_action_by_name.return_value = context.format_action
|
|
|
|
|
|
@given("there are plans for format testing")
|
|
def step_plans_for_format(context: Context) -> None:
|
|
context.mock_plan_service.list_plans.return_value = [
|
|
_make_plan("local/plan-a", "01ARZ3NDEKTSV4RRFFQ69G5FAV"),
|
|
_make_plan("local/plan-b", "01ARZ3NDEKTSV4RRFFQ69G5FAW"),
|
|
]
|
|
|
|
|
|
@given("there is a single plan for format testing")
|
|
def step_single_plan_for_format(context: Context) -> None:
|
|
context.format_plan = _make_plan()
|
|
context.mock_plan_service.get_plan.return_value = context.format_plan
|
|
|
|
|
|
# ------- When steps -------
|
|
|
|
|
|
@when("I run action list with --format json")
|
|
def step_action_list_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(action_app, ["list", "--format", "json"])
|
|
|
|
|
|
@when("I run action show with --format yaml")
|
|
def step_action_show_yaml(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
action_app,
|
|
["show", str(context.format_action.namespaced_name), "--format", "yaml"],
|
|
)
|
|
|
|
|
|
@when("I run action show with --format json")
|
|
def step_action_show_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
action_app,
|
|
["show", str(context.format_action.namespaced_name), "--format", "json"],
|
|
)
|
|
|
|
|
|
@when("I run plan list with --format json")
|
|
def step_plan_list_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(plan_app, ["list", "--format", "json"])
|
|
|
|
|
|
@when("I run plan status with plan id and --format json")
|
|
def step_plan_status_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["status", _ULID, "--format", "json"]
|
|
)
|
|
|
|
|
|
@when("I run plan status with plan id and --format plain")
|
|
def step_plan_status_plain(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["status", _ULID, "--format", "plain"]
|
|
)
|
|
|
|
|
|
_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"}
|
|
|
|
|
|
def _unwrap_envelope(parsed: dict | list) -> dict | list:
|
|
"""Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is."""
|
|
if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()):
|
|
return cast("dict | list", parsed["data"])
|
|
return parsed
|
|
|
|
|
|
@when("I save the output keys as json_keys")
|
|
def step_save_json_keys(context: Context) -> None:
|
|
parsed = json.loads(context.result.output.strip())
|
|
data = _unwrap_envelope(parsed)
|
|
if isinstance(data, list):
|
|
context.saved_keys["json_keys"] = set(data[0].keys()) if data else set()
|
|
else:
|
|
context.saved_keys["json_keys"] = set(data.keys())
|
|
|
|
|
|
@when("I save the output keys as yaml_keys")
|
|
def step_save_yaml_keys(context: Context) -> None:
|
|
parsed = yaml.safe_load(context.result.output.strip())
|
|
data = _unwrap_envelope(parsed)
|
|
if isinstance(data, list):
|
|
context.saved_keys["yaml_keys"] = set(data[0].keys()) if data else set()
|
|
else:
|
|
context.saved_keys["yaml_keys"] = set(data.keys())
|
|
|
|
|
|
# ------- Then steps -------
|
|
|
|
|
|
@then("the output should be valid JSON")
|
|
def step_output_valid_json(context: Context) -> None:
|
|
assert context.result.exit_code == 0, (
|
|
f"CLI exited with {context.result.exit_code}: {context.result.output}"
|
|
)
|
|
parsed = json.loads(context.result.output.strip())
|
|
assert parsed is not None
|
|
# Verify spec-required envelope fields are present
|
|
assert isinstance(parsed, dict), f"Expected envelope dict, got {type(parsed)}"
|
|
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
|
|
assert field in parsed, f"Envelope field '{field}' missing from JSON output"
|
|
|
|
|
|
@then("the output should be valid YAML")
|
|
def step_output_valid_yaml(context: Context) -> None:
|
|
assert context.result.exit_code == 0, (
|
|
f"CLI exited with {context.result.exit_code}: {context.result.output}"
|
|
)
|
|
parsed = yaml.safe_load(context.result.output.strip())
|
|
assert parsed is not None
|
|
# Verify spec-required envelope fields are present
|
|
assert isinstance(parsed, dict), f"Expected envelope dict, got {type(parsed)}"
|
|
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
|
|
assert field in parsed, f"Envelope field '{field}' missing from YAML output"
|
|
|
|
|
|
@then('the JSON should contain key "{key}"')
|
|
def step_json_contains_key(context: Context, key: str) -> None:
|
|
output = context.result.output.strip()
|
|
parsed = json.loads(output)
|
|
# Unwrap envelope: look inside "data" field
|
|
data = _unwrap_envelope(parsed)
|
|
if isinstance(data, list):
|
|
assert data, f"data array is empty, cannot check key '{key}'"
|
|
assert key in data[0], f"Key '{key}' not in {list(data[0].keys())}"
|
|
else:
|
|
assert key in data, f"Key '{key}' not in {list(data.keys())}"
|
|
|
|
|
|
@then('the YAML should contain key "{key}"')
|
|
def step_yaml_contains_key(context: Context, key: str) -> None:
|
|
output = context.result.output.strip()
|
|
parsed = yaml.safe_load(output)
|
|
# Unwrap envelope: look inside "data" field
|
|
data = _unwrap_envelope(parsed)
|
|
if isinstance(data, list):
|
|
assert data, f"data array is empty, cannot check key '{key}'"
|
|
assert key in data[0], f"Key '{key}' not in {list(data[0].keys())}"
|
|
else:
|
|
assert key in data, f"Key '{key}' not in {list(data.keys())}"
|
|
|
|
|
|
@then('the format output should contain "{text}"')
|
|
def step_format_output_contains_text(context: Context, text: str) -> None:
|
|
assert context.result.exit_code == 0, (
|
|
f"CLI exited with {context.result.exit_code}: {context.result.output}"
|
|
)
|
|
assert text in context.result.output, (
|
|
f"'{text}' not found in output: {context.result.output[:300]}"
|
|
)
|
|
|
|
|
|
@then("json_keys and yaml_keys should match")
|
|
def step_keys_match(context: Context) -> None:
|
|
jk = context.saved_keys["json_keys"]
|
|
yk = context.saved_keys["yaml_keys"]
|
|
assert jk == yk, f"Key mismatch: json={jk}, yaml={yk}"
|
|
|
|
|
|
# ------- Additional Given steps -------
|
|
|
|
|
|
@given("there is an archivable action for format testing")
|
|
def step_archivable_action(context: Context) -> None:
|
|
action = _make_action()
|
|
context.format_action = action
|
|
context.mock_action_service.get_action_by_name.return_value = action
|
|
context.mock_action_service.archive_action.return_value = action
|
|
|
|
|
|
@given("there are plans for format status listing")
|
|
def step_plans_for_status_listing(context: Context) -> None:
|
|
context.mock_plan_service.list_plans.return_value = [
|
|
_make_plan("local/plan-a", "01ARZ3NDEKTSV4RRFFQ69G5FAV"),
|
|
_make_plan("local/plan-b", "01ARZ3NDEKTSV4RRFFQ69G5FAW"),
|
|
]
|
|
|
|
|
|
@given("there is an action for plan use format test")
|
|
def step_action_for_plan_use(context: Context) -> None:
|
|
action = _make_action()
|
|
plan = _make_plan()
|
|
context.mock_plan_service.get_action_by_name.return_value = action
|
|
context.mock_plan_service.use_action.return_value = plan
|
|
|
|
|
|
@given("there is a plan for cancel format test")
|
|
def step_plan_for_cancel(context: Context) -> None:
|
|
context.mock_plan_service.cancel_plan.return_value = _make_plan()
|
|
|
|
|
|
# ------- Additional When steps -------
|
|
|
|
|
|
@when("I run action list with --format table")
|
|
def step_action_list_table(context: Context) -> None:
|
|
context.result = context.runner.invoke(action_app, ["list", "--format", "table"])
|
|
|
|
|
|
@when("I run action list with --format plain")
|
|
def step_action_list_plain(context: Context) -> None:
|
|
context.result = context.runner.invoke(action_app, ["list", "--format", "plain"])
|
|
|
|
|
|
@when("I run action archive with --format json")
|
|
def step_action_archive_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
action_app,
|
|
[
|
|
"archive",
|
|
str(context.format_action.namespaced_name),
|
|
"--format",
|
|
"json",
|
|
],
|
|
)
|
|
|
|
|
|
@when("I run plan status without id and --format json")
|
|
def step_plan_status_no_id_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(plan_app, ["status", "--format", "json"])
|
|
|
|
|
|
@when("I run plan use with --format json")
|
|
def step_plan_use_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/fmt-action",
|
|
"proj-a",
|
|
"--format",
|
|
"json",
|
|
],
|
|
)
|
|
|
|
|
|
@when("I run plan cancel with --format json")
|
|
def step_plan_cancel_json(context: Context) -> None:
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
["cancel", _ULID, "--format", "json"],
|
|
)
|
|
|
|
|
|
# ------- Direct formatting module tests -------
|
|
|
|
|
|
def _capture_format_output(data, fmt):
|
|
"""Call format_output capturing stdout (machine-readable formats write there)."""
|
|
from contextlib import redirect_stdout
|
|
from io import StringIO
|
|
|
|
buf = StringIO()
|
|
with redirect_stdout(buf):
|
|
result = format_output(data, fmt)
|
|
return result or buf.getvalue().rstrip("\n")
|
|
|
|
|
|
@when("I call format_output with a dict and format json")
|
|
def step_call_format_json(context: Context) -> None:
|
|
data = {
|
|
"name": "test",
|
|
"count": 42,
|
|
"created_at": datetime(2025, 1, 15).isoformat(),
|
|
}
|
|
context.format_result = _capture_format_output(data, "json")
|
|
|
|
|
|
@when("I call format_output with a dict and format yaml")
|
|
def step_call_format_yaml(context: Context) -> None:
|
|
data = {"name": "test", "count": 42}
|
|
context.format_result = _capture_format_output(data, "yaml")
|
|
|
|
|
|
@when("I call format_output with a dict and format plain")
|
|
def step_call_format_plain(context: Context) -> None:
|
|
data = {
|
|
"name": "test",
|
|
"nested": {"a": 1, "b": 2},
|
|
"items": ["x", "y"],
|
|
}
|
|
context.format_result = _capture_format_output(data, "plain")
|
|
|
|
|
|
@when("I call format_output with a dict and format table")
|
|
def step_call_format_table(context: Context) -> None:
|
|
data = {"name": "test", "count": 42}
|
|
context.format_result = _capture_format_output(data, "table")
|
|
|
|
|
|
@when("I call format_output with a dict and format rich")
|
|
def step_call_format_rich(context: Context) -> None:
|
|
data = {"name": "test", "count": 42}
|
|
context.format_result = _capture_format_output(data, "rich")
|
|
|
|
|
|
@when("I call format_output with a list and format plain")
|
|
def step_call_format_list_plain(context: Context) -> None:
|
|
data = [
|
|
{"name": "a", "value": 1},
|
|
{"name": "b", "value": 2},
|
|
]
|
|
context.format_result = _capture_format_output(data, "plain")
|
|
|
|
|
|
@when("I call serialize_value with enum and nested data")
|
|
def step_call_serialize_with_enum(context: Context) -> None:
|
|
context.serialized = _serialize_value(
|
|
{
|
|
"state": ActionState.AVAILABLE,
|
|
"nested": {"key": "val"},
|
|
"items": [ActionState.ARCHIVED, datetime(2025, 1, 1)],
|
|
"ts": datetime(2025, 6, 1, 12, 0),
|
|
}
|
|
)
|
|
|
|
|
|
# ------- Additional Then steps -------
|
|
|
|
|
|
@then("the format result should be valid JSON dict")
|
|
def step_format_result_json_dict(context: Context) -> None:
|
|
parsed = json.loads(context.format_result)
|
|
assert isinstance(parsed, dict)
|
|
# Verify spec-required envelope fields are present
|
|
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
|
|
assert field in parsed, f"Envelope field '{field}' missing from JSON result"
|
|
# Verify data field contains the original dict
|
|
assert isinstance(parsed["data"], dict)
|
|
|
|
|
|
@then("the format result should be valid YAML dict")
|
|
def step_format_result_yaml_dict(context: Context) -> None:
|
|
parsed = yaml.safe_load(context.format_result)
|
|
assert isinstance(parsed, dict)
|
|
# Verify spec-required envelope fields are present
|
|
for field in ("command", "status", "exit_code", "data", "timing", "messages"):
|
|
assert field in parsed, f"Envelope field '{field}' missing from YAML result"
|
|
# Verify data field contains the original dict
|
|
assert isinstance(parsed["data"], dict)
|
|
|
|
|
|
@then("the format result should contain plain key-value pairs")
|
|
def step_format_result_plain(context: Context) -> None:
|
|
assert "name: test" in context.format_result
|
|
assert "nested:" in context.format_result
|
|
assert "items:" in context.format_result
|
|
|
|
|
|
@then("the format result should contain table output")
|
|
def step_format_result_table(context: Context) -> None:
|
|
assert "name" in context.format_result
|
|
assert "count" in context.format_result
|
|
|
|
|
|
@then("the format result should contain separator lines")
|
|
def step_format_result_separator(context: Context) -> None:
|
|
assert "---" in context.format_result
|
|
|
|
|
|
@then("the serialized result should have string enum values")
|
|
def step_serialized_enum_values(context: Context) -> None:
|
|
assert context.serialized["state"] == "available"
|
|
assert isinstance(context.serialized["nested"], dict)
|
|
assert context.serialized["items"][0] == "archived"
|
|
assert isinstance(context.serialized["ts"], str)
|