Files
cleveragents-core/features/steps/cli_json_envelope_steps.py
freemo a0df5a4cd0
CI / lint (pull_request) Successful in 27s
CI / typecheck (pull_request) Successful in 57s
CI / security (pull_request) Successful in 1m2s
CI / quality (pull_request) Successful in 34s
CI / build (pull_request) Successful in 23s
CI / helm (pull_request) Successful in 26s
CI / unit_tests (pull_request) Failing after 7m7s
CI / docker (pull_request) Has been skipped
CI / e2e_tests (pull_request) Failing after 11m8s
CI / integration_tests (pull_request) Failing after 22m44s
CI / coverage (pull_request) Successful in 10m50s
CI / status-check (pull_request) Failing after 2s
CI / benchmark-publish (pull_request) Has been skipped
CI / benchmark-regression (pull_request) Successful in 57m6s
fix(cli): wrap format_output() in spec-required JSON/YAML envelope across all CLI commands
Implements the spec-required JSON/YAML output envelope for all CLI commands
that use format_output(). The envelope structure is:

  {
    "command": "<command that was run>",
    "status": "ok" | "warn" | "error",
    "exit_code": 0,
    "data": { ... command-specific payload ... },
    "timing": { "duration_ms": 123 },
    "messages": [{ "level": "ok", "text": "..." }]
  }

Changes:
- Add _build_envelope() helper to construct the spec-required envelope
- Add optional command, status, exit_code, messages parameters to format_output()
- Wrap json/yaml output in the envelope; plain/table/rich/color unchanged
- Add timing measurement (duration_ms) to all json/yaml outputs
- Add new BDD feature file (cli_json_envelope.feature) with 14 scenarios
  testing envelope field presence, values, and data payload
- Update 14 existing step files to unwrap the envelope when checking
  specific data keys (backward-compatible via _unwrap_envelope() helper)

Closes #3431
2026-04-05 19:48:40 +00:00

168 lines
7.0 KiB
Python

"""Step definitions for CLI JSON/YAML envelope structure tests."""
from __future__ import annotations
import json
from contextlib import redirect_stdout
from io import StringIO
import yaml
from behave import then, when
from behave.runner import Context
from cleveragents.cli.formatting import format_output
_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"}
def _capture_format_output(data, fmt, **kwargs):
"""Call format_output capturing stdout (machine-readable formats write there)."""
buf = StringIO()
with redirect_stdout(buf):
result = format_output(data, fmt, **kwargs)
return result or buf.getvalue().rstrip("\n")
def _parse_json_envelope(output: str) -> dict:
parsed = json.loads(output.strip())
assert isinstance(parsed, dict), f"Expected dict envelope, got {type(parsed)}"
return parsed
def _parse_yaml_envelope(output: str) -> dict:
parsed = yaml.safe_load(output.strip())
assert isinstance(parsed, dict), f"Expected dict envelope, got {type(parsed)}"
return parsed
# ── Envelope field presence ──────────────────────────────────────────────
@then('the JSON envelope should contain field "{field}"')
def step_json_envelope_has_field(context: Context, field: str) -> None:
envelope = _parse_json_envelope(context.result.output)
assert field in envelope, (
f"Envelope field '{field}' missing. Keys present: {list(envelope.keys())}"
)
@then('the YAML envelope should contain field "{field}"')
def step_yaml_envelope_has_field(context: Context, field: str) -> None:
envelope = _parse_yaml_envelope(context.result.output)
assert field in envelope, (
f"Envelope field '{field}' missing. Keys present: {list(envelope.keys())}"
)
# ── Envelope field values ────────────────────────────────────────────────
@then('the JSON envelope status should be "{expected_status}"')
def step_json_envelope_status(context: Context, expected_status: str) -> None:
envelope = _parse_json_envelope(context.result.output)
assert envelope["status"] == expected_status, (
f"Expected status '{expected_status}', got '{envelope['status']}'"
)
@then("the JSON envelope exit_code should be {expected_code:d}")
def step_json_envelope_exit_code(context: Context, expected_code: int) -> None:
envelope = _parse_json_envelope(context.result.output)
assert envelope["exit_code"] == expected_code, (
f"Expected exit_code {expected_code}, got {envelope['exit_code']}"
)
@then('the JSON envelope timing should contain "{key}"')
def step_json_envelope_timing_key(context: Context, key: str) -> None:
envelope = _parse_json_envelope(context.result.output)
timing = envelope.get("timing")
assert isinstance(timing, dict), f"Expected timing dict, got {type(timing)}"
assert key in timing, f"Key '{key}' missing from timing: {list(timing.keys())}"
assert isinstance(timing[key], (int, float)), (
f"timing.{key} should be numeric, got {type(timing[key])}"
)
@then("the JSON envelope messages should be a non-empty list")
def step_json_envelope_messages_list(context: Context) -> None:
envelope = _parse_json_envelope(context.result.output)
messages = envelope.get("messages")
assert isinstance(messages, list), f"Expected messages list, got {type(messages)}"
assert len(messages) > 0, "messages list should not be empty"
# Each message should have level and text
for msg in messages:
assert "level" in msg, f"Message missing 'level': {msg}"
assert "text" in msg, f"Message missing 'text': {msg}"
# ── Data field contains actual payload ──────────────────────────────────
@then("the JSON envelope data should contain actor records")
def step_json_envelope_data_actors(context: Context) -> None:
envelope = _parse_json_envelope(context.result.output)
data = envelope["data"]
assert isinstance(data, list), f"Expected data list, got {type(data)}"
assert len(data) > 0, "data list should not be empty"
# Each actor record should have namespaced_name
assert "namespaced_name" in data[0], (
f"Actor record missing 'namespaced_name': {list(data[0].keys())}"
)
@then("the YAML envelope data should contain actor records")
def step_yaml_envelope_data_actors(context: Context) -> None:
envelope = _parse_yaml_envelope(context.result.output)
data = envelope["data"]
assert isinstance(data, list), f"Expected data list, got {type(data)}"
assert len(data) > 0, "data list should not be empty"
assert "namespaced_name" in data[0], (
f"Actor record missing 'namespaced_name': {list(data[0].keys())}"
)
# ── format_output() direct call envelope ────────────────────────────────
@then("the JSON result data field should contain original keys")
def step_json_result_data_original_keys(context: Context) -> None:
parsed = json.loads(context.format_result)
data = parsed["data"]
assert isinstance(data, dict), f"Expected data dict, got {type(data)}"
# The original dict had "name" and "count" keys
assert "name" in data, f"'name' missing from data: {list(data.keys())}"
assert "count" in data, f"'count' missing from data: {list(data.keys())}"
@then("the YAML result data field should contain original keys")
def step_yaml_result_data_original_keys(context: Context) -> None:
parsed = yaml.safe_load(context.format_result)
data = parsed["data"]
assert isinstance(data, dict), f"Expected data dict, got {type(data)}"
assert "name" in data, f"'name' missing from data: {list(data.keys())}"
assert "count" in data, f"'count' missing from data: {list(data.keys())}"
@when('I call format_output with command name "{command_name}" and format json')
def step_call_format_output_with_command(context: Context, command_name: str) -> None:
data = {"name": "test", "count": 42}
context.format_result = _capture_format_output(data, "json", command=command_name)
@then('the JSON envelope command should be "{expected_command}"')
def step_json_envelope_command(context: Context, expected_command: str) -> None:
parsed = json.loads(context.format_result)
assert parsed["command"] == expected_command, (
f"Expected command '{expected_command}', got '{parsed['command']}'"
)
@then("the plain result should not contain envelope keys")
def step_plain_no_envelope(context: Context) -> None:
result = context.format_result
# Plain format should not have JSON envelope structure
assert '"command"' not in result, "Plain output should not contain JSON envelope"
assert '"status"' not in result, "Plain output should not contain JSON envelope"
assert '"exit_code"' not in result, "Plain output should not contain JSON envelope"