diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e75eeee..b22872532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -633,6 +633,18 @@ `OperationalMetricKey` values with `MetricEntry` / `MetricCollector` for plan-level metrics. `TraceService` provides recording, querying, metric computation, and optional LangSmith forwarding when `LANGCHAIN_TRACING_V2=true`. (#500) +- Polished CLI help text, progress indicators, and error messages across core command files. + Extracted shared output renderers into `src/cleveragents/cli/renderers.py` with + `render_detail`, `render_list`, `render_error`, `render_success`, `render_warning`, and + `render_empty` functions parameterised by `ColumnSpec`/`FieldSpec` dataclasses, ensuring + structurally-similar outputs are identical by construction. Migrated an initial subset of + commands to the shared renderers and standardised `--format` (rich/plain/json/yaml) behaviour + for those commands with stable column names and ordering for list views. `plain` format + enforces ASCII-only output via `_ascii_safe()` in `formatting.py` for log pipeline + compatibility. Added unified error envelope (`error.code`/`error.message`/`error.details`) for + JSON/YAML error outputs. Includes CLI output contract documentation + (`docs/reference/cli_output.md`), 14 Behave scenarios in `features/cli_output_formats.feature`, + 2 Robot Framework test cases, and 11 ASV benchmarks in `benchmarks/cli_render_bench.py`. (#210) - Added `SafetyProfile` domain model with configurable safety constraints (allowed skill categories, sandbox/checkpoint requirements, human-approval flag, cost/retry limits) and integrated it into the `Action` model via `from_config`/`as_cli_dict`. Persistence backed diff --git a/benchmarks/cli_render_bench.py b/benchmarks/cli_render_bench.py new file mode 100644 index 000000000..69e9d293a --- /dev/null +++ b/benchmarks/cli_render_bench.py @@ -0,0 +1,189 @@ +"""ASV benchmarks for CLI output rendering overhead (issue #210). + +Measures render_detail, render_list, render_error, and render_success +across all six output formats. +""" + +from __future__ import annotations + +import importlib +import io +import sys +from pathlib import Path + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from rich.console import Console # noqa: E402 + +import cleveragents.cli.renderers as _renderers_mod # noqa: E402 +from cleveragents.cli.renderers import ( # noqa: E402 + ColumnSpec, + render_detail, + render_error, + render_list, + render_success, +) + +# --------------------------------------------------------------------------- +# Shared sample data +# --------------------------------------------------------------------------- + +_DETAIL_DATA = { + "name": "bench-item", + "count": 42, + "active": True, + "description": "A benchmark test item for measuring renderer overhead", +} + +_LIST_COLUMNS = [ + ColumnSpec(key="name", label="Name"), + ColumnSpec(key="value", label="Value"), + ColumnSpec(key="status", label="Status"), +] + +_LIST_ROWS_10 = [ + { + "name": f"item-{i}", + "value": i * 10, + "status": "active" if i % 2 == 0 else "inactive", + } + for i in range(10) +] + + +# --------------------------------------------------------------------------- +# Console buffer helpers +# --------------------------------------------------------------------------- + + +def _install_buffer_consoles() -> None: + """Replace module-level consoles with buffer-backed instances.""" + _renderers_mod._console = Console( + file=io.StringIO(), no_color=True, highlight=False + ) + _renderers_mod._err_console = Console( + file=io.StringIO(), no_color=True, highlight=False + ) + + +def _reset_buffers() -> None: + """Seek to start and truncate the console buffers between iterations.""" + for attr in ("_console", "_err_console"): + console = getattr(_renderers_mod, attr) + if console is not None and hasattr(console, "file"): + console.file.seek(0) + console.file.truncate() + + +# --------------------------------------------------------------------------- +# RenderDetailSuite +# --------------------------------------------------------------------------- + + +class RenderDetailSuite: + """Benchmark render_detail across formats.""" + + def setup(self) -> None: + _install_buffer_consoles() + + def teardown(self) -> None: + _renderers_mod._console = None + _renderers_mod._err_console = None + + def time_render_detail_json(self) -> None: + _reset_buffers() + render_detail(_DETAIL_DATA, fmt="json") + + def time_render_detail_yaml(self) -> None: + _reset_buffers() + render_detail(_DETAIL_DATA, fmt="yaml") + + def time_render_detail_plain(self) -> None: + _reset_buffers() + render_detail(_DETAIL_DATA, fmt="plain") + + def time_render_detail_rich(self) -> None: + _reset_buffers() + render_detail(_DETAIL_DATA, fmt="rich") + + +# --------------------------------------------------------------------------- +# RenderListSuite +# --------------------------------------------------------------------------- + + +class RenderListSuite: + """Benchmark render_list across formats.""" + + def setup(self) -> None: + _install_buffer_consoles() + + def teardown(self) -> None: + _renderers_mod._console = None + _renderers_mod._err_console = None + + def time_render_list_json_10_rows(self) -> None: + _reset_buffers() + render_list(_LIST_ROWS_10, columns=_LIST_COLUMNS, fmt="json") + + def time_render_list_plain_10_rows(self) -> None: + _reset_buffers() + render_list(_LIST_ROWS_10, columns=_LIST_COLUMNS, fmt="plain") + + def time_render_list_rich_10_rows(self) -> None: + _reset_buffers() + render_list(_LIST_ROWS_10, columns=_LIST_COLUMNS, fmt="rich") + + +# --------------------------------------------------------------------------- +# RenderErrorSuite +# --------------------------------------------------------------------------- + + +class RenderErrorSuite: + """Benchmark render_error across formats.""" + + def setup(self) -> None: + _install_buffer_consoles() + + def teardown(self) -> None: + _renderers_mod._console = None + _renderers_mod._err_console = None + + def time_render_error_rich(self) -> None: + _reset_buffers() + render_error("BenchError", "something went wrong", fmt="rich") + + def time_render_error_json(self) -> None: + _reset_buffers() + render_error("BenchError", "something went wrong", fmt="json") + + +# --------------------------------------------------------------------------- +# RenderSuccessSuite +# --------------------------------------------------------------------------- + + +class RenderSuccessSuite: + """Benchmark render_success across formats.""" + + def setup(self) -> None: + _install_buffer_consoles() + + def teardown(self) -> None: + _renderers_mod._console = None + _renderers_mod._err_console = None + + def time_render_success_rich(self) -> None: + _reset_buffers() + render_success("Benchmark complete", fmt="rich") + + def time_render_success_plain(self) -> None: + _reset_buffers() + render_success("Benchmark complete", fmt="plain") diff --git a/docs/reference/cli_output.md b/docs/reference/cli_output.md new file mode 100644 index 000000000..c427c391c --- /dev/null +++ b/docs/reference/cli_output.md @@ -0,0 +1,107 @@ +# CLI Output Contract + +## Overview + +All CleverAgents CLI commands use a shared output rendering layer that +guarantees structural consistency across commands and formats. + +## Supported Formats + +| Format | Description | ASCII-only | +|---------|-------------------------------------------|------------| +| `rich` | Full Rich terminal markup with colours | No | +| `color` | ANSI colour codes, no cursor movement | No | +| `table` | ASCII box-drawing table layout | Yes | +| `plain` | Key-value pairs, no markup | Yes | +| `json` | Indented JSON | Yes | +| `yaml` | YAML document | Yes | + +## Output Groups + +### Detail View (`render_detail`) + +Used by `show` and `create` commands. Renders a key-value panel. + +### List View (`render_list`) + +Used by all `list` commands. Renders a table with stable column order. +Columns are defined once per command in a `list[ColumnSpec]` and reused +for all formats, guaranteeing identical field names and ordering. + +### Error Messages (`render_error`) + +All errors use `render_error(label, message, recovery=...)`. + +For `json` / `yaml` formats, errors are wrapped in a unified envelope: + +```json +{ + "error": { + "code": "NotFound", + "message": "Resource not found", + "recovery": "Run 'agents resource add' first" + } +} +``` + +### Success Messages (`render_success`) + +Green checkmark for rich, `OK: ...` for plain. + +### Warning Messages (`render_warning`) + +Yellow text for rich, `WARNING: ...` for plain. + +### Empty Results (`render_empty`) + +Consistent "No found." with optional recovery hint. + +## Stable Field Names + +JSON and YAML outputs use the same field names as the domain model's +`_*_spec_dict()` helpers. Column ordering in list views matches the +`ColumnSpec` declarations. + +## JSON/YAML Error Envelope + +All error output in JSON/YAML format uses: + +```json +{ + "error": { + "code": "