Files
temp/features/steps/skill_cli_coverage_r2_steps.py
T
freemo a0df5a4cd0 fix(cli): wrap format_output() in spec-required JSON/YAML envelope across all CLI commands
Implements the spec-required JSON/YAML output envelope for all CLI commands
that use format_output(). The envelope structure is:

  {
    "command": "<command that was run>",
    "status": "ok" | "warn" | "error",
    "exit_code": 0,
    "data": { ... command-specific payload ... },
    "timing": { "duration_ms": 123 },
    "messages": [{ "level": "ok", "text": "..." }]
  }

Changes:
- Add _build_envelope() helper to construct the spec-required envelope
- Add optional command, status, exit_code, messages parameters to format_output()
- Wrap json/yaml output in the envelope; plain/table/rich/color unchanged
- Add timing measurement (duration_ms) to all json/yaml outputs
- Add new BDD feature file (cli_json_envelope.feature) with 14 scenarios
  testing envelope field presence, values, and data payload
- Update 14 existing step files to unwrap the envelope when checking
  specific data keys (backward-compatible via _unwrap_envelope() helper)

Closes #3431
2026-04-05 19:48:40 +00:00

473 lines
17 KiB
Python

"""Step definitions for skill_cli_coverage_r2.feature.
Targets partial branch coverage in
``cleveragents.cli.commands.skill`` (lines 74, 109, 112, 121, 155,
164, 169, and related conditional branches).
All step text uses the ``r2skill-`` prefix to avoid collisions with
existing step definitions in skill_cli_steps.py and
skill_cli_coverage_steps.py.
"""
from __future__ import annotations
import json
import tempfile
from typing import Any
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from cleveragents.cli.commands.skill import (
_get_skill_service,
_reset_skill_service,
)
from cleveragents.cli.commands.skill import app as skill_app
from cleveragents.domain.models.core.skill import (
Skill,
SkillAgentSource,
SkillInclude,
SkillInlineTool,
SkillMcpSource,
)
from cleveragents.domain.models.core.tool import ToolCapability, ToolSource
# ── helpers ─────────────────────────────────────────────────
def _make_skill(
name: str,
description: str = "test skill",
tool_refs: list[str] | None = None,
includes: list[SkillInclude] | None = None,
mcp_servers: list[SkillMcpSource] | None = None,
agent_skills: list[SkillAgentSource] | None = None,
anonymous_tools: list[SkillInlineTool] | None = None,
) -> Skill:
"""Create a Skill domain object with optional components."""
return Skill(
name=name,
description=description,
tool_refs=tool_refs or [],
includes=includes or [],
mcp_servers=mcp_servers or [],
agent_skills=agent_skills or [],
anonymous_tools=anonymous_tools or [],
)
def _write_yaml(content: str) -> str:
"""Write YAML content to a temp file and return the path."""
with tempfile.NamedTemporaryFile(
mode="w", suffix=".yaml", delete=False, prefix="r2skill_"
) as tmp:
tmp.write(content)
tmp.flush()
return tmp.name
# ── Background ──────────────────────────────────────────────
@given("r2skill- a reset skill CLI service")
def step_r2_background(context: Context) -> None:
"""Reset module-level singleton and prepare runner/service."""
_reset_skill_service()
context.r2_runner = CliRunner()
context.r2_service = _get_skill_service()
context.r2_result = None
context.r2_temp_paths = [] # list[str]
context.r2_second_temp = None # str | None
context.r2_service_a = None
context.r2_service_b = None
# ── Given steps ─────────────────────────────────────────────
@given('r2skill- a registered skill "{name}" with tool_refs')
def step_r2_register_skill_with_tool_refs(context: Context, name: str) -> None:
"""Directly insert a skill with builtin tool_refs into the service."""
skill = _make_skill(name=name, tool_refs=["builtin/read-file"])
context.r2_service._skills[skill.name] = skill
@given('r2skill- the timestamps for "{name}" are removed')
def step_r2_remove_timestamps(context: Context, name: str) -> None:
"""Remove created_at and updated_at for a skill to force None returns."""
context.r2_service._created_at.pop(name, None)
context.r2_service._updated_at.pop(name, None)
@given('r2skill- a registered skill "{name}" with MCP server but no tool list')
def step_r2_register_mcp_no_tools(context: Context, name: str) -> None:
"""Skill with an MCP server whose tools list is None."""
skill = _make_skill(
name=name,
mcp_servers=[SkillMcpSource(server="test-mcp-server", tools=None)],
)
context.r2_service._skills[skill.name] = skill
@given('r2skill- a temp YAML config for "{name}" with only tool_refs')
def step_r2_temp_yaml_tool_refs(context: Context, name: str) -> None:
"""Create a temp YAML config with only tool references."""
yaml_content = f"""\
name: {name}
description: A test skill with tool refs
tools:
- name: builtin/read-file
"""
path = _write_yaml(yaml_content)
context.r2_temp_paths.append(path)
@given('r2skill- a temp YAML config for "{name}" with no tools')
def step_r2_temp_yaml_no_tools(context: Context, name: str) -> None:
"""Create a temp YAML config with no tools/mcp/agent/inline at all."""
yaml_content = f"""\
name: {name}
description: A skill with absolutely no tool sources
"""
path = _write_yaml(yaml_content)
context.r2_temp_paths.append(path)
@given('r2skill- a temp YAML config for "{name}" with MCP and tools')
def step_r2_temp_yaml_mcp_with_tools(context: Context, name: str) -> None:
"""Create a YAML config for a skill with only MCP servers (no builtin tools)."""
yaml_content = f"""\
name: {name}
description: Skill with only MCP sources
mcp_servers:
- name: my-mcp
transport: stdio
tool_filter:
include:
- read
- write
"""
path = _write_yaml(yaml_content)
context.r2_temp_paths.append(path)
@given('r2skill- a temp YAML config for "{name}" with inline tools')
def step_r2_temp_yaml_inline(context: Context, name: str) -> None:
"""Create a YAML config with inline (custom) tools only."""
yaml_content = f"""\
name: {name}
description: Skill with inline tools
inline_tools:
- name: my-custom-tool
description: A custom tool
source: custom
code: "print('hello')"
"""
path = _write_yaml(yaml_content)
context.r2_temp_paths.append(path)
@given('r2skill- the skill "{name}" is already registered via add')
def step_r2_pre_register_via_add(context: Context, name: str) -> None:
"""Register a skill via the CLI add command (first invocation)."""
# Use the last temp path that was created
config_path = context.r2_temp_paths[-1]
result = context.r2_runner.invoke(
skill_app, ["add", "--config", config_path, "--format", "json"]
)
assert result.exit_code == 0, f"Pre-registration failed: {result.output}"
@given('r2skill- a second temp YAML config for "{name}" with different tools')
def step_r2_second_yaml_different_tools(context: Context, name: str) -> None:
"""Create a second YAML config for the same skill with different tools."""
yaml_content = f"""\
name: {name}
description: Updated skill with different tools
tools:
- name: builtin/write-file
- name: builtin/exec-command
"""
path = _write_yaml(yaml_content)
context.r2_second_temp = path
@given('r2skill- a duplicate temp YAML config for "{name}"')
def step_r2_duplicate_yaml(context: Context, name: str) -> None:
"""Create a duplicate YAML config for the same skill name."""
yaml_content = f"""\
name: {name}
description: Duplicate skill
tools:
- name: builtin/read-file
"""
path = _write_yaml(yaml_content)
context.r2_second_temp = path
@given('r2skill- a registered skill "{name}" that includes "{included}"')
def step_r2_register_with_include(context: Context, name: str, included: str) -> None:
"""Register a skill that includes another skill."""
skill = _make_skill(
name=name,
tool_refs=["builtin/exec-command"],
includes=[SkillInclude(name=included)],
)
context.r2_service._skills[skill.name] = skill
@given('r2skill- a registered skill "{name}" with anonymous inline tools')
def step_r2_register_inline_tools(context: Context, name: str) -> None:
"""Register a skill with anonymous inline tools."""
inline = SkillInlineTool(
description="An inline tool",
source=ToolSource.CUSTOM,
code="print('hello')",
capability=ToolCapability(read_only=True),
)
skill = _make_skill(
name=name,
anonymous_tools=[inline],
)
context.r2_service._skills[skill.name] = skill
@given('r2skill- a registered skill "{name}" with no tools at all')
def step_r2_register_no_tools(context: Context, name: str) -> None:
"""Register a skill with no tool sources whatsoever."""
skill = _make_skill(name=name)
context.r2_service._skills[skill.name] = skill
@given('r2skill- a registered skill "{name}" with MCP server and explicit tools')
def step_r2_register_mcp_explicit_tools(context: Context, name: str) -> None:
"""Register a skill with an MCP server that has explicit tool names."""
skill = _make_skill(
name=name,
mcp_servers=[SkillMcpSource(server="my-mcp-srv", tools=["tool-a", "tool-b"])],
)
context.r2_service._skills[skill.name] = skill
@given('r2skill- a temp YAML config for "{name}" with MCP servers')
def step_r2_temp_yaml_mcp_servers(context: Context, name: str) -> None:
"""Create a YAML config that defines MCP servers."""
yaml_content = f"""\
name: {name}
description: Skill with MCP servers
mcp_servers:
- name: mcp-panel-server
transport: stdio
tool_filter:
include:
- read-data
"""
path = _write_yaml(yaml_content)
context.r2_temp_paths.append(path)
# ── When steps ──────────────────────────────────────────────
@when("r2skill- I call _get_skill_service twice without resetting")
def step_r2_get_service_twice(context: Context) -> None:
"""Call _get_skill_service twice to exercise the cached path."""
# First call already done in Background via _get_skill_service()
# Do NOT reset — call again to hit the `if _service is None:` false branch.
context.r2_service_a = _get_skill_service()
context.r2_service_b = _get_skill_service()
@when('r2skill- I invoke show "{name}" with format "{fmt}"')
def step_r2_invoke_show(context: Context, name: str, fmt: str) -> None:
"""Invoke the skill show command."""
context.r2_result = context.r2_runner.invoke(
skill_app, ["show", name, "--format", fmt]
)
@when("r2skill- I invoke add with the temp config in rich format")
def step_r2_invoke_add_rich(context: Context) -> None:
"""Invoke skill add with the most recent temp config, rich format."""
config_path = context.r2_temp_paths[-1]
context.r2_result = context.r2_runner.invoke(
skill_app, ["add", "--config", config_path]
)
@when('r2skill- I invoke add with the temp config in format "{fmt}"')
def step_r2_invoke_add_fmt(context: Context, fmt: str) -> None:
"""Invoke skill add with the most recent temp config and a specified format."""
config_path = context.r2_temp_paths[-1]
context.r2_result = context.r2_runner.invoke(
skill_app, ["add", "--config", config_path, "--format", fmt]
)
@when("r2skill- I invoke add with the second config and --update in rich format")
def step_r2_invoke_add_update_rich(context: Context) -> None:
"""Invoke skill add --update with the second temp config."""
assert context.r2_second_temp is not None
context.r2_result = context.r2_runner.invoke(
skill_app, ["add", "--config", context.r2_second_temp, "--update"]
)
@when("r2skill- I invoke add with the duplicate config without update")
def step_r2_invoke_add_duplicate(context: Context) -> None:
"""Invoke skill add with a duplicate name and no --update flag."""
assert context.r2_second_temp is not None
context.r2_result = context.r2_runner.invoke(
skill_app, ["add", "--config", context.r2_second_temp]
)
@when("r2skill- I invoke list in rich format")
def step_r2_invoke_list_rich(context: Context) -> None:
"""Invoke skill list with rich format (the default)."""
context.r2_result = context.r2_runner.invoke(skill_app, ["list"])
@when('r2skill- I invoke list in format "{fmt}"')
def step_r2_invoke_list_fmt(context: Context, fmt: str) -> None:
"""Invoke skill list with a specified format."""
context.r2_result = context.r2_runner.invoke(skill_app, ["list", "--format", fmt])
@when('r2skill- I invoke remove "{name}" with --yes in rich format')
def step_r2_invoke_remove_yes_rich(context: Context, name: str) -> None:
"""Invoke skill remove with --yes in rich format."""
context.r2_result = context.r2_runner.invoke(skill_app, ["remove", name, "--yes"])
@when('r2skill- I invoke remove "{name}" with --yes in format "{fmt}"')
def step_r2_invoke_remove_yes_fmt(context: Context, name: str, fmt: str) -> None:
"""Invoke skill remove --yes with a specified format."""
context.r2_result = context.r2_runner.invoke(
skill_app, ["remove", name, "--yes", "--format", fmt]
)
@when('r2skill- I invoke tools "{name}" with format "{fmt}"')
def step_r2_invoke_tools_fmt(context: Context, name: str, fmt: str) -> None:
"""Invoke skill tools with a specified format."""
context.r2_result = context.r2_runner.invoke(
skill_app, ["tools", name, "--format", fmt]
)
# ── Then steps ──────────────────────────────────────────────
@then("r2skill- both calls return the same SkillService object")
def step_r2_assert_same_service(context: Context) -> None:
"""Assert the singleton returned the same object."""
assert context.r2_service_a is context.r2_service_b, (
"Expected the same SkillService instance but got different objects"
)
@then("r2skill- the CLI exit code should be 0")
def step_r2_exit_code_0(context: Context) -> None:
"""Assert CLI exited successfully."""
assert context.r2_result is not None
assert context.r2_result.exit_code == 0, (
f"Expected exit_code=0, got {context.r2_result.exit_code}\n"
f"Output: {context.r2_result.output}"
)
@then("r2skill- the CLI exit code should not be 0")
def step_r2_exit_code_not_0(context: Context) -> None:
"""Assert CLI exited with an error."""
assert context.r2_result is not None
assert context.r2_result.exit_code != 0, (
f"Expected non-zero exit code, got {context.r2_result.exit_code}\n"
f"Output: {context.r2_result.output}"
)
_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"}
def _unwrap_r2_envelope(parsed: Any) -> Any:
"""Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is."""
if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()):
return parsed["data"]
return parsed
@then("r2skill- the output should be valid JSON")
def step_r2_output_valid_json(context: Context) -> None:
"""Assert the CLI output parses as valid JSON.
Stores the unwrapped ``data`` field in ``context.r2_parsed_json`` so
that downstream key-check steps work against the actual payload rather
than the spec envelope wrapper.
"""
assert context.r2_result is not None
try:
parsed = json.loads(context.r2_result.output)
except json.JSONDecodeError as e:
raise AssertionError(
f"Output is not valid JSON: {e}\nOutput: {context.r2_result.output}"
) from e
context.r2_parsed_json = _unwrap_r2_envelope(parsed)
@then('r2skill- the JSON output should not contain key "{key}"')
def step_r2_json_no_key(context: Context, key: str) -> None:
"""Assert a key is absent from the parsed JSON dict."""
data: dict[str, Any] = context.r2_parsed_json
assert key not in data, (
f"Expected key '{key}' to be absent, but it was found in: {list(data.keys())}"
)
@then('r2skill- the JSON output should contain key "{key}"')
def step_r2_json_has_key(context: Context, key: str) -> None:
"""Assert a key is present in the parsed JSON dict."""
data: Any = context.r2_parsed_json
if isinstance(data, list):
# Check first element
assert len(data) > 0, "JSON list is empty"
assert key in data[0], (
f"Key '{key}' not found in first element: {list(data[0].keys())}"
)
else:
assert key in data, f"Key '{key}' not found in: {list(data.keys())}"
@then('r2skill- the output should contain "{text}"')
def step_r2_output_contains(context: Context, text: str) -> None:
"""Assert the CLI output contains the specified text."""
assert context.r2_result is not None
assert text in context.r2_result.output, (
f"Expected output to contain '{text}'\n"
f"Actual output: {context.r2_result.output}"
)
@then('r2skill- the output should not contain "{text}"')
def step_r2_output_not_contains(context: Context, text: str) -> None:
"""Assert the CLI output does not contain the specified text."""
assert context.r2_result is not None
assert text not in context.r2_result.output, (
f"Expected output NOT to contain '{text}'\n"
f"Actual output: {context.r2_result.output}"
)
@then('r2skill- the JSON tool list should contain source "{source}"')
def step_r2_json_tool_list_source(context: Context, source: str) -> None:
"""Assert the JSON tool list contains an entry with the given source."""
data: Any = context.r2_parsed_json
# For tools command, the JSON structure is {"skill_name": ..., "tools": [...], ...}
tools_list = data["tools"] if isinstance(data, dict) and "tools" in data else data
assert isinstance(tools_list, list), (
f"Expected tools to be a list, got {type(tools_list).__name__}"
)
sources = [entry.get("source", "") for entry in tools_list]
assert source in sources, f"Expected source '{source}' in tool list, got: {sources}"