Files
temp/features/steps/skill_cli_coverage_r3_steps.py
freemo a0df5a4cd0 fix(cli): wrap format_output() in spec-required JSON/YAML envelope across all CLI commands
Implements the spec-required JSON/YAML output envelope for all CLI commands
that use format_output(). The envelope structure is:

  {
    "command": "<command that was run>",
    "status": "ok" | "warn" | "error",
    "exit_code": 0,
    "data": { ... command-specific payload ... },
    "timing": { "duration_ms": 123 },
    "messages": [{ "level": "ok", "text": "..." }]
  }

Changes:
- Add _build_envelope() helper to construct the spec-required envelope
- Add optional command, status, exit_code, messages parameters to format_output()
- Wrap json/yaml output in the envelope; plain/table/rich/color unchanged
- Add timing measurement (duration_ms) to all json/yaml outputs
- Add new BDD feature file (cli_json_envelope.feature) with 14 scenarios
  testing envelope field presence, values, and data payload
- Update 14 existing step files to unwrap the envelope when checking
  specific data keys (backward-compatible via _unwrap_envelope() helper)

Closes #3431
2026-04-05 19:48:40 +00:00

620 lines
22 KiB
Python

# pyright: reportRedeclaration=false
"""Step definitions for skill_cli_coverage_r3.feature.
Targets remaining uncovered branches in
``cleveragents.cli.commands.skill``: tools --refresh, list/show
capability-summary fallback, refresh agent_skills discovery, refresh
errors, long description truncation, tools ValueError, and inline tool
capability display.
All step text uses the ``r3skill-`` prefix to avoid collisions with
existing step definitions.
"""
from __future__ import annotations
import json
import tempfile
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from cleveragents.cli.commands.skill import (
_get_skill_service,
_reset_skill_service,
)
from cleveragents.cli.commands.skill import app as skill_app
from cleveragents.domain.models.core.skill import (
Skill,
SkillAgentSource,
SkillInclude,
SkillInlineTool,
SkillMcpSource,
)
from cleveragents.domain.models.core.tool import ToolCapability, ToolSource
# ── helpers ─────────────────────────────────────────────────
def _make_skill(
name: str,
description: str = "test skill",
tool_refs: list[str] | None = None,
includes: list[SkillInclude] | None = None,
mcp_servers: list[SkillMcpSource] | None = None,
agent_skills: list[SkillAgentSource] | None = None,
anonymous_tools: list[SkillInlineTool] | None = None,
) -> Skill:
"""Create a Skill domain object with optional components."""
return Skill(
name=name,
description=description,
tool_refs=tool_refs or [],
includes=includes or [],
mcp_servers=mcp_servers or [],
agent_skills=agent_skills or [],
anonymous_tools=anonymous_tools or [],
)
def _register_skill(context: Context, skill: Skill) -> None:
"""Register a skill in the service with timestamps."""
from datetime import datetime
now = datetime.now()
context.r3_service._skills[skill.name] = skill
context.r3_service._created_at[skill.name] = now
context.r3_service._updated_at[skill.name] = now
def _write_yaml(content: str) -> str:
"""Write YAML content to a temp file and return the path."""
with tempfile.NamedTemporaryFile(
mode="w", suffix=".yaml", delete=False, prefix="r3skill_"
) as tmp:
tmp.write(content)
tmp.flush()
return tmp.name
# ── Stub classes for mocking ────────────────────────────────
@dataclass(slots=True)
class StubResolvedValue:
"""Stub for ConfigService.resolve return value."""
value: Any = None
@dataclass(slots=True)
class StubDiscoveredAgentSkill:
"""Stub for DiscoveredAgentSkill."""
name: str = "stub-agent"
description: str = "stub"
path: str = "/tmp/stub"
@dataclass(slots=True)
class StubDiscoveryResult:
"""Stub for DiscoveryResult."""
discovered: list[Any] = field(default_factory=list)
conflicts: list[Any] = field(default_factory=list)
errors: list[str] = field(default_factory=list)
# ── Background ──────────────────────────────────────────────
@given("r3skill- a fresh skill CLI service")
def step_r3_background(context: Context) -> None:
"""Reset module-level singleton and prepare runner/service."""
_reset_skill_service()
context.r3_runner = CliRunner()
context.r3_service = _get_skill_service()
context.r3_result = None
context.r3_temp_paths = [] # list[str]
context.r3_patches = [] # list[Any]
context.r3_second_temp = None # str | None
# ── Given steps ─────────────────────────────────────────────
@given('r3skill- a registered skill "{name}" with tool_refs')
def step_r3_register_skill_with_tool_refs(context: Context, name: str) -> None:
"""Directly insert a skill with builtin tool_refs into the service."""
skill = _make_skill(name=name, tool_refs=["builtin/read-file"])
_register_skill(context, skill)
@given('r3skill- a registered skill "{name}" with a description longer than 50 chars')
def step_r3_register_skill_long_desc(context: Context, name: str) -> None:
"""Register a skill with a description exceeding 50 characters."""
long_desc = "A" * 60
skill = _make_skill(name=name, description=long_desc, tool_refs=["builtin/echo"])
_register_skill(context, skill)
@given('r3skill- a registered skill "{name}" with inline tools having capabilities')
def step_r3_register_inline_caps(context: Context, name: str) -> None:
"""Register a skill with inline tools that have writes and checkpointable."""
inline = SkillInlineTool(
description="Inline with caps",
source=ToolSource.CUSTOM,
code="return True",
timeout=300,
capability=ToolCapability(
read_only=False,
writes=True,
checkpointable=True,
),
)
skill = _make_skill(name=name, anonymous_tools=[inline])
_register_skill(context, skill)
@given('r3skill- timestamps for "{name}" are removed')
def step_r3_remove_timestamps(context: Context, name: str) -> None:
"""Remove created_at and updated_at for a skill."""
context.r3_service._created_at.pop(name, None)
context.r3_service._updated_at.pop(name, None)
@given(
"r3skill- agent_skills_paths config resolves to a valid path with discovered skills"
)
def step_r3_agent_paths_valid(context: Context) -> None:
"""Set up mocks so refresh/tools --refresh discovers agent skills."""
stub_resolved = StubResolvedValue(value="/tmp/skills")
stub_discovery = StubDiscoveryResult(
discovered=[StubDiscoveredAgentSkill()],
errors=[],
)
p1 = patch(
"cleveragents.application.services.config_service.ConfigService",
)
mock_config_cls = p1.start()
mock_instance = MagicMock()
mock_instance.resolve.return_value = stub_resolved
mock_config_cls.return_value = mock_instance
context.r3_patches.append(p1)
p2 = patch(
"cleveragents.skills.discovery.parse_agent_skills_paths",
return_value=["/tmp/skills"],
)
p2.start()
context.r3_patches.append(p2)
p3 = patch(
"cleveragents.skills.discovery.discover_agent_skills",
return_value=stub_discovery,
)
p3.start()
context.r3_patches.append(p3)
@given("r3skill- agent_skills_paths config resolves to empty string")
def step_r3_agent_paths_empty(context: Context) -> None:
"""Set up mock so agent_skills_paths resolves to empty."""
stub_resolved = StubResolvedValue(value="")
p1 = patch(
"cleveragents.application.services.config_service.ConfigService",
)
mock_config_cls = p1.start()
mock_instance = MagicMock()
mock_instance.resolve.return_value = stub_resolved
mock_config_cls.return_value = mock_instance
context.r3_patches.append(p1)
@given("r3skill- agent_skills_paths config resolves with discovery errors")
def step_r3_agent_paths_errors(context: Context) -> None:
"""Set up mocks so discovery returns errors."""
stub_resolved = StubResolvedValue(value="/tmp/skills")
stub_discovery = StubDiscoveryResult(
discovered=[],
errors=["Error scanning /tmp/skills: permission denied"],
)
p1 = patch(
"cleveragents.application.services.config_service.ConfigService",
)
mock_config_cls = p1.start()
mock_instance = MagicMock()
mock_instance.resolve.return_value = stub_resolved
mock_config_cls.return_value = mock_instance
context.r3_patches.append(p1)
p2 = patch(
"cleveragents.skills.discovery.parse_agent_skills_paths",
return_value=["/tmp/skills"],
)
p2.start()
context.r3_patches.append(p2)
p3 = patch(
"cleveragents.skills.discovery.discover_agent_skills",
return_value=stub_discovery,
)
p3.start()
context.r3_patches.append(p3)
@given('r3skill- resolve_tools will raise ValueError for "{name}"')
def step_r3_resolve_tools_valueerror(context: Context, name: str) -> None:
"""Patch resolve_tools to raise ValueError for a specific skill."""
original_resolve = context.r3_service.resolve_tools
def patched_resolve(skill_name: str) -> Any:
"""Raise ValueError for targeted skill only."""
if skill_name == name:
raise ValueError(f"Cycle detected in {name}")
return original_resolve(skill_name)
p = patch.object(context.r3_service, "resolve_tools", side_effect=patched_resolve)
p.start()
context.r3_patches.append(p)
@given("r3skill- compute_capability_summary will raise ValueError")
def step_r3_cap_summary_valueerror(context: Context) -> None:
"""Patch compute_capability_summary to raise ValueError."""
p = patch.object(
context.r3_service,
"compute_capability_summary",
side_effect=ValueError("cap summary error"),
)
p.start()
context.r3_patches.append(p)
@given("r3skill- compute_capability_summary will raise KeyError")
def step_r3_cap_summary_keyerror(context: Context) -> None:
"""Patch compute_capability_summary to raise KeyError."""
p = patch.object(
context.r3_service,
"compute_capability_summary",
side_effect=KeyError("missing"),
)
p.start()
context.r3_patches.append(p)
@given("r3skill- list_skills will raise RuntimeError")
def step_r3_list_skills_runtime_error(context: Context) -> None:
"""Patch list_skills to raise RuntimeError."""
p = patch.object(
context.r3_service,
"list_skills",
side_effect=RuntimeError("unexpected error"),
)
p.start()
context.r3_patches.append(p)
@given('r3skill- a temp YAML for "{name}" that includes "{included}"')
def step_r3_temp_yaml_with_include(context: Context, name: str, included: str) -> None:
"""Create a temp YAML config that includes another skill."""
yaml_content = f"""\
name: {name}
description: Skill with include
tools:
- name: builtin/write-file
includes:
- name: {included}
"""
path = _write_yaml(yaml_content)
context.r3_temp_paths.append(path)
@given('r3skill- a temp YAML for "{name}" with tool_refs and MCP')
def step_r3_temp_yaml_tools_and_mcp(context: Context, name: str) -> None:
"""Create a temp YAML config with tool_refs and MCP servers."""
yaml_content = f"""\
name: {name}
description: Original skill with MCP
tools:
- name: builtin/read-file
mcp_servers:
- name: mcp-server-a
transport: stdio
tool_filter:
include:
- tool-x
"""
path = _write_yaml(yaml_content)
context.r3_temp_paths.append(path)
@given('r3skill- the skill "{name}" is pre-registered via add')
def step_r3_pre_register_via_add(context: Context, name: str) -> None:
"""Register a skill via the CLI add command."""
config_path = context.r3_temp_paths[-1]
result = context.r3_runner.invoke(
skill_app, ["add", "--config", config_path, "--format", "json"]
)
assert result.exit_code == 0, f"Pre-registration failed: {result.output}"
@given('r3skill- a second temp YAML for "{name}" with different includes and MCP')
def step_r3_second_yaml_diff_includes_mcp(context: Context, name: str) -> None:
"""Create a second YAML config with different includes and MCP."""
yaml_content = f"""\
name: {name}
description: Updated skill with changed includes and MCP
tools:
- name: builtin/write-file
includes:
- name: local/new-include
mcp_servers:
- name: mcp-server-b
transport: stdio
tool_filter:
include:
- tool-y
"""
path = _write_yaml(yaml_content)
context.r3_second_temp = path
# ── When steps ──────────────────────────────────────────────
@when('r3skill- I invoke tools "{name}" with --refresh')
def step_r3_invoke_tools_refresh(context: Context, name: str) -> None:
"""Invoke skill tools with --refresh flag."""
context.r3_result = context.r3_runner.invoke(
skill_app, ["tools", name, "--refresh"]
)
_stop_patches(context)
@when('r3skill- I invoke tools "{name}" in rich format')
def step_r3_invoke_tools_rich(context: Context, name: str) -> None:
"""Invoke skill tools in default rich format."""
context.r3_result = context.r3_runner.invoke(skill_app, ["tools", name])
_stop_patches(context)
@when('r3skill- I invoke list in format "{fmt}"')
def step_r3_invoke_list_fmt(context: Context, fmt: str) -> None:
"""Invoke skill list with a specified format."""
context.r3_result = context.r3_runner.invoke(skill_app, ["list", "--format", fmt])
_stop_patches(context)
@when("r3skill- I invoke list in rich format")
def step_r3_invoke_list_rich(context: Context) -> None:
"""Invoke skill list in default rich format."""
context.r3_result = context.r3_runner.invoke(skill_app, ["list"])
_stop_patches(context)
@when('r3skill- I invoke show "{name}" in format "{fmt}"')
def step_r3_invoke_show_fmt(context: Context, name: str, fmt: str) -> None:
"""Invoke skill show with a specified format."""
context.r3_result = context.r3_runner.invoke(
skill_app, ["show", name, "--format", fmt]
)
_stop_patches(context)
@when('r3skill- I invoke show "{name}" in rich format')
def step_r3_invoke_show_rich(context: Context, name: str) -> None:
"""Invoke skill show in default rich format."""
context.r3_result = context.r3_runner.invoke(skill_app, ["show", name])
_stop_patches(context)
@when('r3skill- I invoke refresh "{name}" in rich format')
def step_r3_invoke_refresh_rich(context: Context, name: str) -> None:
"""Invoke skill refresh for a single skill in rich format."""
context.r3_result = context.r3_runner.invoke(skill_app, ["refresh", name])
_stop_patches(context)
@when('r3skill- I invoke refresh "{name}" in format "{fmt}"')
def step_r3_invoke_refresh_fmt(context: Context, name: str, fmt: str) -> None:
"""Invoke skill refresh with a specified format."""
context.r3_result = context.r3_runner.invoke(
skill_app, ["refresh", name, "--format", fmt]
)
_stop_patches(context)
@when("r3skill- I invoke refresh --all in rich format")
def step_r3_invoke_refresh_all_rich(context: Context) -> None:
"""Invoke skill refresh --all in rich format."""
context.r3_result = context.r3_runner.invoke(skill_app, ["refresh", "--all"])
_stop_patches(context)
@when('r3skill- I invoke refresh --all in format "{fmt}"')
def step_r3_invoke_refresh_all_fmt(context: Context, fmt: str) -> None:
"""Invoke skill refresh --all with a specified format."""
context.r3_result = context.r3_runner.invoke(
skill_app, ["refresh", "--all", "--format", fmt]
)
_stop_patches(context)
@when("r3skill- I invoke add with the temp config in rich format")
def step_r3_invoke_add_rich(context: Context) -> None:
"""Invoke skill add with the most recent temp config."""
config_path = context.r3_temp_paths[-1]
context.r3_result = context.r3_runner.invoke(
skill_app, ["add", "--config", config_path]
)
_stop_patches(context)
@when("r3skill- I invoke add with the second config and --update in rich format")
def step_r3_invoke_add_update_rich(context: Context) -> None:
"""Invoke skill add --update with the second temp config."""
assert context.r3_second_temp is not None
context.r3_result = context.r3_runner.invoke(
skill_app, ["add", "--config", context.r3_second_temp, "--update"]
)
_stop_patches(context)
@when('r3skill- I invoke remove "{name}" without --yes and confirm')
def step_r3_invoke_remove_confirm(context: Context, name: str) -> None:
"""Invoke skill remove without --yes and type y to confirm."""
context.r3_result = context.r3_runner.invoke(
skill_app, ["remove", name], input="y\n"
)
_stop_patches(context)
# ── Then steps ──────────────────────────────────────────────
@then("r3skill- the CLI exit code should be 0")
def step_r3_exit_code_0(context: Context) -> None:
"""Assert CLI exited successfully."""
assert context.r3_result is not None
assert context.r3_result.exit_code == 0, (
f"Expected exit_code=0, got {context.r3_result.exit_code}\n"
f"Output: {context.r3_result.output}"
)
@then("r3skill- the CLI exit code should not be 0")
def step_r3_exit_code_not_0(context: Context) -> None:
"""Assert CLI exited with an error."""
assert context.r3_result is not None
assert context.r3_result.exit_code != 0, (
f"Expected non-zero exit code, got {context.r3_result.exit_code}\n"
f"Output: {context.r3_result.output}"
)
@then('r3skill- the output should contain "{text}"')
def step_r3_output_contains(context: Context, text: str) -> None:
"""Assert the CLI output contains the specified text."""
assert context.r3_result is not None
assert text in context.r3_result.output, (
f"Expected output to contain '{text}'\n"
f"Actual output: {context.r3_result.output}"
)
@then('r3skill- the output should not contain "{text}"')
def step_r3_output_not_contains(context: Context, text: str) -> None:
"""Assert the CLI output does not contain the specified text."""
assert context.r3_result is not None
assert text not in context.r3_result.output, (
f"Expected output NOT to contain '{text}'\n"
f"Actual output: {context.r3_result.output}"
)
_ENVELOPE_KEYS_R3 = {"command", "status", "exit_code", "data", "timing", "messages"}
def _unwrap_r3_envelope(parsed: Any) -> Any:
"""Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is."""
if isinstance(parsed, dict) and _ENVELOPE_KEYS_R3.issubset(parsed.keys()):
return parsed["data"]
return parsed
@then("r3skill- the output should be valid JSON")
def step_r3_output_valid_json(context: Context) -> None:
"""Assert the CLI output parses as valid JSON.
Stores the unwrapped ``data`` field in ``context.r3_parsed_json`` so
that downstream key-check steps work against the actual payload rather
than the spec envelope wrapper.
"""
assert context.r3_result is not None
try:
parsed = json.loads(context.r3_result.output)
except json.JSONDecodeError as e:
raise AssertionError(
f"Output is not valid JSON: {e}\nOutput: {context.r3_result.output}"
) from e
context.r3_parsed_json = _unwrap_r3_envelope(parsed)
@then('r3skill- the JSON output should have key "{key}"')
def step_r3_json_has_key(context: Context, key: str) -> None:
"""Assert a key is present in the parsed JSON dict."""
data: Any = context.r3_parsed_json
if isinstance(data, list):
assert len(data) > 0, "JSON list is empty"
assert key in data[0], (
f"Key '{key}' not found in first element: {list(data[0].keys())}"
)
else:
assert key in data, f"Key '{key}' not found in: {list(data.keys())}"
@then('r3skill- the JSON list first item should have key "{key}"')
def step_r3_json_list_first_has_key(context: Context, key: str) -> None:
"""Assert the first item in the JSON list has the given key."""
data: Any = context.r3_parsed_json
assert isinstance(data, list), f"Expected JSON list, got {type(data).__name__}"
assert len(data) > 0, "JSON list is empty"
assert key in data[0], (
f"Key '{key}' not found in first element: {list(data[0].keys())}"
)
@then('r3skill- the JSON list first item "{key}" should be null')
def step_r3_json_list_first_key_null(context: Context, key: str) -> None:
"""Assert the first item's key is null/None."""
data: Any = context.r3_parsed_json
assert isinstance(data, list), f"Expected JSON list, got {type(data).__name__}"
assert len(data) > 0, "JSON list is empty"
assert key in data[0], (
f"Key '{key}' not found in first element: {list(data[0].keys())}"
)
assert data[0][key] is None, f"Expected '{key}' to be null, got {data[0][key]}"
@then('r3skill- the JSON output key "{key}" should be null')
def step_r3_json_key_null(context: Context, key: str) -> None:
"""Assert a top-level JSON key is null/None."""
data: Any = context.r3_parsed_json
assert isinstance(data, dict), f"Expected JSON dict, got {type(data).__name__}"
assert key in data, f"Key '{key}' not found in: {list(data.keys())}"
assert data[key] is None, f"Expected '{key}' to be null, got {data[key]}"
@then("r3skill- the long description in the output should be truncated")
def step_r3_long_desc_truncated(context: Context) -> None:
"""Assert the output contains a truncated description (with ... or ellipsis)."""
assert context.r3_result is not None
output = context.r3_result.output
# The code adds "..." but Rich may render it as unicode ellipsis
has_dots = "..." in output or "\u2026" in output
assert has_dots, (
f"Expected truncated description (... or \u2026) in output\n"
f"Actual output: {output}"
)
# ── Cleanup helper ──────────────────────────────────────────
def _stop_patches(context: Context) -> None:
"""Stop all active patches after a When step."""
import contextlib
for p in getattr(context, "r3_patches", []):
with contextlib.suppress(RuntimeError):
p.stop()
context.r3_patches = []