forked from HAL9000/cleveragents-core
a0df5a4cd0
Implements the spec-required JSON/YAML output envelope for all CLI commands
that use format_output(). The envelope structure is:
{
"command": "<command that was run>",
"status": "ok" | "warn" | "error",
"exit_code": 0,
"data": { ... command-specific payload ... },
"timing": { "duration_ms": 123 },
"messages": [{ "level": "ok", "text": "..." }]
}
Changes:
- Add _build_envelope() helper to construct the spec-required envelope
- Add optional command, status, exit_code, messages parameters to format_output()
- Wrap json/yaml output in the envelope; plain/table/rich/color unchanged
- Add timing measurement (duration_ms) to all json/yaml outputs
- Add new BDD feature file (cli_json_envelope.feature) with 14 scenarios
testing envelope field presence, values, and data payload
- Update 14 existing step files to unwrap the envelope when checking
specific data keys (backward-compatible via _unwrap_envelope() helper)
Closes #3431
473 lines
17 KiB
Python
473 lines
17 KiB
Python
"""Step definitions for skill_cli_coverage_r2.feature.
|
|
|
|
Targets partial branch coverage in
|
|
``cleveragents.cli.commands.skill`` (lines 74, 109, 112, 121, 155,
|
|
164, 169, and related conditional branches).
|
|
|
|
All step text uses the ``r2skill-`` prefix to avoid collisions with
|
|
existing step definitions in skill_cli_steps.py and
|
|
skill_cli_coverage_steps.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import tempfile
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
from typer.testing import CliRunner
|
|
|
|
from cleveragents.cli.commands.skill import (
|
|
_get_skill_service,
|
|
_reset_skill_service,
|
|
)
|
|
from cleveragents.cli.commands.skill import app as skill_app
|
|
from cleveragents.domain.models.core.skill import (
|
|
Skill,
|
|
SkillAgentSource,
|
|
SkillInclude,
|
|
SkillInlineTool,
|
|
SkillMcpSource,
|
|
)
|
|
from cleveragents.domain.models.core.tool import ToolCapability, ToolSource
|
|
|
|
# ── helpers ─────────────────────────────────────────────────
|
|
|
|
|
|
def _make_skill(
|
|
name: str,
|
|
description: str = "test skill",
|
|
tool_refs: list[str] | None = None,
|
|
includes: list[SkillInclude] | None = None,
|
|
mcp_servers: list[SkillMcpSource] | None = None,
|
|
agent_skills: list[SkillAgentSource] | None = None,
|
|
anonymous_tools: list[SkillInlineTool] | None = None,
|
|
) -> Skill:
|
|
"""Create a Skill domain object with optional components."""
|
|
return Skill(
|
|
name=name,
|
|
description=description,
|
|
tool_refs=tool_refs or [],
|
|
includes=includes or [],
|
|
mcp_servers=mcp_servers or [],
|
|
agent_skills=agent_skills or [],
|
|
anonymous_tools=anonymous_tools or [],
|
|
)
|
|
|
|
|
|
def _write_yaml(content: str) -> str:
|
|
"""Write YAML content to a temp file and return the path."""
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w", suffix=".yaml", delete=False, prefix="r2skill_"
|
|
) as tmp:
|
|
tmp.write(content)
|
|
tmp.flush()
|
|
return tmp.name
|
|
|
|
|
|
# ── Background ──────────────────────────────────────────────
|
|
|
|
|
|
@given("r2skill- a reset skill CLI service")
|
|
def step_r2_background(context: Context) -> None:
|
|
"""Reset module-level singleton and prepare runner/service."""
|
|
_reset_skill_service()
|
|
context.r2_runner = CliRunner()
|
|
context.r2_service = _get_skill_service()
|
|
context.r2_result = None
|
|
context.r2_temp_paths = [] # list[str]
|
|
context.r2_second_temp = None # str | None
|
|
context.r2_service_a = None
|
|
context.r2_service_b = None
|
|
|
|
|
|
# ── Given steps ─────────────────────────────────────────────
|
|
|
|
|
|
@given('r2skill- a registered skill "{name}" with tool_refs')
|
|
def step_r2_register_skill_with_tool_refs(context: Context, name: str) -> None:
|
|
"""Directly insert a skill with builtin tool_refs into the service."""
|
|
skill = _make_skill(name=name, tool_refs=["builtin/read-file"])
|
|
context.r2_service._skills[skill.name] = skill
|
|
|
|
|
|
@given('r2skill- the timestamps for "{name}" are removed')
|
|
def step_r2_remove_timestamps(context: Context, name: str) -> None:
|
|
"""Remove created_at and updated_at for a skill to force None returns."""
|
|
context.r2_service._created_at.pop(name, None)
|
|
context.r2_service._updated_at.pop(name, None)
|
|
|
|
|
|
@given('r2skill- a registered skill "{name}" with MCP server but no tool list')
|
|
def step_r2_register_mcp_no_tools(context: Context, name: str) -> None:
|
|
"""Skill with an MCP server whose tools list is None."""
|
|
skill = _make_skill(
|
|
name=name,
|
|
mcp_servers=[SkillMcpSource(server="test-mcp-server", tools=None)],
|
|
)
|
|
context.r2_service._skills[skill.name] = skill
|
|
|
|
|
|
@given('r2skill- a temp YAML config for "{name}" with only tool_refs')
|
|
def step_r2_temp_yaml_tool_refs(context: Context, name: str) -> None:
|
|
"""Create a temp YAML config with only tool references."""
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: A test skill with tool refs
|
|
tools:
|
|
- name: builtin/read-file
|
|
"""
|
|
path = _write_yaml(yaml_content)
|
|
context.r2_temp_paths.append(path)
|
|
|
|
|
|
@given('r2skill- a temp YAML config for "{name}" with no tools')
|
|
def step_r2_temp_yaml_no_tools(context: Context, name: str) -> None:
|
|
"""Create a temp YAML config with no tools/mcp/agent/inline at all."""
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: A skill with absolutely no tool sources
|
|
"""
|
|
path = _write_yaml(yaml_content)
|
|
context.r2_temp_paths.append(path)
|
|
|
|
|
|
@given('r2skill- a temp YAML config for "{name}" with MCP and tools')
|
|
def step_r2_temp_yaml_mcp_with_tools(context: Context, name: str) -> None:
|
|
"""Create a YAML config for a skill with only MCP servers (no builtin tools)."""
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: Skill with only MCP sources
|
|
mcp_servers:
|
|
- name: my-mcp
|
|
transport: stdio
|
|
tool_filter:
|
|
include:
|
|
- read
|
|
- write
|
|
"""
|
|
path = _write_yaml(yaml_content)
|
|
context.r2_temp_paths.append(path)
|
|
|
|
|
|
@given('r2skill- a temp YAML config for "{name}" with inline tools')
|
|
def step_r2_temp_yaml_inline(context: Context, name: str) -> None:
|
|
"""Create a YAML config with inline (custom) tools only."""
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: Skill with inline tools
|
|
inline_tools:
|
|
- name: my-custom-tool
|
|
description: A custom tool
|
|
source: custom
|
|
code: "print('hello')"
|
|
"""
|
|
path = _write_yaml(yaml_content)
|
|
context.r2_temp_paths.append(path)
|
|
|
|
|
|
@given('r2skill- the skill "{name}" is already registered via add')
|
|
def step_r2_pre_register_via_add(context: Context, name: str) -> None:
|
|
"""Register a skill via the CLI add command (first invocation)."""
|
|
# Use the last temp path that was created
|
|
config_path = context.r2_temp_paths[-1]
|
|
result = context.r2_runner.invoke(
|
|
skill_app, ["add", "--config", config_path, "--format", "json"]
|
|
)
|
|
assert result.exit_code == 0, f"Pre-registration failed: {result.output}"
|
|
|
|
|
|
@given('r2skill- a second temp YAML config for "{name}" with different tools')
|
|
def step_r2_second_yaml_different_tools(context: Context, name: str) -> None:
|
|
"""Create a second YAML config for the same skill with different tools."""
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: Updated skill with different tools
|
|
tools:
|
|
- name: builtin/write-file
|
|
- name: builtin/exec-command
|
|
"""
|
|
path = _write_yaml(yaml_content)
|
|
context.r2_second_temp = path
|
|
|
|
|
|
@given('r2skill- a duplicate temp YAML config for "{name}"')
|
|
def step_r2_duplicate_yaml(context: Context, name: str) -> None:
|
|
"""Create a duplicate YAML config for the same skill name."""
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: Duplicate skill
|
|
tools:
|
|
- name: builtin/read-file
|
|
"""
|
|
path = _write_yaml(yaml_content)
|
|
context.r2_second_temp = path
|
|
|
|
|
|
@given('r2skill- a registered skill "{name}" that includes "{included}"')
|
|
def step_r2_register_with_include(context: Context, name: str, included: str) -> None:
|
|
"""Register a skill that includes another skill."""
|
|
skill = _make_skill(
|
|
name=name,
|
|
tool_refs=["builtin/exec-command"],
|
|
includes=[SkillInclude(name=included)],
|
|
)
|
|
context.r2_service._skills[skill.name] = skill
|
|
|
|
|
|
@given('r2skill- a registered skill "{name}" with anonymous inline tools')
|
|
def step_r2_register_inline_tools(context: Context, name: str) -> None:
|
|
"""Register a skill with anonymous inline tools."""
|
|
inline = SkillInlineTool(
|
|
description="An inline tool",
|
|
source=ToolSource.CUSTOM,
|
|
code="print('hello')",
|
|
capability=ToolCapability(read_only=True),
|
|
)
|
|
skill = _make_skill(
|
|
name=name,
|
|
anonymous_tools=[inline],
|
|
)
|
|
context.r2_service._skills[skill.name] = skill
|
|
|
|
|
|
@given('r2skill- a registered skill "{name}" with no tools at all')
|
|
def step_r2_register_no_tools(context: Context, name: str) -> None:
|
|
"""Register a skill with no tool sources whatsoever."""
|
|
skill = _make_skill(name=name)
|
|
context.r2_service._skills[skill.name] = skill
|
|
|
|
|
|
@given('r2skill- a registered skill "{name}" with MCP server and explicit tools')
|
|
def step_r2_register_mcp_explicit_tools(context: Context, name: str) -> None:
|
|
"""Register a skill with an MCP server that has explicit tool names."""
|
|
skill = _make_skill(
|
|
name=name,
|
|
mcp_servers=[SkillMcpSource(server="my-mcp-srv", tools=["tool-a", "tool-b"])],
|
|
)
|
|
context.r2_service._skills[skill.name] = skill
|
|
|
|
|
|
@given('r2skill- a temp YAML config for "{name}" with MCP servers')
|
|
def step_r2_temp_yaml_mcp_servers(context: Context, name: str) -> None:
|
|
"""Create a YAML config that defines MCP servers."""
|
|
yaml_content = f"""\
|
|
name: {name}
|
|
description: Skill with MCP servers
|
|
mcp_servers:
|
|
- name: mcp-panel-server
|
|
transport: stdio
|
|
tool_filter:
|
|
include:
|
|
- read-data
|
|
"""
|
|
path = _write_yaml(yaml_content)
|
|
context.r2_temp_paths.append(path)
|
|
|
|
|
|
# ── When steps ──────────────────────────────────────────────
|
|
|
|
|
|
@when("r2skill- I call _get_skill_service twice without resetting")
|
|
def step_r2_get_service_twice(context: Context) -> None:
|
|
"""Call _get_skill_service twice to exercise the cached path."""
|
|
# First call already done in Background via _get_skill_service()
|
|
# Do NOT reset — call again to hit the `if _service is None:` false branch.
|
|
context.r2_service_a = _get_skill_service()
|
|
context.r2_service_b = _get_skill_service()
|
|
|
|
|
|
@when('r2skill- I invoke show "{name}" with format "{fmt}"')
|
|
def step_r2_invoke_show(context: Context, name: str, fmt: str) -> None:
|
|
"""Invoke the skill show command."""
|
|
context.r2_result = context.r2_runner.invoke(
|
|
skill_app, ["show", name, "--format", fmt]
|
|
)
|
|
|
|
|
|
@when("r2skill- I invoke add with the temp config in rich format")
|
|
def step_r2_invoke_add_rich(context: Context) -> None:
|
|
"""Invoke skill add with the most recent temp config, rich format."""
|
|
config_path = context.r2_temp_paths[-1]
|
|
context.r2_result = context.r2_runner.invoke(
|
|
skill_app, ["add", "--config", config_path]
|
|
)
|
|
|
|
|
|
@when('r2skill- I invoke add with the temp config in format "{fmt}"')
|
|
def step_r2_invoke_add_fmt(context: Context, fmt: str) -> None:
|
|
"""Invoke skill add with the most recent temp config and a specified format."""
|
|
config_path = context.r2_temp_paths[-1]
|
|
context.r2_result = context.r2_runner.invoke(
|
|
skill_app, ["add", "--config", config_path, "--format", fmt]
|
|
)
|
|
|
|
|
|
@when("r2skill- I invoke add with the second config and --update in rich format")
|
|
def step_r2_invoke_add_update_rich(context: Context) -> None:
|
|
"""Invoke skill add --update with the second temp config."""
|
|
assert context.r2_second_temp is not None
|
|
context.r2_result = context.r2_runner.invoke(
|
|
skill_app, ["add", "--config", context.r2_second_temp, "--update"]
|
|
)
|
|
|
|
|
|
@when("r2skill- I invoke add with the duplicate config without update")
|
|
def step_r2_invoke_add_duplicate(context: Context) -> None:
|
|
"""Invoke skill add with a duplicate name and no --update flag."""
|
|
assert context.r2_second_temp is not None
|
|
context.r2_result = context.r2_runner.invoke(
|
|
skill_app, ["add", "--config", context.r2_second_temp]
|
|
)
|
|
|
|
|
|
@when("r2skill- I invoke list in rich format")
|
|
def step_r2_invoke_list_rich(context: Context) -> None:
|
|
"""Invoke skill list with rich format (the default)."""
|
|
context.r2_result = context.r2_runner.invoke(skill_app, ["list"])
|
|
|
|
|
|
@when('r2skill- I invoke list in format "{fmt}"')
|
|
def step_r2_invoke_list_fmt(context: Context, fmt: str) -> None:
|
|
"""Invoke skill list with a specified format."""
|
|
context.r2_result = context.r2_runner.invoke(skill_app, ["list", "--format", fmt])
|
|
|
|
|
|
@when('r2skill- I invoke remove "{name}" with --yes in rich format')
|
|
def step_r2_invoke_remove_yes_rich(context: Context, name: str) -> None:
|
|
"""Invoke skill remove with --yes in rich format."""
|
|
context.r2_result = context.r2_runner.invoke(skill_app, ["remove", name, "--yes"])
|
|
|
|
|
|
@when('r2skill- I invoke remove "{name}" with --yes in format "{fmt}"')
|
|
def step_r2_invoke_remove_yes_fmt(context: Context, name: str, fmt: str) -> None:
|
|
"""Invoke skill remove --yes with a specified format."""
|
|
context.r2_result = context.r2_runner.invoke(
|
|
skill_app, ["remove", name, "--yes", "--format", fmt]
|
|
)
|
|
|
|
|
|
@when('r2skill- I invoke tools "{name}" with format "{fmt}"')
|
|
def step_r2_invoke_tools_fmt(context: Context, name: str, fmt: str) -> None:
|
|
"""Invoke skill tools with a specified format."""
|
|
context.r2_result = context.r2_runner.invoke(
|
|
skill_app, ["tools", name, "--format", fmt]
|
|
)
|
|
|
|
|
|
# ── Then steps ──────────────────────────────────────────────
|
|
|
|
|
|
@then("r2skill- both calls return the same SkillService object")
|
|
def step_r2_assert_same_service(context: Context) -> None:
|
|
"""Assert the singleton returned the same object."""
|
|
assert context.r2_service_a is context.r2_service_b, (
|
|
"Expected the same SkillService instance but got different objects"
|
|
)
|
|
|
|
|
|
@then("r2skill- the CLI exit code should be 0")
|
|
def step_r2_exit_code_0(context: Context) -> None:
|
|
"""Assert CLI exited successfully."""
|
|
assert context.r2_result is not None
|
|
assert context.r2_result.exit_code == 0, (
|
|
f"Expected exit_code=0, got {context.r2_result.exit_code}\n"
|
|
f"Output: {context.r2_result.output}"
|
|
)
|
|
|
|
|
|
@then("r2skill- the CLI exit code should not be 0")
|
|
def step_r2_exit_code_not_0(context: Context) -> None:
|
|
"""Assert CLI exited with an error."""
|
|
assert context.r2_result is not None
|
|
assert context.r2_result.exit_code != 0, (
|
|
f"Expected non-zero exit code, got {context.r2_result.exit_code}\n"
|
|
f"Output: {context.r2_result.output}"
|
|
)
|
|
|
|
|
|
_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"}
|
|
|
|
|
|
def _unwrap_r2_envelope(parsed: Any) -> Any:
|
|
"""Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is."""
|
|
if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()):
|
|
return parsed["data"]
|
|
return parsed
|
|
|
|
|
|
@then("r2skill- the output should be valid JSON")
|
|
def step_r2_output_valid_json(context: Context) -> None:
|
|
"""Assert the CLI output parses as valid JSON.
|
|
|
|
Stores the unwrapped ``data`` field in ``context.r2_parsed_json`` so
|
|
that downstream key-check steps work against the actual payload rather
|
|
than the spec envelope wrapper.
|
|
"""
|
|
assert context.r2_result is not None
|
|
try:
|
|
parsed = json.loads(context.r2_result.output)
|
|
except json.JSONDecodeError as e:
|
|
raise AssertionError(
|
|
f"Output is not valid JSON: {e}\nOutput: {context.r2_result.output}"
|
|
) from e
|
|
context.r2_parsed_json = _unwrap_r2_envelope(parsed)
|
|
|
|
|
|
@then('r2skill- the JSON output should not contain key "{key}"')
|
|
def step_r2_json_no_key(context: Context, key: str) -> None:
|
|
"""Assert a key is absent from the parsed JSON dict."""
|
|
data: dict[str, Any] = context.r2_parsed_json
|
|
assert key not in data, (
|
|
f"Expected key '{key}' to be absent, but it was found in: {list(data.keys())}"
|
|
)
|
|
|
|
|
|
@then('r2skill- the JSON output should contain key "{key}"')
|
|
def step_r2_json_has_key(context: Context, key: str) -> None:
|
|
"""Assert a key is present in the parsed JSON dict."""
|
|
data: Any = context.r2_parsed_json
|
|
if isinstance(data, list):
|
|
# Check first element
|
|
assert len(data) > 0, "JSON list is empty"
|
|
assert key in data[0], (
|
|
f"Key '{key}' not found in first element: {list(data[0].keys())}"
|
|
)
|
|
else:
|
|
assert key in data, f"Key '{key}' not found in: {list(data.keys())}"
|
|
|
|
|
|
@then('r2skill- the output should contain "{text}"')
|
|
def step_r2_output_contains(context: Context, text: str) -> None:
|
|
"""Assert the CLI output contains the specified text."""
|
|
assert context.r2_result is not None
|
|
assert text in context.r2_result.output, (
|
|
f"Expected output to contain '{text}'\n"
|
|
f"Actual output: {context.r2_result.output}"
|
|
)
|
|
|
|
|
|
@then('r2skill- the output should not contain "{text}"')
|
|
def step_r2_output_not_contains(context: Context, text: str) -> None:
|
|
"""Assert the CLI output does not contain the specified text."""
|
|
assert context.r2_result is not None
|
|
assert text not in context.r2_result.output, (
|
|
f"Expected output NOT to contain '{text}'\n"
|
|
f"Actual output: {context.r2_result.output}"
|
|
)
|
|
|
|
|
|
@then('r2skill- the JSON tool list should contain source "{source}"')
|
|
def step_r2_json_tool_list_source(context: Context, source: str) -> None:
|
|
"""Assert the JSON tool list contains an entry with the given source."""
|
|
data: Any = context.r2_parsed_json
|
|
# For tools command, the JSON structure is {"skill_name": ..., "tools": [...], ...}
|
|
tools_list = data["tools"] if isinstance(data, dict) and "tools" in data else data
|
|
assert isinstance(tools_list, list), (
|
|
f"Expected tools to be a list, got {type(tools_list).__name__}"
|
|
)
|
|
sources = [entry.get("source", "") for entry in tools_list]
|
|
assert source in sources, f"Expected source '{source}' in tool list, got: {sources}"
|