Files
cleveragents-core/features/steps/tool_cli_coverage_steps.py

804 lines
26 KiB
Python

"""Step definitions for the Tool and Validation CLI coverage feature.
These steps exercise additional code paths not covered by the main
``tool_cli.feature`` scenarios: ``as_cli_dict`` branch, lifecycle / slot /
validation-mode rich rendering, dict without namespace, wraps / transform
fields, ValidationError handling, plan_id attach, and confirmation-decline
paths.
"""
from __future__ import annotations
import os
import tempfile
from collections import OrderedDict
from typing import Any
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from cleveragents.cli.commands.tool import app as tool_app
from cleveragents.cli.commands.validation import app as validation_app
from cleveragents.core.exceptions import CleverAgentsError, ValidationError
_runner = CliRunner()
def _patch_tool_svc(context: Context) -> Any:
return patch(
"cleveragents.cli.commands.tool._get_tool_registry_service",
return_value=context.mock_service,
)
def _patch_val_svc(context: Context) -> Any:
return patch(
"cleveragents.cli.commands.validation._get_tool_registry_service",
return_value=context.mock_service,
)
_TOOL_YAML = """\
name: local/test-tool
description: A test tool
source: custom
code: |
def run(inputs):
return {"result": True}
"""
_VALIDATION_YAML = """\
name: local/test-val
description: A test validation
source: custom
mode: required
code: |
def run(inputs):
return {"passed": True}
"""
# -----------------------------------------------------------------------
# Background
# -----------------------------------------------------------------------
@given("a tool CLI coverage runner with mocks")
def step_coverage_runner(context: Context) -> None:
context.runner = _runner
context.tool_result = None
context.validation_result = None
context.mock_service = MagicMock()
context.tool_yaml_path = None
context.validation_yaml_path = None
# -----------------------------------------------------------------------
# Given - tools with special attributes
# -----------------------------------------------------------------------
def _make_cli_dict_tool(name: str = "local/domain-tool") -> MagicMock:
"""Create a mock tool object that has an ``as_cli_dict`` method."""
ns, sn = name.split("/", 1)
mock = MagicMock()
mock.name = name
mock.as_cli_dict.return_value = OrderedDict(
[
("name", name),
("description", "Domain model tool"),
("source", "custom"),
("tool_type", "tool"),
("namespace", ns),
("short_name", sn),
("capability", {"read_only": False}),
("timeout", 120),
]
)
return mock
@given("a tool with as_cli_dict method exists")
def step_tool_with_as_cli_dict(context: Context) -> None:
mock_tool = _make_cli_dict_tool()
context.mock_service.get_tool.return_value = mock_tool
@given("a tool with lifecycle hooks exists")
def step_tool_with_lifecycle(context: Context) -> None:
mock_tool = MagicMock()
mock_tool.name = "local/lifecycle-tool"
def _cli_dict() -> OrderedDict[str, Any]:
return OrderedDict(
[
("name", "local/lifecycle-tool"),
("description", "Tool with lifecycle"),
("source", "custom"),
("tool_type", "tool"),
("namespace", "local"),
("short_name", "lifecycle-tool"),
("capability", {}),
("timeout", 60),
(
"lifecycle",
{"on_start": "setup()", "on_stop": "teardown()"},
),
]
)
mock_tool.as_cli_dict = _cli_dict
context.mock_service.get_tool.return_value = mock_tool
@given("a tool with resource slots exists")
def step_tool_with_slots(context: Context) -> None:
mock_tool = MagicMock()
mock_tool.name = "local/slotted-tool"
def _cli_dict() -> OrderedDict[str, Any]:
return OrderedDict(
[
("name", "local/slotted-tool"),
("description", "Tool with resource slots"),
("source", "custom"),
("tool_type", "tool"),
("namespace", "local"),
("short_name", "slotted-tool"),
("capability", {}),
("timeout", 60),
(
"resource_slots",
[
{
"name": "db",
"resource_type": "database",
"access": "readwrite",
}
],
),
]
)
mock_tool.as_cli_dict = _cli_dict
context.mock_service.get_tool.return_value = mock_tool
@given("a tool with validation mode exists")
def step_tool_with_validation_mode(context: Context) -> None:
mock_tool = MagicMock()
mock_tool.name = "local/val-mode-tool"
def _cli_dict() -> OrderedDict[str, Any]:
return OrderedDict(
[
("name", "local/val-mode-tool"),
("description", "Tool with validation mode"),
("source", "custom"),
("tool_type", "validation"),
("namespace", "local"),
("short_name", "val-mode-tool"),
("capability", {}),
("timeout", 60),
("mode", "required"),
]
)
mock_tool.as_cli_dict = _cli_dict
context.mock_service.get_tool.return_value = mock_tool
@given("a tool with wraps field exists")
def step_tool_with_wraps(context: Context) -> None:
mock_tool = MagicMock()
mock_tool.name = "local/wrap-tool"
def _cli_dict() -> OrderedDict[str, Any]:
return OrderedDict(
[
("name", "local/wrap-tool"),
("description", "Tool with wraps"),
("source", "custom"),
("tool_type", "tool"),
("namespace", "local"),
("short_name", "wrap-tool"),
("capability", {}),
("timeout", 60),
("wraps", "local/inner-tool"),
]
)
mock_tool.as_cli_dict = _cli_dict
context.mock_service.get_tool.return_value = mock_tool
# -----------------------------------------------------------------------
# Given - tool list edge cases
# -----------------------------------------------------------------------
@given("a tool list returns minimal objects")
def step_tool_list_minimal(context: Context) -> None:
"""Return items that are plain strings (not dicts) so the fallback
``OrderedDict({"name": str(tool)})`` path in ``_tool_spec_dict`` runs."""
context.mock_service.list_tools.return_value = ["raw-string-tool"]
# -----------------------------------------------------------------------
# Given - validation with wraps/transform
# -----------------------------------------------------------------------
@given("a validation with wraps and transform exists")
def step_validation_wraps_transform(context: Context) -> None:
result = {
"name": "local/wrapped-val",
"description": "Validation with wraps and transform",
"source": "custom",
"tool_type": "validation",
"mode": "required",
"wraps": "local/inner-tool",
"transform": "uppercase",
}
context.mock_service.register_tool.return_value = result
# -----------------------------------------------------------------------
# Given - object-like attachment
# -----------------------------------------------------------------------
@given("an object-like attachment from service")
def step_object_like_attachment(context: Context) -> None:
mock_att = MagicMock()
mock_att.attachment_id = "att-obj-001"
mock_att.validation_name = "local/v1"
mock_att.resource_id = "res/x"
mock_att.mode = "required"
mock_att.project_name = None
mock_att.plan_id = None
mock_att.created_at = "2026-02-01T00:00:00"
context.mock_service.attach_validation.return_value = mock_att
# -----------------------------------------------------------------------
# Given - tool add yaml config
# -----------------------------------------------------------------------
@given("a tool add yaml config exists")
def step_tool_add_yaml(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(_TOOL_YAML)
context.tool_yaml_path = path
ns, sn = "local", "test-tool"
context.mock_service.register_tool.return_value = {
"name": f"{ns}/{sn}",
"description": "A test tool",
"source": "custom",
"tool_type": "tool",
"namespace": ns,
"short_name": sn,
"capability": {},
"timeout": 300,
}
@given("tool service raises validation error on register")
def step_tool_svc_validation_error(context: Context) -> None:
context.mock_service.register_tool.side_effect = ValidationError(
"Invalid tool config"
)
# -----------------------------------------------------------------------
# Given - validation add yaml config
# -----------------------------------------------------------------------
@given("a validation add yaml config exists")
def step_validation_add_yaml(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(_VALIDATION_YAML)
context.validation_yaml_path = path
context.mock_service.register_tool.return_value = {
"name": "local/test-val",
"description": "A test validation",
"source": "custom",
"tool_type": "validation",
"mode": "required",
}
# -----------------------------------------------------------------------
# Given - CleverAgentsError on list
# -----------------------------------------------------------------------
@given("tool service raises CleverAgentsError on list")
def step_tool_svc_list_error(context: Context) -> None:
context.mock_service.list_tools.side_effect = CleverAgentsError(
"Database unavailable"
)
# -----------------------------------------------------------------------
# Given - attachment service returns attachment (for plan-id scenario)
# -----------------------------------------------------------------------
@given("an attachment service returns attachment")
def step_attachment_svc_returns(context: Context) -> None:
context.mock_service.attach_validation.return_value = {
"attachment_id": "att-plan-001",
"validation_name": "local/val",
"resource_id": "res/abc",
"mode": "required",
"project_name": None,
"plan_id": "plan-123",
"created_at": "2026-02-01T00:00:00",
}
# -----------------------------------------------------------------------
# Given - dict tool without explicit namespace
# -----------------------------------------------------------------------
@given("a dict tool without explicit namespace")
def step_dict_tool_no_namespace(context: Context) -> None:
context.mock_service.get_tool.return_value = {
"name": "other/thing",
"description": "No explicit namespace",
"source": "custom",
"tool_type": "tool",
"capability": {},
"timeout": 60,
}
# -----------------------------------------------------------------------
# Given - dict validation with wraps/transform
# -----------------------------------------------------------------------
@given("a dict validation with wraps and transform")
def step_dict_validation_wraps_transform(context: Context) -> None:
result = {
"name": "local/wrapped-val",
"description": "Dict val with wraps",
"source": "custom",
"tool_type": "validation",
"mode": "required",
"wraps": "local/inner",
"transform": "lowercase",
}
context.mock_service.register_tool.return_value = result
# -----------------------------------------------------------------------
# Given - detach / remove helpers
# -----------------------------------------------------------------------
@given("an attachment service returns true on detach")
def step_attachment_svc_detach_true(context: Context) -> None:
context.mock_service.detach_validation.return_value = True
@given("a tool exists for removal")
def step_tool_exists_for_removal(context: Context) -> None:
context.mock_service.get_tool.return_value = {
"name": "local/rm-tool",
"description": "Removable",
"source": "custom",
"tool_type": "tool",
"namespace": "local",
"short_name": "rm-tool",
"capability": {},
"timeout": 60,
}
context.mock_service.remove_tool.return_value = True
# -----------------------------------------------------------------------
# When steps - tool show
# -----------------------------------------------------------------------
@when('I run tool show for "{name}"')
def step_run_tool_show_coverage(context: Context, name: str) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(tool_app, ["show", name])
@when('I run tool show formatted "{name}" using fmt "{fmt}"')
def step_run_tool_show_fmt(context: Context, name: str, fmt: str) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(tool_app, ["show", name, "--format", fmt])
@when("I run tool add from config")
def step_run_tool_add_config(context: Context) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(
tool_app, ["add", "--config", context.tool_yaml_path]
)
# -----------------------------------------------------------------------
# When steps - tool list
# -----------------------------------------------------------------------
@when('I run tool list with fmt "{fmt}"')
def step_run_tool_list_cov_fmt(context: Context, fmt: str) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(tool_app, ["list", "--format", fmt])
@when('I run tool list with source "{src}"')
def step_run_tool_list_cov_source(context: Context, src: str) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(tool_app, ["list", "--source", src])
# -----------------------------------------------------------------------
# When steps - tool add with format
# -----------------------------------------------------------------------
@when('I run tool add with fmt "{fmt}"')
def step_run_tool_add_cov_fmt(context: Context, fmt: str) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(
tool_app,
["add", "--config", context.tool_yaml_path, "--format", fmt],
)
# -----------------------------------------------------------------------
# When steps - validation add
# -----------------------------------------------------------------------
@when('I run validation add with fmt "{fmt}"')
def step_run_validation_add_fmt(context: Context, fmt: str) -> None:
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app,
["add", "--config", context.validation_yaml_path, "--format", fmt],
)
@when('I run validation show via add with "{name}"')
def step_run_validation_show_via_add(context: Context, name: str) -> None:
"""Register validation (add) and capture the output which exercises
``_print_validation`` with wraps/transform fields."""
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(
f"name: {name}\n"
"description: Wrapped val\n"
"source: custom\n"
"mode: required\n"
"code: |\n def run(i): return True\n"
)
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app, ["add", "--config", path]
)
@when('I run validation show via add for "{name}" with fmt "{fmt}"')
def step_run_validation_add_for_name_fmt(context: Context, name: str, fmt: str) -> None:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(
f"name: {name}\n"
"description: Dict val\n"
"source: custom\n"
"mode: required\n"
"code: |\n def run(i): return True\n"
)
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app,
["add", "--config", path, "--format", fmt],
)
# -----------------------------------------------------------------------
# When steps - validation attach
# -----------------------------------------------------------------------
@when('I run validation attach "{resource}" "{val_name}"')
def step_run_validation_attach_cov(
context: Context, resource: str, val_name: str
) -> None:
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app, ["attach", resource, val_name]
)
@when('I run validation attach with plan "{plan}"')
def step_run_validation_attach_plan(context: Context, plan: str) -> None:
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app,
["attach", "res/abc", "local/val", "--plan", plan],
)
# -----------------------------------------------------------------------
# When steps - declining confirmation
# -----------------------------------------------------------------------
@when('I run validation detach "{att_id}" declining confirmation')
def step_run_validation_detach_decline(context: Context, att_id: str) -> None:
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app, ["detach", att_id], input="n\n"
)
@when('I run tool remove "{name}" declining confirmation')
def step_run_tool_remove_decline(context: Context, name: str) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(tool_app, ["remove", name], input="n\n")
# -----------------------------------------------------------------------
# Then steps
# -----------------------------------------------------------------------
@then("the tool show output should succeed")
def step_tool_show_succeed(context: Context) -> None:
assert context.tool_result is not None
assert context.tool_result.exit_code == 0, (
f"Expected exit 0, got {context.tool_result.exit_code}. "
f"Output: {context.tool_result.output}"
)
@then("the tool list result should succeed")
def step_tool_list_succeed(context: Context) -> None:
assert context.tool_result is not None
assert context.tool_result.exit_code == 0, (
f"Expected exit 0, got {context.tool_result.exit_code}. "
f"Output: {context.tool_result.output}"
)
@then("the tool add result should succeed")
def step_tool_add_succeed(context: Context) -> None:
assert context.tool_result is not None
assert context.tool_result.exit_code == 0, (
f"Expected exit 0, got {context.tool_result.exit_code}. "
f"Output: {context.tool_result.output}"
)
@then("the tool add result should abort")
def step_tool_add_abort(context: Context) -> None:
assert context.tool_result is not None
assert context.tool_result.exit_code != 0, (
f"Expected non-zero exit, got {context.tool_result.exit_code}. "
f"Output: {context.tool_result.output}"
)
@then("the validation add result should succeed")
def step_validation_add_succeed(context: Context) -> None:
assert context.validation_result is not None
assert context.validation_result.exit_code == 0, (
f"Expected exit 0, got {context.validation_result.exit_code}. "
f"Output: {context.validation_result.output}"
)
@then("the validation add result should abort")
def step_validation_add_abort(context: Context) -> None:
assert context.validation_result is not None
assert context.validation_result.exit_code != 0, (
f"Expected non-zero exit, got {context.validation_result.exit_code}. "
f"Output: {context.validation_result.output}"
)
@then("the validation attach result should succeed")
def step_validation_attach_succeed(context: Context) -> None:
assert context.validation_result is not None
assert context.validation_result.exit_code == 0, (
f"Expected exit 0, got {context.validation_result.exit_code}. "
f"Output: {context.validation_result.output}"
)
@then("the tool list result should abort")
def step_tool_list_abort(context: Context) -> None:
assert context.tool_result is not None
assert context.tool_result.exit_code != 0, (
f"Expected non-zero exit, got {context.tool_result.exit_code}. "
f"Output: {context.tool_result.output}"
)
@then("the validation detach result should abort")
def step_validation_detach_abort(context: Context) -> None:
assert context.validation_result is not None
assert context.validation_result.exit_code != 0, (
f"Expected non-zero exit, got {context.validation_result.exit_code}. "
f"Output: {context.validation_result.output}"
)
@then("the tool remove result should abort")
def step_tool_remove_abort(context: Context) -> None:
assert context.tool_result is not None
assert context.tool_result.exit_code != 0, (
f"Expected non-zero exit, got {context.tool_result.exit_code}. "
f"Output: {context.tool_result.output}"
)
# -----------------------------------------------------------------------
# Additional coverage Given steps
# -----------------------------------------------------------------------
@given("a dict tool with mode and wraps")
def step_dict_tool_mode_wraps(context: Context) -> None:
"""Dict tool with ``mode`` and ``wraps`` keys to cover lines 130/132."""
context.mock_service.get_tool.return_value = {
"name": "local/dict-mw",
"description": "Dict with mode+wraps",
"source": "custom",
"tool_type": "validation",
"namespace": "local",
"short_name": "dict-mw",
"capability": {"read_only": True, "writes": False},
"timeout": 60,
"mode": "required",
"wraps": "local/inner",
}
@given("tools with long descriptions exist")
def step_tools_long_desc(context: Context) -> None:
"""List with real capability keys and a >40-char description."""
context.mock_service.list_tools.return_value = [
{
"name": "local/long-desc",
"description": (
"A very long description that exceeds forty characters easily"
),
"source": "custom",
"tool_type": "tool",
"namespace": "local",
"short_name": "long-desc",
"capability": {"read_only": True, "writes": True},
"timeout": 120,
},
]
@given("tools as objects exist")
def step_tools_as_objects(context: Context) -> None:
"""Return mock objects (not dicts) so ``_get_name`` uses ``getattr``."""
obj = MagicMock()
obj.name = "local/obj-tool"
obj.as_cli_dict.return_value = OrderedDict(
[
("name", "local/obj-tool"),
("description", "Object tool"),
("source", "custom"),
("tool_type", "tool"),
("namespace", "local"),
("short_name", "obj-tool"),
("capability", {}),
("timeout", 60),
]
)
context.mock_service.list_tools.return_value = [obj]
@given("a non-mapping YAML tool file exists")
def step_non_mapping_yaml_tool(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write("- just\n- a\n- list\n")
context.tool_yaml_path = path
@given("a non-mapping YAML validation file exists")
def step_non_mapping_yaml_val(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write("- just\n- a\n- list\n")
context.validation_yaml_path = path
@given("validation add returns as_cli_dict object")
def step_val_add_returns_cli_dict(context: Context) -> None:
mock_val = MagicMock()
mock_val.as_cli_dict.return_value = OrderedDict(
[
("name", "local/cli-dict-val"),
("description", "Val via as_cli_dict"),
("source", "custom"),
("tool_type", "validation"),
("mode", "required"),
]
)
context.mock_service.register_tool.return_value = mock_val
@given("validation add returns raw string object")
def step_val_add_returns_string(context: Context) -> None:
context.mock_service.register_tool.return_value = "raw-string-validation"
# -----------------------------------------------------------------------
# Additional coverage When steps
# -----------------------------------------------------------------------
@when("I run tool list as rich")
def step_run_tool_list_rich(context: Context) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(tool_app, ["list"])
@when('I run tool list regex filtered "{pattern}"')
def step_run_tool_list_regex(context: Context, pattern: str) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(tool_app, ["list", pattern])
@when("I run tool add from non-mapping config")
def step_run_tool_add_non_mapping(context: Context) -> None:
with _patch_tool_svc(context):
context.tool_result = _runner.invoke(
tool_app, ["add", "--config", context.tool_yaml_path]
)
@when("I run validation add from non-mapping config")
def step_run_val_add_non_mapping(context: Context) -> None:
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app,
["add", "--config", context.validation_yaml_path],
)
@when("I run validation add via cli_dict config")
def step_run_val_add_cli_dict(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(_VALIDATION_YAML)
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app, ["add", "--config", path]
)
@when("I run validation add via string config")
def step_run_val_add_string(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(_VALIDATION_YAML)
with _patch_val_svc(context):
context.validation_result = _runner.invoke(
validation_app, ["add", "--config", path]
)