diff --git a/benchmarks/tool_cli_bench.py b/benchmarks/tool_cli_bench.py new file mode 100644 index 000000000..5b3569e97 --- /dev/null +++ b/benchmarks/tool_cli_bench.py @@ -0,0 +1,176 @@ +"""ASV benchmarks for Tool CLI command throughput. + +Measures the performance of: +- Config-only add (YAML load + validate + Tool.from_config) +- List command rendering +- Show command rendering +- Remove command rendering +""" + +from __future__ import annotations + +import importlib +import os +import sys +import tempfile +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.cli.commands.tool import app as tool_app # noqa: E402 +from cleveragents.cli.commands.validation import app as validation_app # noqa: E402 + +_TOOL_YAML = """\ +name: local/bench-tool +description: Benchmark tool +source: custom +code: | + def run(inputs): + return {"result": True} +""" + +_VALIDATION_YAML = """\ +name: local/bench-val +description: Benchmark validation +source: custom +mode: required +code: | + def run(inputs): + return {"passed": True} +""" + +_runner = CliRunner() + + +def _mock_tool( + name: str = "local/bench-tool", + tool_type: str = "tool", +) -> dict[str, Any]: + ns, sn = name.split("/", 1) + return { + "name": name, + "description": f"Benchmark {tool_type}", + "source": "custom", + "tool_type": tool_type, + "namespace": ns, + "short_name": sn, + "capability": {"read_only": False, "writes": False, "checkpointable": False}, + "timeout": 300, + } + + +class ToolCLIAddSuite: + """Benchmark tool add --config throughput.""" + + def setup(self) -> None: + """Write a temporary YAML file and set up mock service.""" + fd, self._path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_TOOL_YAML) + self._mock_service = MagicMock() + self._mock_service.register_tool.return_value = _mock_tool() + self._mock_service.get_tool.return_value = None + self._patcher = patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + """Clean up.""" + self._patcher.stop() + Path(self._path).unlink(missing_ok=True) + + def time_add_from_config(self) -> None: + """Benchmark add --config end-to-end.""" + _runner.invoke(tool_app, ["add", "--config", self._path]) + + +class ToolCLIListSuite: + """Benchmark tool list throughput.""" + + def setup(self) -> None: + """Set up mock service with tools.""" + self._mock_service = MagicMock() + self._mock_service.list_tools.return_value = [ + _mock_tool(f"local/tool-{i}") for i in range(50) + ] + self._patcher = patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + + def time_list_all(self) -> None: + """Benchmark listing all tools.""" + _runner.invoke(tool_app, ["list"]) + + def time_list_with_namespace(self) -> None: + """Benchmark listing with namespace filter.""" + _runner.invoke(tool_app, ["list", "--namespace", "local"]) + + def time_list_with_type(self) -> None: + """Benchmark listing with type filter.""" + _runner.invoke(tool_app, ["list", "--type", "tool"]) + + +class ToolCLIShowSuite: + """Benchmark tool show throughput.""" + + def setup(self) -> None: + self._mock_service = MagicMock() + self._mock_service.get_tool.return_value = _mock_tool() + self._patcher = patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + + def time_show(self) -> None: + """Benchmark showing a tool.""" + _runner.invoke(tool_app, ["show", "local/bench-tool"]) + + +class ValidationCLIAddSuite: + """Benchmark validation add --config throughput.""" + + def setup(self) -> None: + fd, self._path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_VALIDATION_YAML) + self._mock_service = MagicMock() + self._mock_service.register_tool.return_value = _mock_tool( + "local/bench-val", "validation" + ) + self._mock_service.get_tool.return_value = None + self._patcher = patch( + "cleveragents.cli.commands.validation._get_tool_registry_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + Path(self._path).unlink(missing_ok=True) + + def time_add_from_config(self) -> None: + """Benchmark validation add --config end-to-end.""" + _runner.invoke(validation_app, ["add", "--config", self._path]) diff --git a/docs/reference/tool_cli.md b/docs/reference/tool_cli.md new file mode 100644 index 000000000..fb13ff0dd --- /dev/null +++ b/docs/reference/tool_cli.md @@ -0,0 +1,179 @@ +# Tool & Validation CLI Reference + +The `agents tool` and `agents validation` command groups manage tools and +validations in the CleverAgents tool registry. + +## Tool Commands + +### `agents tool add` + +Register a tool from a YAML configuration file. + +```bash +agents tool add --config ./tools/line-counter.yaml +agents tool add --config ./tools/line-counter.yaml --update +agents tool add --config ./tools/line-counter.yaml --format json +``` + +**Options:** + +| Flag | Description | +|---------------|--------------------------------------| +| `--config` | Path to YAML configuration file | +| `--update` | Update if tool already exists | +| `--format` | Output format (json/yaml/plain/table/rich) | + +### `agents tool remove` + +Remove a tool by namespaced name. + +```bash +agents tool remove local/line-counter +agents tool remove --yes local/line-counter +``` + +**Options:** + +| Flag | Description | +|--------|---------------------------| +| `--yes`| Skip confirmation prompt | + +### `agents tool list` + +List tools with optional filters. + +```bash +agents tool list +agents tool list --namespace local +agents tool list --source mcp +agents tool list --type validation +agents tool list "line-.*" +agents tool list --format json +``` + +**Options:** + +| Flag | Description | +|---------------|------------------------------------------| +| `--namespace` | Filter by namespace | +| `--source` | Filter by source type | +| `--type` | Filter by type (tool or validation) | +| `--format` | Output format (json/yaml/plain/table/rich) | + +### `agents tool show` + +Show full details for a tool. + +```bash +agents tool show local/line-counter +agents tool show --format json local/line-counter +``` + +**Options:** + +| Flag | Description | +|------------|------------------------------------------| +| `--format` | Output format (json/yaml/plain/table/rich) | + +## Validation Commands + +### `agents validation add` + +Register a validation from a YAML configuration file. + +```bash +agents validation add --config ./validations/coverage-check.yaml +agents validation add --config ./validations/coverage-check.yaml --update +``` + +**Options:** + +| Flag | Description | +|---------------|--------------------------------------| +| `--config` | Path to YAML configuration file | +| `--update` | Update if validation already exists | +| `--format` | Output format (json/yaml/plain/table/rich) | + +### `agents validation attach` + +Attach a validation to a resource. + +```bash +agents validation attach git-checkout/my-repo local/coverage-check +agents validation attach --project myproj git-checkout/my-repo local/lint +agents validation attach --mode informational resource/r1 local/val1 +agents validation attach git-checkout/my-repo local/check threshold=80 +``` + +**Options:** + +| Flag | Description | +|-------------|------------------------------------------------| +| `--project` | Project scope | +| `--plan` | Plan ID scope | +| `--mode` | Validation mode (required or informational) | +| `--format` | Output format (json/yaml/plain/table/rich) | + +### `agents validation detach` + +Detach a validation from a resource. + +```bash +agents validation detach 01HXYZ1234567890ABCDEFGHIJ +agents validation detach --yes 01HXYZ1234567890ABCDEFGHIJ +``` + +**Options:** + +| Flag | Description | +|--------|---------------------------| +| `--yes`| Skip confirmation prompt | +| `--format` | Output format (json/yaml/plain/table/rich) | + +## YAML Configuration Examples + +### Tool Configuration + +```yaml +name: local/line-counter +description: Count lines in a file +source: custom +code: | + def run(inputs): + with open(inputs["path"]) as f: + return {"count": sum(1 for _ in f)} +input_schema: + type: object + properties: + path: + type: string +capability: + read_only: true + idempotent: true +timeout: 60 +``` + +### Validation Configuration + +```yaml +name: local/coverage-check +description: Check code coverage meets threshold +source: custom +mode: required +code: | + def run(inputs): + return {"passed": inputs["coverage"] >= inputs.get("threshold", 80)} +``` + +### Wrapped Validation + +```yaml +name: local/lint-check +description: Run linting as a validation +source: wrapped +wraps: local/run-lint +transform: | + def transform(result): + return {"passed": result.get("exit_code") == 0} +mode: informational +``` diff --git a/features/steps/tool_cli_coverage_steps.py b/features/steps/tool_cli_coverage_steps.py new file mode 100644 index 000000000..ed39f8dc5 --- /dev/null +++ b/features/steps/tool_cli_coverage_steps.py @@ -0,0 +1,803 @@ +"""Step definitions for the Tool and Validation CLI coverage feature. + +These steps exercise additional code paths not covered by the main +``tool_cli.feature`` scenarios: ``as_cli_dict`` branch, lifecycle / slot / +validation-mode rich rendering, dict without namespace, wraps / transform +fields, ValidationError handling, plan_id attach, and confirmation-decline +paths. +""" + +from __future__ import annotations + +import os +import tempfile +from collections import OrderedDict +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner + +from cleveragents.cli.commands.tool import app as tool_app +from cleveragents.cli.commands.validation import app as validation_app +from cleveragents.core.exceptions import CleverAgentsError, ValidationError + +_runner = CliRunner() + + +def _patch_tool_svc(context: Context) -> Any: + return patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=context.mock_service, + ) + + +def _patch_val_svc(context: Context) -> Any: + return patch( + "cleveragents.cli.commands.validation._get_tool_registry_service", + return_value=context.mock_service, + ) + + +_TOOL_YAML = """\ +name: local/test-tool +description: A test tool +source: custom +code: | + def run(inputs): + return {"result": True} +""" + +_VALIDATION_YAML = """\ +name: local/test-val +description: A test validation +source: custom +mode: required +code: | + def run(inputs): + return {"passed": True} +""" + + +# ----------------------------------------------------------------------- +# Background +# ----------------------------------------------------------------------- + + +@given("a tool CLI coverage runner with mocks") +def step_coverage_runner(context: Context) -> None: + context.runner = _runner + context.tool_result = None + context.validation_result = None + context.mock_service = MagicMock() + context.tool_yaml_path = None + context.validation_yaml_path = None + + +# ----------------------------------------------------------------------- +# Given - tools with special attributes +# ----------------------------------------------------------------------- + + +def _make_cli_dict_tool(name: str = "local/domain-tool") -> MagicMock: + """Create a mock tool object that has an ``as_cli_dict`` method.""" + ns, sn = name.split("/", 1) + mock = MagicMock() + mock.name = name + mock.as_cli_dict.return_value = OrderedDict( + [ + ("name", name), + ("description", "Domain model tool"), + ("source", "custom"), + ("tool_type", "tool"), + ("namespace", ns), + ("short_name", sn), + ("capability", {"read_only": False}), + ("timeout", 120), + ] + ) + return mock + + +@given("a tool with as_cli_dict method exists") +def step_tool_with_as_cli_dict(context: Context) -> None: + mock_tool = _make_cli_dict_tool() + context.mock_service.get_tool.return_value = mock_tool + + +@given("a tool with lifecycle hooks exists") +def step_tool_with_lifecycle(context: Context) -> None: + mock_tool = MagicMock() + mock_tool.name = "local/lifecycle-tool" + + def _cli_dict() -> OrderedDict[str, Any]: + return OrderedDict( + [ + ("name", "local/lifecycle-tool"), + ("description", "Tool with lifecycle"), + ("source", "custom"), + ("tool_type", "tool"), + ("namespace", "local"), + ("short_name", "lifecycle-tool"), + ("capability", {}), + ("timeout", 60), + ( + "lifecycle", + {"on_start": "setup()", "on_stop": "teardown()"}, + ), + ] + ) + + mock_tool.as_cli_dict = _cli_dict + context.mock_service.get_tool.return_value = mock_tool + + +@given("a tool with resource slots exists") +def step_tool_with_slots(context: Context) -> None: + mock_tool = MagicMock() + mock_tool.name = "local/slotted-tool" + + def _cli_dict() -> OrderedDict[str, Any]: + return OrderedDict( + [ + ("name", "local/slotted-tool"), + ("description", "Tool with resource slots"), + ("source", "custom"), + ("tool_type", "tool"), + ("namespace", "local"), + ("short_name", "slotted-tool"), + ("capability", {}), + ("timeout", 60), + ( + "resource_slots", + [ + { + "name": "db", + "resource_type": "database", + "access": "readwrite", + } + ], + ), + ] + ) + + mock_tool.as_cli_dict = _cli_dict + context.mock_service.get_tool.return_value = mock_tool + + +@given("a tool with validation mode exists") +def step_tool_with_validation_mode(context: Context) -> None: + mock_tool = MagicMock() + mock_tool.name = "local/val-mode-tool" + + def _cli_dict() -> OrderedDict[str, Any]: + return OrderedDict( + [ + ("name", "local/val-mode-tool"), + ("description", "Tool with validation mode"), + ("source", "custom"), + ("tool_type", "validation"), + ("namespace", "local"), + ("short_name", "val-mode-tool"), + ("capability", {}), + ("timeout", 60), + ("mode", "required"), + ] + ) + + mock_tool.as_cli_dict = _cli_dict + context.mock_service.get_tool.return_value = mock_tool + + +@given("a tool with wraps field exists") +def step_tool_with_wraps(context: Context) -> None: + mock_tool = MagicMock() + mock_tool.name = "local/wrap-tool" + + def _cli_dict() -> OrderedDict[str, Any]: + return OrderedDict( + [ + ("name", "local/wrap-tool"), + ("description", "Tool with wraps"), + ("source", "custom"), + ("tool_type", "tool"), + ("namespace", "local"), + ("short_name", "wrap-tool"), + ("capability", {}), + ("timeout", 60), + ("wraps", "local/inner-tool"), + ] + ) + + mock_tool.as_cli_dict = _cli_dict + context.mock_service.get_tool.return_value = mock_tool + + +# ----------------------------------------------------------------------- +# Given - tool list edge cases +# ----------------------------------------------------------------------- + + +@given("a tool list returns minimal objects") +def step_tool_list_minimal(context: Context) -> None: + """Return items that are plain strings (not dicts) so the fallback + ``OrderedDict({"name": str(tool)})`` path in ``_tool_spec_dict`` runs.""" + context.mock_service.list_tools.return_value = ["raw-string-tool"] + + +# ----------------------------------------------------------------------- +# Given - validation with wraps/transform +# ----------------------------------------------------------------------- + + +@given("a validation with wraps and transform exists") +def step_validation_wraps_transform(context: Context) -> None: + result = { + "name": "local/wrapped-val", + "description": "Validation with wraps and transform", + "source": "custom", + "tool_type": "validation", + "mode": "required", + "wraps": "local/inner-tool", + "transform": "uppercase", + } + context.mock_service.register_tool.return_value = result + + +# ----------------------------------------------------------------------- +# Given - object-like attachment +# ----------------------------------------------------------------------- + + +@given("an object-like attachment from service") +def step_object_like_attachment(context: Context) -> None: + mock_att = MagicMock() + mock_att.attachment_id = "att-obj-001" + mock_att.validation_name = "local/v1" + mock_att.resource_id = "res/x" + mock_att.mode = "required" + mock_att.project_name = None + mock_att.plan_id = None + mock_att.created_at = "2026-02-01T00:00:00" + context.mock_service.attach_validation.return_value = mock_att + + +# ----------------------------------------------------------------------- +# Given - tool add yaml config +# ----------------------------------------------------------------------- + + +@given("a tool add yaml config exists") +def step_tool_add_yaml(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_TOOL_YAML) + context.tool_yaml_path = path + ns, sn = "local", "test-tool" + context.mock_service.register_tool.return_value = { + "name": f"{ns}/{sn}", + "description": "A test tool", + "source": "custom", + "tool_type": "tool", + "namespace": ns, + "short_name": sn, + "capability": {}, + "timeout": 300, + } + + +@given("tool service raises validation error on register") +def step_tool_svc_validation_error(context: Context) -> None: + context.mock_service.register_tool.side_effect = ValidationError( + "Invalid tool config" + ) + + +# ----------------------------------------------------------------------- +# Given - validation add yaml config +# ----------------------------------------------------------------------- + + +@given("a validation add yaml config exists") +def step_validation_add_yaml(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_VALIDATION_YAML) + context.validation_yaml_path = path + context.mock_service.register_tool.return_value = { + "name": "local/test-val", + "description": "A test validation", + "source": "custom", + "tool_type": "validation", + "mode": "required", + } + + +# ----------------------------------------------------------------------- +# Given - CleverAgentsError on list +# ----------------------------------------------------------------------- + + +@given("tool service raises CleverAgentsError on list") +def step_tool_svc_list_error(context: Context) -> None: + context.mock_service.list_tools.side_effect = CleverAgentsError( + "Database unavailable" + ) + + +# ----------------------------------------------------------------------- +# Given - attachment service returns attachment (for plan-id scenario) +# ----------------------------------------------------------------------- + + +@given("an attachment service returns attachment") +def step_attachment_svc_returns(context: Context) -> None: + context.mock_service.attach_validation.return_value = { + "attachment_id": "att-plan-001", + "validation_name": "local/val", + "resource_id": "res/abc", + "mode": "required", + "project_name": None, + "plan_id": "plan-123", + "created_at": "2026-02-01T00:00:00", + } + + +# ----------------------------------------------------------------------- +# Given - dict tool without explicit namespace +# ----------------------------------------------------------------------- + + +@given("a dict tool without explicit namespace") +def step_dict_tool_no_namespace(context: Context) -> None: + context.mock_service.get_tool.return_value = { + "name": "other/thing", + "description": "No explicit namespace", + "source": "custom", + "tool_type": "tool", + "capability": {}, + "timeout": 60, + } + + +# ----------------------------------------------------------------------- +# Given - dict validation with wraps/transform +# ----------------------------------------------------------------------- + + +@given("a dict validation with wraps and transform") +def step_dict_validation_wraps_transform(context: Context) -> None: + result = { + "name": "local/wrapped-val", + "description": "Dict val with wraps", + "source": "custom", + "tool_type": "validation", + "mode": "required", + "wraps": "local/inner", + "transform": "lowercase", + } + context.mock_service.register_tool.return_value = result + + +# ----------------------------------------------------------------------- +# Given - detach / remove helpers +# ----------------------------------------------------------------------- + + +@given("an attachment service returns true on detach") +def step_attachment_svc_detach_true(context: Context) -> None: + context.mock_service.detach_validation.return_value = True + + +@given("a tool exists for removal") +def step_tool_exists_for_removal(context: Context) -> None: + context.mock_service.get_tool.return_value = { + "name": "local/rm-tool", + "description": "Removable", + "source": "custom", + "tool_type": "tool", + "namespace": "local", + "short_name": "rm-tool", + "capability": {}, + "timeout": 60, + } + context.mock_service.remove_tool.return_value = True + + +# ----------------------------------------------------------------------- +# When steps - tool show +# ----------------------------------------------------------------------- + + +@when('I run tool show for "{name}"') +def step_run_tool_show_coverage(context: Context, name: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["show", name]) + + +@when('I run tool show formatted "{name}" using fmt "{fmt}"') +def step_run_tool_show_fmt(context: Context, name: str, fmt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["show", name, "--format", fmt]) + + +@when("I run tool add from config") +def step_run_tool_add_config(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, ["add", "--config", context.tool_yaml_path] + ) + + +# ----------------------------------------------------------------------- +# When steps - tool list +# ----------------------------------------------------------------------- + + +@when('I run tool list with fmt "{fmt}"') +def step_run_tool_list_cov_fmt(context: Context, fmt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", "--format", fmt]) + + +@when('I run tool list with source "{src}"') +def step_run_tool_list_cov_source(context: Context, src: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", "--source", src]) + + +# ----------------------------------------------------------------------- +# When steps - tool add with format +# ----------------------------------------------------------------------- + + +@when('I run tool add with fmt "{fmt}"') +def step_run_tool_add_cov_fmt(context: Context, fmt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, + ["add", "--config", context.tool_yaml_path, "--format", fmt], + ) + + +# ----------------------------------------------------------------------- +# When steps - validation add +# ----------------------------------------------------------------------- + + +@when('I run validation add with fmt "{fmt}"') +def step_run_validation_add_fmt(context: Context, fmt: str) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, + ["add", "--config", context.validation_yaml_path, "--format", fmt], + ) + + +@when('I run validation show via add with "{name}"') +def step_run_validation_show_via_add(context: Context, name: str) -> None: + """Register validation (add) and capture the output which exercises + ``_print_validation`` with wraps/transform fields.""" + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write( + f"name: {name}\n" + "description: Wrapped val\n" + "source: custom\n" + "mode: required\n" + "code: |\n def run(i): return True\n" + ) + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["add", "--config", path] + ) + + +@when('I run validation show via add for "{name}" with fmt "{fmt}"') +def step_run_validation_add_for_name_fmt(context: Context, name: str, fmt: str) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write( + f"name: {name}\n" + "description: Dict val\n" + "source: custom\n" + "mode: required\n" + "code: |\n def run(i): return True\n" + ) + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, + ["add", "--config", path, "--format", fmt], + ) + + +# ----------------------------------------------------------------------- +# When steps - validation attach +# ----------------------------------------------------------------------- + + +@when('I run validation attach "{resource}" "{val_name}"') +def step_run_validation_attach_cov( + context: Context, resource: str, val_name: str +) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["attach", resource, val_name] + ) + + +@when('I run validation attach with plan "{plan}"') +def step_run_validation_attach_plan(context: Context, plan: str) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, + ["attach", "res/abc", "local/val", "--plan", plan], + ) + + +# ----------------------------------------------------------------------- +# When steps - declining confirmation +# ----------------------------------------------------------------------- + + +@when('I run validation detach "{att_id}" declining confirmation') +def step_run_validation_detach_decline(context: Context, att_id: str) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["detach", att_id], input="n\n" + ) + + +@when('I run tool remove "{name}" declining confirmation') +def step_run_tool_remove_decline(context: Context, name: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["remove", name], input="n\n") + + +# ----------------------------------------------------------------------- +# Then steps +# ----------------------------------------------------------------------- + + +@then("the tool show output should succeed") +def step_tool_show_succeed(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0, ( + f"Expected exit 0, got {context.tool_result.exit_code}. " + f"Output: {context.tool_result.output}" + ) + + +@then("the tool list result should succeed") +def step_tool_list_succeed(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0, ( + f"Expected exit 0, got {context.tool_result.exit_code}. " + f"Output: {context.tool_result.output}" + ) + + +@then("the tool add result should succeed") +def step_tool_add_succeed(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0, ( + f"Expected exit 0, got {context.tool_result.exit_code}. " + f"Output: {context.tool_result.output}" + ) + + +@then("the tool add result should abort") +def step_tool_add_abort(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code != 0, ( + f"Expected non-zero exit, got {context.tool_result.exit_code}. " + f"Output: {context.tool_result.output}" + ) + + +@then("the validation add result should succeed") +def step_validation_add_succeed(context: Context) -> None: + assert context.validation_result is not None + assert context.validation_result.exit_code == 0, ( + f"Expected exit 0, got {context.validation_result.exit_code}. " + f"Output: {context.validation_result.output}" + ) + + +@then("the validation add result should abort") +def step_validation_add_abort(context: Context) -> None: + assert context.validation_result is not None + assert context.validation_result.exit_code != 0, ( + f"Expected non-zero exit, got {context.validation_result.exit_code}. " + f"Output: {context.validation_result.output}" + ) + + +@then("the validation attach result should succeed") +def step_validation_attach_succeed(context: Context) -> None: + assert context.validation_result is not None + assert context.validation_result.exit_code == 0, ( + f"Expected exit 0, got {context.validation_result.exit_code}. " + f"Output: {context.validation_result.output}" + ) + + +@then("the tool list result should abort") +def step_tool_list_abort(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code != 0, ( + f"Expected non-zero exit, got {context.tool_result.exit_code}. " + f"Output: {context.tool_result.output}" + ) + + +@then("the validation detach result should abort") +def step_validation_detach_abort(context: Context) -> None: + assert context.validation_result is not None + assert context.validation_result.exit_code != 0, ( + f"Expected non-zero exit, got {context.validation_result.exit_code}. " + f"Output: {context.validation_result.output}" + ) + + +@then("the tool remove result should abort") +def step_tool_remove_abort(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code != 0, ( + f"Expected non-zero exit, got {context.tool_result.exit_code}. " + f"Output: {context.tool_result.output}" + ) + + +# ----------------------------------------------------------------------- +# Additional coverage Given steps +# ----------------------------------------------------------------------- + + +@given("a dict tool with mode and wraps") +def step_dict_tool_mode_wraps(context: Context) -> None: + """Dict tool with ``mode`` and ``wraps`` keys to cover lines 130/132.""" + context.mock_service.get_tool.return_value = { + "name": "local/dict-mw", + "description": "Dict with mode+wraps", + "source": "custom", + "tool_type": "validation", + "namespace": "local", + "short_name": "dict-mw", + "capability": {"read_only": True, "writes": False}, + "timeout": 60, + "mode": "required", + "wraps": "local/inner", + } + + +@given("tools with long descriptions exist") +def step_tools_long_desc(context: Context) -> None: + """List with real capability keys and a >40-char description.""" + context.mock_service.list_tools.return_value = [ + { + "name": "local/long-desc", + "description": ( + "A very long description that exceeds forty characters easily" + ), + "source": "custom", + "tool_type": "tool", + "namespace": "local", + "short_name": "long-desc", + "capability": {"read_only": True, "writes": True}, + "timeout": 120, + }, + ] + + +@given("tools as objects exist") +def step_tools_as_objects(context: Context) -> None: + """Return mock objects (not dicts) so ``_get_name`` uses ``getattr``.""" + obj = MagicMock() + obj.name = "local/obj-tool" + obj.as_cli_dict.return_value = OrderedDict( + [ + ("name", "local/obj-tool"), + ("description", "Object tool"), + ("source", "custom"), + ("tool_type", "tool"), + ("namespace", "local"), + ("short_name", "obj-tool"), + ("capability", {}), + ("timeout", 60), + ] + ) + context.mock_service.list_tools.return_value = [obj] + + +@given("a non-mapping YAML tool file exists") +def step_non_mapping_yaml_tool(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write("- just\n- a\n- list\n") + context.tool_yaml_path = path + + +@given("a non-mapping YAML validation file exists") +def step_non_mapping_yaml_val(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write("- just\n- a\n- list\n") + context.validation_yaml_path = path + + +@given("validation add returns as_cli_dict object") +def step_val_add_returns_cli_dict(context: Context) -> None: + mock_val = MagicMock() + mock_val.as_cli_dict.return_value = OrderedDict( + [ + ("name", "local/cli-dict-val"), + ("description", "Val via as_cli_dict"), + ("source", "custom"), + ("tool_type", "validation"), + ("mode", "required"), + ] + ) + context.mock_service.register_tool.return_value = mock_val + + +@given("validation add returns raw string object") +def step_val_add_returns_string(context: Context) -> None: + context.mock_service.register_tool.return_value = "raw-string-validation" + + +# ----------------------------------------------------------------------- +# Additional coverage When steps +# ----------------------------------------------------------------------- + + +@when("I run tool list as rich") +def step_run_tool_list_rich(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list"]) + + +@when('I run tool list regex filtered "{pattern}"') +def step_run_tool_list_regex(context: Context, pattern: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", pattern]) + + +@when("I run tool add from non-mapping config") +def step_run_tool_add_non_mapping(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, ["add", "--config", context.tool_yaml_path] + ) + + +@when("I run validation add from non-mapping config") +def step_run_val_add_non_mapping(context: Context) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, + ["add", "--config", context.validation_yaml_path], + ) + + +@when("I run validation add via cli_dict config") +def step_run_val_add_cli_dict(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_VALIDATION_YAML) + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["add", "--config", path] + ) + + +@when("I run validation add via string config") +def step_run_val_add_string(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_VALIDATION_YAML) + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["add", "--config", path] + ) diff --git a/features/steps/tool_cli_steps.py b/features/steps/tool_cli_steps.py new file mode 100644 index 000000000..7e85f159b --- /dev/null +++ b/features/steps/tool_cli_steps.py @@ -0,0 +1,699 @@ +"""Step definitions for the Tool and Validation CLI feature.""" + +from __future__ import annotations + +import os +import tempfile +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner + +from cleveragents.cli.commands.tool import app as tool_app +from cleveragents.cli.commands.validation import app as validation_app +from cleveragents.core.exceptions import ( + CleverAgentsError, + NotFoundError, + ValidationError, +) + +_TOOL_YAML = """\ +name: local/test-tool +description: A test tool +source: custom +code: | + def run(inputs): + return {"result": True} +""" + +_VALIDATION_YAML = """\ +name: local/test-val +description: A test validation +source: custom +mode: required +code: | + def run(inputs): + return {"passed": True} +""" + +_INVALID_YAML = """\ +name: local/bad-tool +: missing key + not_valid_yaml { +""" + +_runner = CliRunner() + + +def _make_mock_tool( + name: str = "local/test-tool", + tool_type: str = "tool", + source: str = "custom", +) -> dict[str, Any]: + """Create a mock tool dict as returned by the repository layer.""" + ns, sn = name.split("/", 1) + return { + "name": name, + "description": f"Test {tool_type}: {name}", + "source": source, + "tool_type": tool_type, + "namespace": ns, + "short_name": sn, + "capability": {"read_only": False, "writes": False, "checkpointable": False}, + "timeout": 300, + } + + +def _make_mock_attachment( + attachment_id: str = "att-123", + validation_name: str = "local/test-val", + resource_id: str = "resource/r1", +) -> dict[str, Any]: + return { + "attachment_id": attachment_id, + "validation_name": validation_name, + "resource_id": resource_id, + "mode": "required", + "project_name": None, + "plan_id": None, + "created_at": "2026-01-01T00:00:00", + } + + +def _patch_tool_svc(context: Context) -> Any: + return patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=context.mock_service, + ) + + +def _patch_val_svc(context: Context) -> Any: + return patch( + "cleveragents.cli.commands.validation._get_tool_registry_service", + return_value=context.mock_service, + ) + + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("a tool CLI runner with mocks") +def step_tool_cli_runner(context: Context) -> None: + context.runner = _runner + context.tool_result = None + context.validation_result = None + context.tool_yaml_path = None + context.validation_yaml_path = None + context.invalid_yaml_path = None + context.mock_service = MagicMock() + context.patches = [] + + +@given("a mocked tool registry service") +def step_mocked_tool_registry_service(context: Context) -> None: + svc = context.mock_service + svc.register_tool.return_value = _make_mock_tool() + svc.update_tool.return_value = _make_mock_tool() + svc.remove_tool.return_value = True + svc.list_tools.return_value = [] + svc.get_tool.return_value = None + svc.attach_validation.return_value = _make_mock_attachment() + svc.detach_validation.return_value = True + + +# --------------------------------------------------------------------------- +# Given steps - Tool +# --------------------------------------------------------------------------- + + +@given("a valid tool config YAML file") +def step_valid_tool_yaml(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_TOOL_YAML) + context.tool_yaml_path = path + + +@given("the tool already exists in the registry") +def step_tool_already_exists(context: Context) -> None: + context.mock_service.get_tool.return_value = _make_mock_tool() + context.mock_service.update_tool.return_value = _make_mock_tool() + + +@given("the tool does not exist in the registry") +def step_tool_does_not_exist(context: Context) -> None: + context.mock_service.get_tool.return_value = None + + +@given("the tool registry raises a duplicate error on register") +def step_tool_duplicate_error(context: Context) -> None: + context.mock_service.register_tool.side_effect = CleverAgentsError( + "Tool 'local/test-tool' already exists" + ) + + +@given("the tool registry raises a CleverAgentsError on register") +def step_tool_register_clever_error(context: Context) -> None: + context.mock_service.register_tool.side_effect = CleverAgentsError( + "Registration failed" + ) + + +@given("the tool registry raises a CleverAgentsError on list") +def step_tool_list_clever_error(context: Context) -> None: + context.mock_service.list_tools.side_effect = CleverAgentsError("List failed") + + +@given("the tool registry raises a CleverAgentsError on get") +def step_tool_get_clever_error(context: Context) -> None: + context.mock_service.get_tool.side_effect = CleverAgentsError("Get failed") + + +@given("an invalid YAML tool config file") +def step_invalid_tool_yaml(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_INVALID_YAML) + context.invalid_yaml_path = path + + +@given("there are mocked tools in the registry") +def step_mocked_tools_exist(context: Context) -> None: + context.mock_service.list_tools.return_value = [ + _make_mock_tool("local/line-counter", "tool", "custom"), + _make_mock_tool("local/docker-build", "tool", "mcp"), + _make_mock_tool("local/coverage-check", "validation", "custom"), + ] + + +@given("there are no mocked tools") +def step_no_mocked_tools(context: Context) -> None: + context.mock_service.list_tools.return_value = [] + + +@given('there is a mocked tool with name "{name}"') +def step_mocked_tool_exists(context: Context, name: str) -> None: + context.mock_service.get_tool.return_value = _make_mock_tool(name) + context.mock_service.remove_tool.return_value = True + + +@given('there is no tool with name "{name}"') +def step_no_tool_exists(context: Context, name: str) -> None: + context.mock_service.get_tool.return_value = None + + +@given("the tool registry raises an error on remove") +def step_tool_remove_error(context: Context) -> None: + context.mock_service.remove_tool.side_effect = CleverAgentsError("Tool is in use") + + +@given("the tool registry returns false on remove") +def step_tool_remove_false(context: Context) -> None: + context.mock_service.remove_tool.return_value = False + + +# --------------------------------------------------------------------------- +# Given steps - Validation +# --------------------------------------------------------------------------- + + +@given("a valid validation config YAML file") +def step_valid_validation_yaml(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_VALIDATION_YAML) + context.validation_yaml_path = path + context.mock_service.register_tool.return_value = _make_mock_tool( + "local/test-val", "validation" + ) + + +@given("the validation already exists in the registry") +def step_validation_already_exists(context: Context) -> None: + context.mock_service.get_tool.return_value = _make_mock_tool( + "local/test-val", "validation" + ) + context.mock_service.update_tool.return_value = _make_mock_tool( + "local/test-val", "validation" + ) + + +@given("the validation does not exist in the registry") +def step_validation_does_not_exist(context: Context) -> None: + context.mock_service.get_tool.return_value = None + + +@given("an invalid YAML validation config file") +def step_invalid_validation_yaml(context: Context) -> None: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_INVALID_YAML) + context.invalid_yaml_path = path + + +@given("the validation registry raises a CleverAgentsError on register") +def step_validation_register_clever_error(context: Context) -> None: + context.mock_service.register_tool.side_effect = CleverAgentsError( + "Registration failed" + ) + + +@given("the validation registry raises a validation error on register") +def step_validation_register_validation_error(context: Context) -> None: + context.mock_service.register_tool.side_effect = ValidationError( + "Invalid validation config" + ) + + +@given("a mocked validation exists for attaching") +def step_mock_validation_for_attach(context: Context) -> None: + context.mock_service.attach_validation.return_value = _make_mock_attachment() + + +@given("the validation is not found for attaching") +def step_validation_not_found_for_attach(context: Context) -> None: + context.mock_service.attach_validation.side_effect = NotFoundError( + resource_type="validation", + resource_id="local/missing-val", + ) + + +@given("the validation registry raises a CleverAgentsError on attach") +def step_validation_attach_clever_error(context: Context) -> None: + context.mock_service.attach_validation.side_effect = CleverAgentsError( + "Attach failed" + ) + + +@given("the validation registry raises a validation error on attach") +def step_validation_attach_validation_error(context: Context) -> None: + context.mock_service.attach_validation.side_effect = ValidationError( + "Invalid attachment" + ) + + +@given("a mocked attachment exists for detaching") +def step_mock_attachment_for_detach(context: Context) -> None: + context.mock_service.detach_validation.return_value = True + + +@given("the attachment is not found for detaching") +def step_attachment_not_found_for_detach(context: Context) -> None: + context.mock_service.detach_validation.return_value = False + + +@given("the validation registry raises a CleverAgentsError on detach") +def step_validation_detach_clever_error(context: Context) -> None: + context.mock_service.detach_validation.side_effect = CleverAgentsError( + "Detach failed" + ) + + +# --------------------------------------------------------------------------- +# When steps - Tool add +# --------------------------------------------------------------------------- + + +@when("I run tool CLI add with --config pointing to the YAML file") +def step_run_tool_add(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, ["add", "--config", context.tool_yaml_path] + ) + + +@when("I run tool CLI add with --config and --update") +def step_run_tool_add_update(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, ["add", "--config", context.tool_yaml_path, "--update"] + ) + + +@when("I run tool CLI add with --config pointing to a missing file") +def step_run_tool_add_missing(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, ["add", "--config", "/tmp/nonexistent-tool-config.yaml"] + ) + + +@when("I run tool CLI add with --config pointing to the invalid YAML file") +def step_run_tool_add_invalid_yaml(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, ["add", "--config", context.invalid_yaml_path] + ) + + +@when("I run tool CLI add with --config and --format json") +def step_run_tool_add_json(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, + ["add", "--config", context.tool_yaml_path, "--format", "json"], + ) + + +# --------------------------------------------------------------------------- +# When steps - Tool list +# --------------------------------------------------------------------------- + + +@when("I run tool CLI list") +def step_run_tool_list(context: Context) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list"]) + + +@when('I run tool CLI list with namespace filter "{ns}"') +def step_run_tool_list_namespace(context: Context, ns: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", "--namespace", ns]) + + +@when('I run tool CLI list with source filter "{src}"') +def step_run_tool_list_source(context: Context, src: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", "--source", src]) + + +@when('I run tool CLI list with type filter "{tt}"') +def step_run_tool_list_type(context: Context, tt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", "--type", tt]) + + +@when('I run tool CLI list with invalid type "{tt}"') +def step_run_tool_list_invalid_type(context: Context, tt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", "--type", tt]) + + +@when('I run tool CLI list with regex filter "{regex}"') +def step_run_tool_list_regex(context: Context, regex: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", regex]) + + +@when('I run tool CLI list with invalid regex "{regex}"') +def step_run_tool_list_invalid_regex(context: Context, regex: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", regex]) + + +@when('I run tool CLI list with format "{fmt}"') +def step_run_tool_list_format(context: Context, fmt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["list", "--format", fmt]) + + +# --------------------------------------------------------------------------- +# When steps - Tool show +# --------------------------------------------------------------------------- + + +@when('I run tool CLI show with name "{name}"') +def step_run_tool_show(context: Context, name: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["show", name]) + + +@when('I run tool CLI show for name "{name}" using format "{fmt}"') +def step_run_tool_show_format(context: Context, name: str, fmt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["show", name, "--format", fmt]) + + +# --------------------------------------------------------------------------- +# When steps - Tool remove +# --------------------------------------------------------------------------- + + +@when('I run tool CLI remove with name "{name}" and --yes') +def step_run_tool_remove(context: Context, name: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke(tool_app, ["remove", name, "--yes"]) + + +@when('I run tool CLI remove confirmed name "{name}" with fmt "{fmt}"') +def step_run_tool_remove_format(context: Context, name: str, fmt: str) -> None: + with _patch_tool_svc(context): + context.tool_result = _runner.invoke( + tool_app, ["remove", name, "--yes", "--format", fmt] + ) + + +# --------------------------------------------------------------------------- +# When steps - Validation add +# --------------------------------------------------------------------------- + + +@when("I run validation CLI add with --config pointing to the YAML file") +def step_run_validation_add(context: Context) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["add", "--config", context.validation_yaml_path] + ) + + +@when("I run validation CLI add with --config and --update") +def step_run_validation_add_update(context: Context) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, + ["add", "--config", context.validation_yaml_path, "--update"], + ) + + +@when("I run validation CLI add with --config pointing to a missing file") +def step_run_validation_add_missing(context: Context) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, + ["add", "--config", "/tmp/nonexistent-val-config.yaml"], + ) + + +@when("I run validation CLI add with --config pointing to the invalid YAML file") +def step_run_validation_add_invalid_yaml(context: Context) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["add", "--config", context.invalid_yaml_path] + ) + + +@when("I run validation CLI add with --config and --format json") +def step_run_validation_add_json(context: Context) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, + ["add", "--config", context.validation_yaml_path, "--format", "json"], + ) + + +# --------------------------------------------------------------------------- +# When steps - Validation attach +# --------------------------------------------------------------------------- + + +@when('I run validation CLI attach "{resource}" "{val_name}"') +def step_run_validation_attach(context: Context, resource: str, val_name: str) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["attach", resource, val_name] + ) + + +@when( + 'I run validation CLI attach scoped "{resource}" "{val_name}" ' + 'with project "{project}"' +) +def step_run_validation_attach_project( + context: Context, resource: str, val_name: str, project: str +) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["attach", resource, val_name, "--project", project] + ) + + +@when( + 'I run validation CLI attach with mode "{resource}" "{val_name}" mode is "{mode}"' +) +def step_run_validation_attach_mode( + context: Context, resource: str, val_name: str, mode: str +) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["attach", resource, val_name, "--mode", mode] + ) + + +@when( + 'I run validation CLI attach with extra args "{resource}" "{val_name}" ' + 'arg is "{arg}"' +) +def step_run_validation_attach_args( + context: Context, resource: str, val_name: str, arg: str +) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["attach", resource, val_name, arg] + ) + + +@when('I run validation CLI attach formatted "{resource}" "{val_name}" fmt is "{fmt}"') +def step_run_validation_attach_format( + context: Context, resource: str, val_name: str, fmt: str +) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["attach", resource, val_name, "--format", fmt] + ) + + +# --------------------------------------------------------------------------- +# When steps - Validation detach +# --------------------------------------------------------------------------- + + +@when('I run validation CLI detach "{att_id}" with --yes') +def step_run_validation_detach(context: Context, att_id: str) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["detach", att_id, "--yes"] + ) + + +@when('I run validation CLI detach formatted "{att_id}" confirmed fmt "{fmt}"') +def step_run_validation_detach_format(context: Context, att_id: str, fmt: str) -> None: + with _patch_val_svc(context): + context.validation_result = _runner.invoke( + validation_app, ["detach", att_id, "--yes", "--format", fmt] + ) + + +# --------------------------------------------------------------------------- +# Then assertions - Tool +# --------------------------------------------------------------------------- + + +@then("the tool CLI add should succeed") +def step_tool_add_success(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0, ( + f"Expected exit 0, got {context.tool_result.exit_code}. " + f"Output: {context.tool_result.output}" + ) + + +@then('the tool CLI output should contain "{text}"') +def step_tool_output_contains(context: Context, text: str) -> None: + assert context.tool_result is not None + assert text in context.tool_result.output, ( + f"Expected '{text}' in output. Got: {context.tool_result.output}" + ) + + +@then("the tool CLI command should abort") +def step_tool_command_aborts(context: Context) -> None: + result = context.tool_result + assert result is not None + assert result.exit_code != 0, ( + f"Expected non-zero exit, got {result.exit_code}. Output: {result.output}" + ) + + +@then("the tool CLI list should show all tools") +def step_tool_list_shows_all(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0 + assert ( + "Tools" in context.tool_result.output or "local/" in context.tool_result.output + ) + + +@then("the tool CLI list should succeed with results") +def step_tool_list_succeeds(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0 + + +@then("the tool CLI should show no tools message") +def step_tool_list_empty(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0 + assert "No tools found" in context.tool_result.output + + +@then("the tool CLI should show the tool details") +def step_tool_show_details(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0 + assert "local/" in context.tool_result.output + + +@then("the tool CLI show should succeed") +def step_tool_show_succeeds(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0 + + +@then("the tool CLI remove should succeed") +def step_tool_remove_succeeds(context: Context) -> None: + assert context.tool_result is not None + assert context.tool_result.exit_code == 0 + + +# --------------------------------------------------------------------------- +# Then assertions - Validation +# --------------------------------------------------------------------------- + + +@then("the validation CLI add should succeed") +def step_validation_add_success(context: Context) -> None: + assert context.validation_result is not None + assert context.validation_result.exit_code == 0, ( + f"Expected exit 0, got {context.validation_result.exit_code}. " + f"Output: {context.validation_result.output}" + ) + + +@then('the validation CLI output should contain "{text}"') +def step_validation_output_contains(context: Context, text: str) -> None: + assert context.validation_result is not None + assert text in context.validation_result.output, ( + f"Expected '{text}' in output. Got: {context.validation_result.output}" + ) + + +@then("the validation CLI command should abort") +def step_validation_command_aborts(context: Context) -> None: + result = context.validation_result + assert result is not None + assert result.exit_code != 0, ( + f"Expected non-zero exit, got {result.exit_code}. Output: {result.output}" + ) + + +@then("the validation CLI attach should succeed") +def step_validation_attach_succeeds(context: Context) -> None: + assert context.validation_result is not None + assert context.validation_result.exit_code == 0, ( + f"Expected exit 0, got {context.validation_result.exit_code}. " + f"Output: {context.validation_result.output}" + ) + + +@then("the validation CLI detach should succeed") +def step_validation_detach_succeeds(context: Context) -> None: + assert context.validation_result is not None + assert context.validation_result.exit_code == 0, ( + f"Expected exit 0, got {context.validation_result.exit_code}. " + f"Output: {context.validation_result.output}" + ) diff --git a/features/tool_cli.feature b/features/tool_cli.feature new file mode 100644 index 000000000..b1f9b0652 --- /dev/null +++ b/features/tool_cli.feature @@ -0,0 +1,311 @@ +Feature: Tool and Validation CLI commands + As a developer + I want to manage tools and validations via CLI commands + So that I can register, list, show, and remove tools and validations + + Background: + Given a tool CLI runner with mocks + And a mocked tool registry service + + # Tool add command tests + Scenario: Add tool via config file + Given a valid tool config YAML file + When I run tool CLI add with --config pointing to the YAML file + Then the tool CLI add should succeed + And the tool CLI output should contain "Tool Registered" + + Scenario: Add tool with --update flag when tool exists + Given a valid tool config YAML file + And the tool already exists in the registry + When I run tool CLI add with --config and --update + Then the tool CLI add should succeed + And the tool CLI output should contain "Tool Updated" + + Scenario: Add tool with --update flag when tool does not exist + Given a valid tool config YAML file + And the tool does not exist in the registry + When I run tool CLI add with --config and --update + Then the tool CLI add should succeed + And the tool CLI output should contain "Tool Registered" + + Scenario: Add tool with missing config file fails + When I run tool CLI add with --config pointing to a missing file + Then the tool CLI command should abort + + Scenario: Add tool with invalid YAML fails + Given an invalid YAML tool config file + When I run tool CLI add with --config pointing to the invalid YAML file + Then the tool CLI command should abort + + Scenario: Add tool with duplicate name fails + Given a valid tool config YAML file + And the tool registry raises a duplicate error on register + When I run tool CLI add with --config pointing to the YAML file + Then the tool CLI command should abort + + Scenario: Add tool with json format output + Given a valid tool config YAML file + When I run tool CLI add with --config and --format json + Then the tool CLI add should succeed + And the tool CLI output should contain "local/test-tool" + + # Tool list command tests + Scenario: List all tools + Given there are mocked tools in the registry + When I run tool CLI list + Then the tool CLI list should show all tools + + Scenario: List tools with namespace filter + Given there are mocked tools in the registry + When I run tool CLI list with namespace filter "local" + Then the tool CLI list should succeed with results + + Scenario: List tools with source filter + Given there are mocked tools in the registry + When I run tool CLI list with source filter "custom" + Then the tool CLI list should succeed with results + + Scenario: List tools with type filter + Given there are mocked tools in the registry + When I run tool CLI list with type filter "tool" + Then the tool CLI list should succeed with results + + Scenario: List tools with type filter validation + Given there are mocked tools in the registry + When I run tool CLI list with type filter "validation" + Then the tool CLI list should succeed with results + + Scenario: List tools with invalid type filter + When I run tool CLI list with invalid type "unknown" + Then the tool CLI command should abort + + Scenario: List tools with regex filter + Given there are mocked tools in the registry + When I run tool CLI list with regex filter "line-.*" + Then the tool CLI list should succeed with results + + Scenario: List tools with invalid regex + When I run tool CLI list with invalid regex "[invalid" + Then the tool CLI command should abort + + Scenario: List tools when empty + Given there are no mocked tools + When I run tool CLI list + Then the tool CLI should show no tools message + + Scenario: List tools with json format + Given there are mocked tools in the registry + When I run tool CLI list with format "json" + Then the tool CLI list should succeed with results + + Scenario: List tools with yaml format + Given there are mocked tools in the registry + When I run tool CLI list with format "yaml" + Then the tool CLI list should succeed with results + + Scenario: List tools with plain format + Given there are mocked tools in the registry + When I run tool CLI list with format "plain" + Then the tool CLI list should succeed with results + + Scenario: List tools with table format + Given there are mocked tools in the registry + When I run tool CLI list with format "table" + Then the tool CLI list should succeed with results + + # Tool show command tests + Scenario: Show tool by name + Given there is a mocked tool with name "local/test-tool" + When I run tool CLI show with name "local/test-tool" + Then the tool CLI should show the tool details + + Scenario: Show tool not found + Given there is no tool with name "local/missing" + When I run tool CLI show with name "local/missing" + Then the tool CLI command should abort + + Scenario: Show tool with json format + Given there is a mocked tool with name "local/test-tool" + When I run tool CLI show for name "local/test-tool" using format "json" + Then the tool CLI show should succeed + + Scenario: Show tool with yaml format + Given there is a mocked tool with name "local/test-tool" + When I run tool CLI show for name "local/test-tool" using format "yaml" + Then the tool CLI show should succeed + + Scenario: Show tool with plain format + Given there is a mocked tool with name "local/test-tool" + When I run tool CLI show for name "local/test-tool" using format "plain" + Then the tool CLI show should succeed + + # Tool remove command tests + Scenario: Remove tool by name with confirmation + Given there is a mocked tool with name "local/test-tool" + When I run tool CLI remove with name "local/test-tool" and --yes + Then the tool CLI remove should succeed + And the tool CLI output should contain "Removed tool" + + Scenario: Remove tool not found + Given there is no tool with name "local/missing" + When I run tool CLI remove with name "local/missing" and --yes + Then the tool CLI command should abort + + Scenario: Remove tool with json format + Given there is a mocked tool with name "local/test-tool" + When I run tool CLI remove confirmed name "local/test-tool" with fmt "json" + Then the tool CLI remove should succeed + + Scenario: Remove tool with service error + Given there is a mocked tool with name "local/test-tool" + And the tool registry raises an error on remove + When I run tool CLI remove with name "local/test-tool" and --yes + Then the tool CLI command should abort + + # Validation add command tests + Scenario: Add validation via config file + Given a valid validation config YAML file + When I run validation CLI add with --config pointing to the YAML file + Then the validation CLI add should succeed + And the validation CLI output should contain "Validation Registered" + + Scenario: Add validation with --update flag when validation exists + Given a valid validation config YAML file + And the validation already exists in the registry + When I run validation CLI add with --config and --update + Then the validation CLI add should succeed + And the validation CLI output should contain "Validation Updated" + + Scenario: Add validation with missing config file fails + When I run validation CLI add with --config pointing to a missing file + Then the validation CLI command should abort + + Scenario: Add validation with invalid YAML fails + Given an invalid YAML validation config file + When I run validation CLI add with --config pointing to the invalid YAML file + Then the validation CLI command should abort + + Scenario: Add validation with json format output + Given a valid validation config YAML file + When I run validation CLI add with --config and --format json + Then the validation CLI add should succeed + + # Validation attach command tests + Scenario: Attach validation to resource + Given a mocked validation exists for attaching + When I run validation CLI attach "resource/r1" "local/test-val" + Then the validation CLI attach should succeed + And the validation CLI output should contain "Attached validation" + + Scenario: Attach validation with project scope + Given a mocked validation exists for attaching + When I run validation CLI attach scoped "resource/r1" "local/test-val" with project "myproj" + Then the validation CLI attach should succeed + + Scenario: Attach validation with mode informational + Given a mocked validation exists for attaching + When I run validation CLI attach with mode "resource/r1" "local/test-val" mode is "informational" + Then the validation CLI attach should succeed + + Scenario: Attach validation with extra args + Given a mocked validation exists for attaching + When I run validation CLI attach with extra args "resource/r1" "local/test-val" arg is "threshold=80" + Then the validation CLI attach should succeed + + Scenario: Attach validation with invalid arg format + Given a mocked validation exists for attaching + When I run validation CLI attach with extra args "resource/r1" "local/test-val" arg is "badarg" + Then the validation CLI command should abort + + Scenario: Attach validation not found + Given the validation is not found for attaching + When I run validation CLI attach "resource/r1" "local/missing-val" + Then the validation CLI command should abort + + Scenario: Attach validation with json format + Given a mocked validation exists for attaching + When I run validation CLI attach formatted "resource/r1" "local/test-val" fmt is "json" + Then the validation CLI attach should succeed + + # Validation detach command tests + Scenario: Detach validation with confirmation + Given a mocked attachment exists for detaching + When I run validation CLI detach "att-123" with --yes + Then the validation CLI detach should succeed + And the validation CLI output should contain "Detached validation" + + Scenario: Detach validation not found + Given the attachment is not found for detaching + When I run validation CLI detach "att-missing" with --yes + Then the validation CLI command should abort + + Scenario: Detach validation with json format + Given a mocked attachment exists for detaching + When I run validation CLI detach formatted "att-123" confirmed fmt "json" + Then the validation CLI detach should succeed + + # Tool add with CleverAgentsError + Scenario: Add tool with CleverAgentsError from service + Given a valid tool config YAML file + And the tool registry raises a CleverAgentsError on register + When I run tool CLI add with --config pointing to the YAML file + Then the tool CLI command should abort + + # Tool list with CleverAgentsError + Scenario: List tools with CleverAgentsError from service + Given the tool registry raises a CleverAgentsError on list + When I run tool CLI list + Then the tool CLI command should abort + + # Tool show with CleverAgentsError + Scenario: Show tool with CleverAgentsError from service + Given the tool registry raises a CleverAgentsError on get + When I run tool CLI show with name "local/test-tool" + Then the tool CLI command should abort + + # Validation add with CleverAgentsError + Scenario: Add validation with CleverAgentsError from service + Given a valid validation config YAML file + And the validation registry raises a CleverAgentsError on register + When I run validation CLI add with --config pointing to the YAML file + Then the validation CLI command should abort + + # Validation attach with CleverAgentsError + Scenario: Attach validation with CleverAgentsError from service + Given the validation registry raises a CleverAgentsError on attach + When I run validation CLI attach "resource/r1" "local/test-val" + Then the validation CLI command should abort + + # Validation detach with CleverAgentsError + Scenario: Detach validation with CleverAgentsError from service + Given the validation registry raises a CleverAgentsError on detach + When I run validation CLI detach "att-123" with --yes + Then the validation CLI command should abort + + # Validation add with update and non-existent + Scenario: Add validation with --update flag when validation does not exist + Given a valid validation config YAML file + And the validation does not exist in the registry + When I run validation CLI add with --config and --update + Then the validation CLI add should succeed + And the validation CLI output should contain "Validation Registered" + + # Tool remove with remove returning false + Scenario: Remove tool with remove returning false + Given there is a mocked tool with name "local/fail-remove" + And the tool registry returns false on remove + When I run tool CLI remove with name "local/fail-remove" and --yes + Then the tool CLI command should abort + + # Validation add with validation error + Scenario: Add validation with validation error from service + Given a valid validation config YAML file + And the validation registry raises a validation error on register + When I run validation CLI add with --config pointing to the YAML file + Then the validation CLI command should abort + + # Validation attach with validation error + Scenario: Attach validation with validation error from service + Given the validation registry raises a validation error on attach + When I run validation CLI attach "resource/r1" "local/test-val" + Then the validation CLI command should abort diff --git a/features/tool_cli_coverage.feature b/features/tool_cli_coverage.feature new file mode 100644 index 000000000..54bd4a0dd --- /dev/null +++ b/features/tool_cli_coverage.feature @@ -0,0 +1,163 @@ +Feature: Tool and Validation CLI coverage + As a developer + I want full coverage of tool and validation CLI commands + So that all code paths are tested + + Background: + Given a tool CLI coverage runner with mocks + + # Coverage: Tool with as_cli_dict method (domain model) + Scenario: Show tool that has as_cli_dict method + Given a tool with as_cli_dict method exists + When I run tool show for "local/domain-tool" + Then the tool show output should succeed + + Scenario: Show tool with lifecycle hooks + Given a tool with lifecycle hooks exists + When I run tool show for "local/lifecycle-tool" + Then the tool show output should succeed + + Scenario: Show tool with resource slots + Given a tool with resource slots exists + When I run tool show for "local/slotted-tool" + Then the tool show output should succeed + + Scenario: Show tool with validation mode field + Given a tool with validation mode exists + When I run tool show for "local/val-mode-tool" + Then the tool show output should succeed + + Scenario: Show tool with wraps field + Given a tool with wraps field exists + When I run tool show for "local/wrap-tool" + Then the tool show output should succeed + + Scenario: Show tool with plain format uses dict rendering + Given a tool with as_cli_dict method exists + When I run tool show formatted "local/domain-tool" using fmt "plain" + Then the tool show output should succeed + + Scenario: Show tool with table format + Given a tool with as_cli_dict method exists + When I run tool show formatted "local/domain-tool" using fmt "table" + Then the tool show output should succeed + + Scenario: Show tool with yaml format + Given a tool with as_cli_dict method exists + When I run tool show formatted "local/domain-tool" using fmt "yaml" + Then the tool show output should succeed + + # Coverage: _tool_spec_dict with string tool (fallback) + Scenario: List tools returns items without name key + Given a tool list returns minimal objects + When I run tool list with fmt "json" + Then the tool list result should succeed + + # Coverage: validation _print_validation with wraps and transform + Scenario: Show validation with wraps and transform + Given a validation with wraps and transform exists + When I run validation show via add with "local/wrapped-val" + Then the validation add result should succeed + + # Coverage: validation _attachment_dict with object + Scenario: Attach validation returns object-like attachment + Given an object-like attachment from service + When I run validation attach "res/x" "local/v1" + Then the validation attach result should succeed + + # Coverage: tool add with yaml format + Scenario: Add tool with yaml format + Given a tool add yaml config exists + When I run tool add with fmt "yaml" + Then the tool add result should succeed + + # Coverage: tool add with ValidationError + Scenario: Add tool with ValidationError from service + Given a tool add yaml config exists + And tool service raises validation error on register + When I run tool add from config + Then the tool add result should abort + + # Coverage: validation add with yaml format + Scenario: Add validation with yaml format + Given a validation add yaml config exists + When I run validation add with fmt "yaml" + Then the validation add result should succeed + + # Coverage: tool list with CleverAgentsError from source filter + Scenario: List tools with source and CleverAgentsError + Given tool service raises CleverAgentsError on list + When I run tool list with source "bad" + Then the tool list result should abort + + # Coverage: validation attach with plan id + Scenario: Attach validation with plan id + Given an attachment service returns attachment + When I run validation attach with plan "plan-123" + Then the validation attach result should succeed + + # Coverage: _tool_spec_dict with dict tool missing namespace + Scenario: Tool spec dict from dict without explicit namespace + Given a dict tool without explicit namespace + When I run tool show for "other/thing" + Then the tool show output should succeed + + # Coverage: _validation_spec_dict with dict having wraps/transform + Scenario: Validation spec dict includes wraps and transform + Given a dict validation with wraps and transform + When I run validation show via add for "local/wrapped-val" with fmt "json" + Then the validation add result should succeed + + # Coverage: validation detach without confirmation (abort) + Scenario: Detach validation declines confirmation + Given an attachment service returns true on detach + When I run validation detach "att-1" declining confirmation + Then the validation detach result should abort + + # Coverage: tool remove without confirmation (abort) + Scenario: Remove tool declines confirmation + Given a tool exists for removal + When I run tool remove "local/rm-tool" declining confirmation + Then the tool remove result should abort + + # Coverage: dict tool with mode and wraps keys (_tool_spec_dict dict branch) + Scenario: Show dict tool with mode and wraps + Given a dict tool with mode and wraps + When I run tool show for "local/dict-mw" + Then the tool show output should succeed + + # Coverage: non-empty capability dict and long description truncation + Scenario: List tools with long description and capabilities + Given tools with long descriptions exist + When I run tool list as rich + Then the tool list result should succeed + + # Coverage: _get_name on object (not dict) in regex filter path + Scenario: List tools with regex filters object names + Given tools as objects exist + When I run tool list regex filtered "obj" + Then the tool list result should succeed + + # Coverage: tool add with non-mapping YAML triggers ValueError + Scenario: Add tool with non-mapping YAML + Given a non-mapping YAML tool file exists + When I run tool add from non-mapping config + Then the tool add result should abort + + # Coverage: validation add with non-mapping YAML triggers ValueError + Scenario: Add validation with non-mapping YAML + Given a non-mapping YAML validation file exists + When I run validation add from non-mapping config + Then the validation add result should abort + + # Coverage: _validation_spec_dict as_cli_dict branch + Scenario: Validation add returns object with as_cli_dict + Given validation add returns as_cli_dict object + When I run validation add via cli_dict config + Then the validation add result should succeed + + # Coverage: _validation_spec_dict string fallback + Scenario: Validation add returns raw string + Given validation add returns raw string object + When I run validation add via string config + Then the validation add result should succeed diff --git a/implementation_plan.md b/implementation_plan.md index 79446cbfc..c010b9e0c 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -2793,23 +2793,23 @@ No standalone Q0-Advanced commits planned. Advanced QA enhancements are bundled - [X] Git [Jeff]: `git branch -d feature/m3-tool-binding` - [X] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. -- [ ] **COMMIT (Owner: Jeff | Group: C1.tool.cli | Branch: feature/m3-tool-cli | Planned: Day 16 | Expected: Day 20) - Commit message: "feat(cli): add tool and validation commands"** - - [ ] Git [Jeff]: `git checkout master` - - [ ] Git [Jeff]: `git pull origin master` - - [ ] Git [Jeff]: `git checkout -b feature/m3-tool-cli` - - [ ] Git [Jeff]: `git fetch origin && git merge origin/master` - - [ ] Code [Jeff]: Implement `agents tool add/remove/list/show` with YAML config input, schema validation, `--type` filter, and `--format json|yaml|plain` outputs. - - [ ] Code [Jeff]: Implement `agents validation add` and `agents validation attach|detach [--mode required|informational]` (resource-scoped attachments only). - - [ ] Docs [Jeff]: Update CLI reference with tool/validation commands and output formats. - - [ ] Tests (Behave) [Jeff]: Add CLI scenarios for tool registration and validation attachment lifecycle. - - [ ] Tests (Robot) [Jeff]: Add Robot CLI suites for tool and validation commands. - - [ ] Tests (ASV) [Jeff]: Add `benchmarks/tool_cli_bench.py` for CLI parsing overhead. - - [ ] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. - - [ ] Quality [Jeff]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes across **entire** code base, do not ignore any failure even if it seems unrelated to this commit, fix it. - - [ ] Git [Jeff]: Perform an appropriate `git add` command to add all the files that should be part of this commit to the git index - - [ ] Git [Jeff]: `git commit -m "feat(cli): add tool and validation commands"` - - [ ] Git [Jeff]: `git push -u origin feature/m3-tool-cli` - - [ ] Forgejo PR [Jeff]: Open PR from `feature/m3-tool-cli` to `master` with description "Add tool/validation CLI commands with attachment workflows and tests.". +- [X] **COMMIT (Owner: Jeff | Group: C1.tool.cli | Branch: feature/m3-tool-cli | Planned: Day 16 | Expected: Day 20) - Commit message: "feat(cli): add tool and validation commands"** + - [X] Git [Jeff]: `git checkout master` + - [X] Git [Jeff]: `git pull origin master` + - [X] Git [Jeff]: `git checkout -b feature/m3-tool-cli` + - [X] Git [Jeff]: `git fetch origin && git merge origin/master` + - [X] Code [Jeff]: Implement `agents tool add/remove/list/show` with YAML config input, schema validation, `--type` filter, and `--format json|yaml|plain` outputs. + - [X] Code [Jeff]: Implement `agents validation add` and `agents validation attach|detach [--mode required|informational]` (resource-scoped attachments only). + - [X] Docs [Jeff]: Update CLI reference with tool/validation commands and output formats. + - [X] Tests (Behave) [Jeff]: Add CLI scenarios for tool registration and validation attachment lifecycle. + - [X] Tests (Robot) [Jeff]: Add Robot CLI suites for tool and validation commands. + - [X] Tests (ASV) [Jeff]: Add `benchmarks/tool_cli_bench.py` for CLI parsing overhead. + - [X] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. + - [X] Quality [Jeff]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes across **entire** code base, do not ignore any failure even if it seems unrelated to this commit, fix it. + - [X] Git [Jeff]: Perform an appropriate `git add` command to add all the files that should be part of this commit to the git index + - [X] Git [Jeff]: `git commit -m "feat(cli): add tool and validation commands"` + - [X] Git [Jeff]: `git push -u origin feature/m3-tool-cli` + - [X] Forgejo PR [Jeff]: Open PR from `feature/m3-tool-cli` to `master` with description "Add tool/validation CLI commands with attachment workflows and tests.". **Parallel Group C0.skill: Skill Registry & YAML [Aditya + Jeff + Luis]** (depends on C0.domain + C0.registry; must land before C3.protocol) **PARALLEL SUBTRACK C0.skill.schema [Aditya]**: Skill YAML schema + examples diff --git a/robot/helper_tool_cli.py b/robot/helper_tool_cli.py new file mode 100644 index 000000000..aa266f49e --- /dev/null +++ b/robot/helper_tool_cli.py @@ -0,0 +1,210 @@ +"""Helper script for tool_cli.robot smoke tests. + +Each subcommand is a self-contained check that prints a sentinel on success. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +# Ensure local source tree is importable +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.cli.commands.tool import app as tool_app # noqa: E402 +from cleveragents.cli.commands.validation import app as validation_app # noqa: E402 + +runner = CliRunner() + +_TOOL_YAML = """\ +name: local/smoke-tool +description: Smoke test tool +source: custom +code: | + def run(inputs): + return {"result": True} +""" + +_VALIDATION_YAML = """\ +name: local/smoke-val +description: Smoke test validation +source: custom +mode: required +code: | + def run(inputs): + return {"passed": True} +""" + + +def _mock_tool( + name: str = "local/smoke-tool", + tool_type: str = "tool", +) -> dict[str, Any]: + ns, sn = name.split("/", 1) + return { + "name": name, + "description": f"Smoke test {tool_type}", + "source": "custom", + "tool_type": tool_type, + "namespace": ns, + "short_name": sn, + "capability": {"read_only": False, "writes": False, "checkpointable": False}, + "timeout": 300, + } + + +def _mock_attachment() -> dict[str, Any]: + return { + "attachment_id": "att-smoke-001", + "validation_name": "local/smoke-val", + "resource_id": "resource/r1", + "mode": "required", + "project_name": None, + "plan_id": None, + "created_at": "2026-01-01T00:00:00", + } + + +def _write_yaml(content: str) -> str: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(content) + return path + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + + +def tool_add_config() -> None: + """Verify tool add --config works.""" + path = _write_yaml(_TOOL_YAML) + try: + svc = MagicMock() + svc.register_tool.return_value = _mock_tool() + svc.get_tool.return_value = None + with patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=svc, + ): + result = runner.invoke(tool_app, ["add", "--config", path]) + assert result.exit_code == 0, f"exit={result.exit_code}: {result.output}" + print("tool-cli-add-config-ok") + finally: + Path(path).unlink(missing_ok=True) + + +def tool_list_all() -> None: + """Verify tool list works.""" + svc = MagicMock() + svc.list_tools.return_value = [_mock_tool()] + with patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=svc, + ): + result = runner.invoke(tool_app, ["list"]) + assert result.exit_code == 0, f"exit={result.exit_code}: {result.output}" + print("tool-cli-list-ok") + + +def tool_show_name() -> None: + """Verify tool show accepts a namespaced name.""" + svc = MagicMock() + svc.get_tool.return_value = _mock_tool() + with patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=svc, + ): + result = runner.invoke(tool_app, ["show", "local/smoke-tool"]) + assert result.exit_code == 0, f"exit={result.exit_code}: {result.output}" + print("tool-cli-show-ok") + + +def tool_remove_name() -> None: + """Verify tool remove accepts a namespaced name.""" + svc = MagicMock() + svc.get_tool.return_value = _mock_tool() + svc.remove_tool.return_value = True + with patch( + "cleveragents.cli.commands.tool._get_tool_registry_service", + return_value=svc, + ): + result = runner.invoke(tool_app, ["remove", "local/smoke-tool", "--yes"]) + assert result.exit_code == 0, f"exit={result.exit_code}: {result.output}" + print("tool-cli-remove-ok") + + +def validation_add_config() -> None: + """Verify validation add --config works.""" + path = _write_yaml(_VALIDATION_YAML) + try: + svc = MagicMock() + svc.register_tool.return_value = _mock_tool("local/smoke-val", "validation") + svc.get_tool.return_value = None + with patch( + "cleveragents.cli.commands.validation._get_tool_registry_service", + return_value=svc, + ): + result = runner.invoke(validation_app, ["add", "--config", path]) + assert result.exit_code == 0, f"exit={result.exit_code}: {result.output}" + print("validation-cli-add-ok") + finally: + Path(path).unlink(missing_ok=True) + + +def validation_attach() -> None: + """Verify validation attach works.""" + svc = MagicMock() + svc.attach_validation.return_value = _mock_attachment() + with patch( + "cleveragents.cli.commands.validation._get_tool_registry_service", + return_value=svc, + ): + result = runner.invoke( + validation_app, + ["attach", "resource/r1", "local/smoke-val"], + ) + assert result.exit_code == 0, f"exit={result.exit_code}: {result.output}" + print("validation-cli-attach-ok") + + +def validation_detach() -> None: + """Verify validation detach works.""" + svc = MagicMock() + svc.detach_validation.return_value = True + with patch( + "cleveragents.cli.commands.validation._get_tool_registry_service", + return_value=svc, + ): + result = runner.invoke( + validation_app, + ["detach", "att-smoke-001", "--yes"], + ) + assert result.exit_code == 0, f"exit={result.exit_code}: {result.output}" + print("validation-cli-detach-ok") + + +_COMMANDS = { + "tool-add-config": tool_add_config, + "tool-list": tool_list_all, + "tool-show": tool_show_name, + "tool-remove": tool_remove_name, + "validation-add-config": validation_add_config, + "validation-attach": validation_attach, + "validation-detach": validation_detach, +} + +if __name__ == "__main__": + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>") + sys.exit(1) + _COMMANDS[sys.argv[1]]() diff --git a/robot/tool_cli.robot b/robot/tool_cli.robot new file mode 100644 index 000000000..cca076684 --- /dev/null +++ b/robot/tool_cli.robot @@ -0,0 +1,57 @@ +*** Settings *** +Documentation Smoke tests for Tool and Validation CLI commands +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_tool_cli.py + +*** Test Cases *** +Tool Add Config Creates Tool + [Documentation] Verify that ``tool add --config`` registers a tool + ${result}= Run Process ${PYTHON} ${HELPER} tool-add-config cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tool-cli-add-config-ok + +Tool List Shows Tools + [Documentation] Verify that ``tool list`` shows registered tools + ${result}= Run Process ${PYTHON} ${HELPER} tool-list cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tool-cli-list-ok + +Tool Show Accepts Namespaced Name + [Documentation] Verify that ``tool show`` accepts a namespaced name argument + ${result}= Run Process ${PYTHON} ${HELPER} tool-show cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tool-cli-show-ok + +Tool Remove Accepts Namespaced Name + [Documentation] Verify that ``tool remove --yes`` removes a tool + ${result}= Run Process ${PYTHON} ${HELPER} tool-remove cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tool-cli-remove-ok + +Validation Add Config Creates Validation + [Documentation] Verify that ``validation add --config`` registers a validation + ${result}= Run Process ${PYTHON} ${HELPER} validation-add-config cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} validation-cli-add-ok + +Validation Attach Links Validation To Resource + [Documentation] Verify that ``validation attach`` links a validation to a resource + ${result}= Run Process ${PYTHON} ${HELPER} validation-attach cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} validation-cli-attach-ok + +Validation Detach Removes Attachment + [Documentation] Verify that ``validation detach --yes`` removes an attachment + ${result}= Run Process ${PYTHON} ${HELPER} validation-detach cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} validation-cli-detach-ok diff --git a/src/cleveragents/cli/commands/tool.py b/src/cleveragents/cli/commands/tool.py new file mode 100644 index 000000000..5b16ebb7a --- /dev/null +++ b/src/cleveragents/cli/commands/tool.py @@ -0,0 +1,467 @@ +"""Tool management commands for CleverAgents CLI. + +The ``agents tool`` command group manages tools (callable operations) +in the CleverAgents tool registry. + +## Commands + +| Command | Description | +|-----------------------|-------------------------------------------| +| ``agents tool add`` | Register tool from YAML config file | +| ``agents tool remove``| Remove a tool by namespaced name | +| ``agents tool list`` | List tools with optional filters | +| ``agents tool show`` | Show full tool details | + +## Config-Only Add + +Tools are registered **exclusively** via a YAML configuration file: + +```bash +agents tool add --config ./tools/line-counter.yaml +``` + +### YAML Configuration File + +```yaml +name: local/line-counter +description: Count lines in a file +source: custom +code: | + def run(inputs): + with open(inputs["path"]) as f: + return {"count": sum(1 for _ in f)} +``` + +## Error Handling + +| Error | Cause | +|----------------------------|------------------------------------------| +| ``Config file error`` | File not found or not readable | +| ``Schema validation error``| YAML does not match Tool schema | +| ``Duplicate tool`` | Tool name already registered | + +Based on implementation_plan.md -- Task C1.tool.cli. +""" + +from __future__ import annotations + +import re +from collections import OrderedDict +from pathlib import Path +from typing import Annotated, Any + +import typer +import yaml +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from cleveragents.cli.formatting import OutputFormat, format_output +from cleveragents.core.exceptions import ( + CleverAgentsError, + NotFoundError, + ValidationError, +) +from cleveragents.domain.models.core.tool import Tool, ToolType + +# Create sub-app for tool commands +app = typer.Typer(help="Manage tools (callable operations) in the tool registry.") +console = Console() + +# Reusable --format option description +_FORMAT_HELP = "Output format: json, yaml, plain, table, or rich (default: rich)" + + +def _get_tool_registry_service() -> Any: + """Get the ToolRegistryService from the container.""" + from cleveragents.application.container import get_container + + container = get_container() + database_url: str = container.database_url() + + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from cleveragents.application.services.tool_registry_service import ( + ToolRegistryService, + ) + from cleveragents.infrastructure.database.repositories import ( + ToolRegistryRepository, + ValidationAttachmentRepository, + ) + + engine = create_engine(database_url, echo=False) + factory = sessionmaker(bind=engine, expire_on_commit=False) + tool_repo = ToolRegistryRepository(session_factory=factory) + attachment_repo = ValidationAttachmentRepository(session_factory=factory) + return ToolRegistryService( + tool_repo=tool_repo, + attachment_repo=attachment_repo, + ) + + +def _tool_spec_dict(tool: Any) -> OrderedDict[str, Any]: + """Return tool data as a dict for CLI rendering. + + Supports both domain model objects (with ``as_cli_dict``) and + plain dicts returned by the repository layer. + """ + if hasattr(tool, "as_cli_dict"): + return tool.as_cli_dict() + + # Handle dict-like objects from the repository + if isinstance(tool, dict): + result: OrderedDict[str, Any] = OrderedDict() + result["name"] = tool.get("name", "") + result["description"] = tool.get("description", "") + result["source"] = tool.get("source", "") + result["tool_type"] = tool.get("tool_type", "tool") + ns = tool.get("namespace", "") + sn = tool.get("short_name", "") + if not ns and "/" in result["name"]: + ns = str(result["name"]).split("/", 1)[0] + sn = str(result["name"]).split("/", 1)[1] + result["namespace"] = ns + result["short_name"] = sn + result["capability"] = tool.get("capability", {}) + result["timeout"] = tool.get("timeout", 300) + # Include validation fields if present + if tool.get("mode"): + result["mode"] = tool["mode"] + if tool.get("wraps"): + result["wraps"] = tool["wraps"] + return result + + return OrderedDict({"name": str(tool)}) + + +def _print_tool( + tool: Any, + title: str = "Tool", + fmt: str = OutputFormat.RICH.value, +) -> None: + """Print tool details in the requested format.""" + if fmt != OutputFormat.RICH.value: + data = _tool_spec_dict(tool) + console.print(format_output(dict(data), fmt)) + return + + data = _tool_spec_dict(tool) + name = data.get("name", "") + desc = data.get("description", "") + source = data.get("source", "") + tool_type = data.get("tool_type", "tool") + namespace = data.get("namespace", "") + short_name = data.get("short_name", "") + timeout = data.get("timeout", 300) + + cap = data.get("capability", {}) + cap_lines: list[str] = [] + if isinstance(cap, dict): + for cap_key, cap_val in cap.items(): + cap_lines.append(f" {cap_key}: {cap_val}") + cap_display = "\n".join(cap_lines) if cap_lines else " (default)" + + lifecycle = data.get("lifecycle") + if lifecycle and isinstance(lifecycle, dict): + lc_lines = [f" {k}: {v}" for k, v in lifecycle.items() if v] + lc_display = "\n".join(lc_lines) if lc_lines else " (none)" + else: + lc_display = " (none)" + + slots = data.get("resource_slots") + if slots and isinstance(slots, list): + slot_lines: list[str] = [] + for slot in slots: + if isinstance(slot, dict): + slot_lines.append( + f" - {slot.get('name', '?')} " + f"({slot.get('resource_type', '?')}, " + f"{slot.get('access', '?')})" + ) + slots_display = "\n".join(slot_lines) if slot_lines else " (none)" + else: + slots_display = " (none)" + + details = ( + f"[bold]Name:[/bold] {name}\n" + f"[bold]Namespace:[/bold] {namespace}\n" + f"[bold]Short Name:[/bold] {short_name}\n" + f"[bold]Description:[/bold] {desc}\n" + f"[bold]Source:[/bold] {source}\n" + f"[bold]Type:[/bold] {tool_type}\n" + f"[bold]Timeout:[/bold] {timeout}s\n" + f"[bold]Capability:[/bold]\n{cap_display}\n" + f"[bold]Resource Slots:[/bold]\n{slots_display}\n" + f"[bold]Lifecycle:[/bold]\n{lc_display}" + ) + + # Show validation-specific fields + mode = data.get("mode") + if mode: + details += f"\n[bold]Validation Mode:[/bold] {mode}" + wraps = data.get("wraps") + if wraps: + details += f"\n[bold]Wraps:[/bold] {wraps}" + + console.print(Panel(details, title=title, expand=False)) + + +@app.command("add") +def add( + config: Annotated[ + Path, + typer.Option( + "--config", + "-c", + help="Path to the tool YAML configuration file", + exists=False, + ), + ], + update: Annotated[ + bool, + typer.Option("--update", help="Update if tool already exists"), + ] = False, + fmt: Annotated[ + str, + typer.Option("--format", "-f", help=_FORMAT_HELP), + ] = "rich", +) -> None: + """Register a new tool from a YAML configuration file. + + Tools are registered ONLY via ``--config ``. + + Examples: + agents tool add --config ./tools/line-counter.yaml + agents tool add --config ./tools/line-counter.yaml --update + """ + try: + if not config.exists(): + raise FileNotFoundError(f"Config file not found: {config}") + + raw = config.read_text(encoding="utf-8") + config_dict = yaml.safe_load(raw) + if not isinstance(config_dict, dict): + raise ValueError("YAML config must be a mapping") + + tool = Tool.from_config(config_dict) + service = _get_tool_registry_service() + + if update: + existing = service.get_tool(tool.name) + if existing is not None: + registered = service.update_tool(tool) + _print_tool(registered, title="Tool Updated", fmt=fmt) + return + + registered = service.register_tool(tool) + _print_tool(registered, title="Tool Registered", fmt=fmt) + + except FileNotFoundError as exc: + console.print(f"[red]Config file error:[/red] {exc}") + raise typer.Abort() from exc + except yaml.YAMLError as exc: + console.print(f"[red]YAML parse error:[/red] {exc}") + raise typer.Abort() from exc + except ValueError as exc: + console.print(f"[red]Schema validation error:[/red] {exc}") + raise typer.Abort() from exc + except ValidationError as exc: + console.print(f"[red]Validation Error:[/red] {exc.message}") + raise typer.Abort() from exc + except CleverAgentsError as exc: + console.print(f"[red]Error:[/red] {exc.message}") + raise typer.Abort() from exc + + +@app.command("remove") +def remove( + name: Annotated[ + str, + typer.Argument(help="Namespaced name of the tool to remove"), + ], + yes: Annotated[ + bool, + typer.Option("--yes", "-y", help="Skip confirmation prompt"), + ] = False, + fmt: Annotated[ + str, + typer.Option("--format", "-f", help=_FORMAT_HELP), + ] = "rich", +) -> None: + """Remove a tool by namespaced name. + + Examples: + agents tool remove local/line-counter + agents tool remove --yes local/line-counter + """ + try: + service = _get_tool_registry_service() + + existing = service.get_tool(name) + if existing is None: + console.print(f"[red]Tool not found:[/red] {name}") + raise typer.Abort() + + if not yes: + confirm = typer.confirm(f"Remove tool '{name}'?") + if not confirm: + console.print("[yellow]Aborted.[/yellow]") + raise typer.Abort() + + removed = service.remove_tool(name) + if not removed: + console.print(f"[red]Failed to remove tool:[/red] {name}") + raise typer.Abort() + + if fmt != OutputFormat.RICH.value: + data = {"name": name, "removed": True} + console.print(format_output(data, fmt)) + return + + console.print(f"[green]Removed tool:[/green] {name}") + + except CleverAgentsError as exc: + console.print(f"[red]Error:[/red] {exc.message}") + raise typer.Abort() from exc + + +@app.command("list") +def list_tools( + namespace: Annotated[ + str | None, + typer.Option("--namespace", "-n", help="Filter by namespace"), + ] = None, + source: Annotated[ + str | None, + typer.Option("--source", "-s", help="Filter by source type"), + ] = None, + tool_type: Annotated[ + str | None, + typer.Option("--type", "-t", help="Filter by type (tool or validation)"), + ] = None, + regex: Annotated[ + str | None, + typer.Argument(help="Optional regex pattern to filter tool names"), + ] = None, + fmt: Annotated[ + str, + typer.Option("--format", "-f", help=_FORMAT_HELP), + ] = "rich", +) -> None: + """List tools with optional filters. + + Examples: + agents tool list + agents tool list --namespace local + agents tool list --source mcp + agents tool list --type validation + agents tool list "line-.*" + agents tool list --format json + """ + try: + # Validate type filter if provided + if tool_type is not None: + try: + ToolType(tool_type.lower()) + except ValueError as exc: + console.print( + f"[red]Invalid type:[/red] {tool_type}. " + "Valid values: tool, validation" + ) + raise typer.Abort() from exc + + service = _get_tool_registry_service() + tools = service.list_tools( + namespace=namespace, + tool_type=tool_type, + source=source, + ) + + # Apply regex filter if provided + if regex: + try: + pattern = re.compile(regex) + except re.error as exc: + console.print(f"[red]Invalid regex pattern:[/red] {regex}") + raise typer.Abort() from exc + + def _get_name(t: Any) -> str: + if isinstance(t, dict): + return str(t.get("name", "")) + return str(getattr(t, "name", "")) + + tools = [t for t in tools if pattern.search(_get_name(t))] + + if not tools: + console.print("[yellow]No tools found.[/yellow]") + console.print("Register one with 'agents tool add --config '") + return + + # Non-rich formats use the formatting helper + if fmt != OutputFormat.RICH.value: + data = [dict(_tool_spec_dict(t)) for t in tools] + console.print(format_output(data, fmt)) + return + + # Rich table + table = Table(title=f"Tools ({len(tools)} total)") + table.add_column("Name", style="cyan") + table.add_column("Type", style="blue") + table.add_column("Source", style="magenta") + table.add_column("Description", style="dim") + table.add_column("Timeout", justify="right") + + for tool in tools: + spec = _tool_spec_dict(tool) + desc = str(spec.get("description", "")) + if len(desc) > 40: + desc = desc[:37] + "..." + table.add_row( + str(spec.get("name", "")), + str(spec.get("tool_type", "tool")), + str(spec.get("source", "")), + desc, + str(spec.get("timeout", 300)), + ) + + console.print(table) + + except CleverAgentsError as exc: + console.print(f"[red]Error:[/red] {exc.message}") + raise typer.Abort() from exc + + +@app.command("show") +def show( + name: Annotated[ + str, + typer.Argument(help="Namespaced name of the tool to show"), + ], + fmt: Annotated[ + str, + typer.Option("--format", "-f", help=_FORMAT_HELP), + ] = "rich", +) -> None: + """Show details for a tool. + + Specify the namespaced name (e.g. 'local/line-counter'). + """ + try: + service = _get_tool_registry_service() + tool = service.get_tool(name) + + if tool is None: + raise NotFoundError( + resource_type="tool", + resource_id=name, + ) + + _print_tool(tool, title="Tool Details", fmt=fmt) + + except NotFoundError as exc: + console.print(f"[red]Tool not found:[/red] {name}") + raise typer.Abort() from exc + except CleverAgentsError as exc: + console.print(f"[red]Error:[/red] {exc.message}") + raise typer.Abort() from exc diff --git a/src/cleveragents/cli/commands/validation.py b/src/cleveragents/cli/commands/validation.py new file mode 100644 index 000000000..d931b97b3 --- /dev/null +++ b/src/cleveragents/cli/commands/validation.py @@ -0,0 +1,376 @@ +"""Validation management commands for CleverAgents CLI. + +The ``agents validation`` command group manages validations (pass/fail tool +subtypes) and their lifecycle attachments to resources. + +## Commands + +| Command | Description | +|--------------------------------|------------------------------------------| +| ``agents validation add`` | Register validation from YAML config | +| ``agents validation attach`` | Attach validation to a resource | +| ``agents validation detach`` | Detach a validation attachment | + +## Config-Only Add + +Validations are registered **exclusively** via a YAML configuration file: + +```bash +agents validation add --config ./validations/coverage-check.yaml +``` + +### YAML Configuration File + +```yaml +name: local/coverage-check +description: Check code coverage meets threshold +source: custom +mode: required +code: | + def run(inputs): + return {"passed": inputs["coverage"] >= 80} +``` + +## Attach / Detach Lifecycle + +```bash +# Attach validation to a resource +agents validation attach git-checkout/my-repo local/coverage-check + +# Detach by attachment ID +agents validation detach 01HXYZ1234567890ABCDEFGHIJ +``` + +Based on implementation_plan.md -- Task C1.tool.cli. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Any + +import typer +import yaml +from rich.console import Console +from rich.panel import Panel + +from cleveragents.cli.formatting import OutputFormat, format_output +from cleveragents.core.exceptions import ( + CleverAgentsError, + NotFoundError, + ValidationError, +) +from cleveragents.domain.models.core.tool import Validation + +# Create sub-app for validation commands +app = typer.Typer(help="Manage validations (pass/fail tools) and resource attachments.") +console = Console() + +# Reusable --format option description +_FORMAT_HELP = "Output format: json, yaml, plain, table, or rich (default: rich)" + + +def _get_tool_registry_service() -> Any: + """Get the ToolRegistryService from the container.""" + from cleveragents.application.container import get_container + + container = get_container() + database_url: str = container.database_url() + + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from cleveragents.application.services.tool_registry_service import ( + ToolRegistryService, + ) + from cleveragents.infrastructure.database.repositories import ( + ToolRegistryRepository, + ValidationAttachmentRepository, + ) + + engine = create_engine(database_url, echo=False) + factory = sessionmaker(bind=engine, expire_on_commit=False) + tool_repo = ToolRegistryRepository(session_factory=factory) + attachment_repo = ValidationAttachmentRepository(session_factory=factory) + return ToolRegistryService( + tool_repo=tool_repo, + attachment_repo=attachment_repo, + ) + + +def _validation_spec_dict(tool: Any) -> dict[str, Any]: + """Return validation data as a dict for CLI rendering.""" + if hasattr(tool, "as_cli_dict"): + return dict(tool.as_cli_dict()) + + if isinstance(tool, dict): + result: dict[str, Any] = { + "name": tool.get("name", ""), + "description": tool.get("description", ""), + "source": tool.get("source", ""), + "tool_type": tool.get("tool_type", "validation"), + "mode": tool.get("mode", "required"), + } + if tool.get("wraps"): + result["wraps"] = tool["wraps"] + if tool.get("transform"): + result["transform"] = tool["transform"] + return result + + return {"name": str(tool)} + + +def _attachment_dict(attachment: Any) -> dict[str, Any]: + """Convert an attachment to a plain dict for output formatting.""" + if isinstance(attachment, dict): + return { + "attachment_id": attachment.get("attachment_id", ""), + "validation_name": attachment.get("validation_name", ""), + "resource_id": attachment.get("resource_id", ""), + "mode": attachment.get("mode", "required"), + "project_name": attachment.get("project_name"), + "plan_id": attachment.get("plan_id"), + "created_at": attachment.get("created_at", ""), + } + return { + "attachment_id": getattr(attachment, "attachment_id", ""), + "validation_name": getattr(attachment, "validation_name", ""), + "resource_id": getattr(attachment, "resource_id", ""), + "mode": getattr(attachment, "mode", "required"), + "project_name": getattr(attachment, "project_name", None), + "plan_id": getattr(attachment, "plan_id", None), + "created_at": str(getattr(attachment, "created_at", "")), + } + + +def _print_validation( + tool: Any, + title: str = "Validation", + fmt: str = OutputFormat.RICH.value, +) -> None: + """Print validation details in the requested format.""" + data = _validation_spec_dict(tool) + if fmt != OutputFormat.RICH.value: + console.print(format_output(data, fmt)) + return + + name = data.get("name", "") + desc = data.get("description", "") + source = data.get("source", "") + mode = data.get("mode", "required") + + details = ( + f"[bold]Name:[/bold] {name}\n" + f"[bold]Description:[/bold] {desc}\n" + f"[bold]Source:[/bold] {source}\n" + f"[bold]Mode:[/bold] {mode}" + ) + + wraps = data.get("wraps") + if wraps: + details += f"\n[bold]Wraps:[/bold] {wraps}" + transform = data.get("transform") + if transform: + details += f"\n[bold]Transform:[/bold] {transform}" + + console.print(Panel(details, title=title, expand=False)) + + +@app.command("add") +def add( + config: Annotated[ + Path, + typer.Option( + "--config", + "-c", + help="Path to the validation YAML configuration file", + exists=False, + ), + ], + update: Annotated[ + bool, + typer.Option("--update", help="Update if validation already exists"), + ] = False, + fmt: Annotated[ + str, + typer.Option("--format", "-f", help=_FORMAT_HELP), + ] = "rich", +) -> None: + """Register a new validation from a YAML configuration file. + + Validations are created ONLY via ``--config ``. + + Examples: + agents validation add --config ./validations/coverage-check.yaml + agents validation add --config ./validations/coverage-check.yaml --update + """ + try: + if not config.exists(): + raise FileNotFoundError(f"Config file not found: {config}") + + raw = config.read_text(encoding="utf-8") + config_dict = yaml.safe_load(raw) + if not isinstance(config_dict, dict): + raise ValueError("YAML config must be a mapping") + + validation = Validation.from_config(config_dict) + service = _get_tool_registry_service() + + if update: + existing = service.get_tool(validation.name) + if existing is not None: + registered = service.update_tool(validation) + _print_validation(registered, title="Validation Updated", fmt=fmt) + return + + registered = service.register_tool(validation) + _print_validation(registered, title="Validation Registered", fmt=fmt) + + except FileNotFoundError as exc: + console.print(f"[red]Config file error:[/red] {exc}") + raise typer.Abort() from exc + except yaml.YAMLError as exc: + console.print(f"[red]YAML parse error:[/red] {exc}") + raise typer.Abort() from exc + except ValueError as exc: + console.print(f"[red]Schema validation error:[/red] {exc}") + raise typer.Abort() from exc + except ValidationError as exc: + console.print(f"[red]Validation Error:[/red] {exc.message}") + raise typer.Abort() from exc + except CleverAgentsError as exc: + console.print(f"[red]Error:[/red] {exc.message}") + raise typer.Abort() from exc + + +@app.command("attach") +def attach( + resource: Annotated[ + str, + typer.Argument(help="Resource reference to attach validation to"), + ], + validation_name: Annotated[ + str, + typer.Argument(help="Namespaced name of the validation"), + ], + args: Annotated[ + list[str] | None, + typer.Argument(help="Additional arguments (key=value pairs)"), + ] = None, + project: Annotated[ + str | None, + typer.Option("--project", "-p", help="Project scope"), + ] = None, + plan_id: Annotated[ + str | None, + typer.Option("--plan", help="Plan ID scope"), + ] = None, + mode: Annotated[ + str, + typer.Option("--mode", "-m", help="Validation mode: required or informational"), + ] = "required", + fmt: Annotated[ + str, + typer.Option("--format", "-f", help=_FORMAT_HELP), + ] = "rich", +) -> None: + """Attach a validation to a resource. + + Examples: + agents validation attach git-checkout/my-repo local/coverage-check + agents validation attach --project myproj git-checkout/my-repo local/lint + agents validation attach --mode informational res1 local/val1 + """ + try: + # Parse extra args + extra_args: dict[str, str] | None = None + if args: + extra_args = {} + for arg in args: + if "=" not in arg: + console.print( + f"[red]Invalid argument format:[/red] {arg} " + "(expected key=value)" + ) + raise typer.Abort() + key, val = arg.split("=", 1) + extra_args[key] = val + + service = _get_tool_registry_service() + attachment = service.attach_validation( + validation_name=validation_name, + resource_id=resource, + mode=mode, + project_name=project, + plan_id=plan_id, + args=extra_args, + ) + + att_data = _attachment_dict(attachment) + + if fmt != OutputFormat.RICH.value: + console.print(format_output(att_data, fmt)) + return + + att_id = att_data.get("attachment_id", "") + console.print( + f"[green]Attached validation:[/green] {validation_name} -> " + f"{resource} (id: {att_id})" + ) + + except NotFoundError as exc: + console.print(f"[red]Validation not found:[/red] {validation_name}") + raise typer.Abort() from exc + except ValidationError as exc: + console.print(f"[red]Validation Error:[/red] {exc.message}") + raise typer.Abort() from exc + except CleverAgentsError as exc: + console.print(f"[red]Error:[/red] {exc.message}") + raise typer.Abort() from exc + + +@app.command("detach") +def detach( + attachment_id: Annotated[ + str, + typer.Argument(help="Attachment ID to detach"), + ], + yes: Annotated[ + bool, + typer.Option("--yes", "-y", help="Skip confirmation prompt"), + ] = False, + fmt: Annotated[ + str, + typer.Option("--format", "-f", help=_FORMAT_HELP), + ] = "rich", +) -> None: + """Detach a validation from a resource. + + Examples: + agents validation detach 01HXYZ1234567890ABCDEFGHIJ + agents validation detach --yes 01HXYZ1234567890ABCDEFGHIJ + """ + try: + if not yes: + confirm = typer.confirm(f"Detach validation attachment '{attachment_id}'?") + if not confirm: + console.print("[yellow]Aborted.[/yellow]") + raise typer.Abort() + + service = _get_tool_registry_service() + removed = service.detach_validation(attachment_id) + + if not removed: + console.print(f"[red]Attachment not found:[/red] {attachment_id}") + raise typer.Abort() + + if fmt != OutputFormat.RICH.value: + data = {"attachment_id": attachment_id, "detached": True} + console.print(format_output(data, fmt)) + return + + console.print(f"[green]Detached validation:[/green] {attachment_id}") + + except CleverAgentsError as exc: + console.print(f"[red]Error:[/red] {exc.message}") + raise typer.Abort() from exc diff --git a/src/cleveragents/cli/main.py b/src/cleveragents/cli/main.py index e394d245a..01d1e26af 100644 --- a/src/cleveragents/cli/main.py +++ b/src/cleveragents/cli/main.py @@ -83,6 +83,8 @@ def _register_subcommands() -> None: plan, project, resource, + tool, + validation, ) from cleveragents.cli.commands.auto_debug import app as auto_debug_app except Exception as exc: # pragma: no cover @@ -122,6 +124,16 @@ def _register_subcommands() -> None: name="resource", help="Manage resources and resource types in the resource registry", ) + app.add_typer( + tool.app, + name="tool", + help="Manage tools (callable operations) in the tool registry", + ) + app.add_typer( + validation.app, + name="validation", + help="Manage validations (pass/fail tools) and resource attachments", + ) _subcommands_registered = True @@ -482,6 +494,8 @@ def main(args: list[str] | None = None) -> int: "actor", "action", # v3 plan lifecycle actions "resource", # Resource registry management + "tool", # Tool registry management + "validation", # Validation management "auto-debug", # Auto-debug commands "tell", # Shortcut for plan tell "build", # Shortcut for plan build