"""Step definitions for the Tool and Validation CLI feature.""" from __future__ import annotations import os import tempfile from typing import Any from unittest.mock import MagicMock, patch from behave import given, then, when from behave.runner import Context from typer.testing import CliRunner from cleveragents.cli.commands.tool import app as tool_app from cleveragents.cli.commands.validation import app as validation_app from cleveragents.core.exceptions import ( CleverAgentsError, NotFoundError, ValidationError, ) _TOOL_YAML = """\ name: local/test-tool description: A test tool source: custom code: | def run(inputs): return {"result": True} """ _VALIDATION_YAML = """\ name: local/test-val description: A test validation source: custom mode: required code: | def run(inputs): return {"passed": True} """ _INVALID_YAML = """\ name: local/bad-tool : missing key not_valid_yaml { """ _runner = CliRunner() def _make_mock_tool( name: str = "local/test-tool", tool_type: str = "tool", source: str = "custom", ) -> dict[str, Any]: """Create a mock tool dict as returned by the repository layer.""" ns, sn = name.split("/", 1) return { "name": name, "description": f"Test {tool_type}: {name}", "source": source, "tool_type": tool_type, "namespace": ns, "short_name": sn, "capability": {"read_only": False, "writes": False, "checkpointable": False}, "timeout": 300, } def _make_mock_attachment( attachment_id: str = "att-123", validation_name: str = "local/test-val", resource_id: str = "resource/r1", ) -> dict[str, Any]: return { "attachment_id": attachment_id, "validation_name": validation_name, "resource_id": resource_id, "mode": "required", "project_name": None, "plan_id": None, "created_at": "2026-01-01T00:00:00", } def _patch_tool_svc(context: Context) -> Any: return patch( "cleveragents.cli.commands.tool._get_tool_registry_service", return_value=context.mock_service, ) def _patch_val_svc(context: Context) -> Any: return patch( "cleveragents.cli.commands.validation._get_tool_registry_service", return_value=context.mock_service, ) # --------------------------------------------------------------------------- # Background # --------------------------------------------------------------------------- @given("a tool CLI runner with mocks") def step_tool_cli_runner(context: Context) -> None: context.runner = _runner context.tool_result = None context.validation_result = None context.tool_yaml_path = None context.validation_yaml_path = None context.invalid_yaml_path = None context.mock_service = MagicMock() context.patches = [] @given("a mocked tool registry service") def step_mocked_tool_registry_service(context: Context) -> None: svc = context.mock_service svc.register_tool.return_value = _make_mock_tool() svc.update_tool.return_value = _make_mock_tool() svc.remove_tool.return_value = True svc.list_tools.return_value = [] svc.get_tool.return_value = None svc.attach_validation.return_value = _make_mock_attachment() svc.detach_validation.return_value = True # --------------------------------------------------------------------------- # Given steps - Tool # --------------------------------------------------------------------------- @given("a valid tool config YAML file") def step_valid_tool_yaml(context: Context) -> None: fd, path = tempfile.mkstemp(suffix=".yaml") with os.fdopen(fd, "w") as fh: fh.write(_TOOL_YAML) context.tool_yaml_path = path @given("the tool already exists in the registry") def step_tool_already_exists(context: Context) -> None: context.mock_service.get_tool.return_value = _make_mock_tool() context.mock_service.update_tool.return_value = _make_mock_tool() @given("the tool does not exist in the registry") def step_tool_does_not_exist(context: Context) -> None: context.mock_service.get_tool.return_value = None @given("the tool registry raises a duplicate error on register") def step_tool_duplicate_error(context: Context) -> None: context.mock_service.register_tool.side_effect = CleverAgentsError( "Tool 'local/test-tool' already exists" ) @given("the tool registry raises a CleverAgentsError on register") def step_tool_register_clever_error(context: Context) -> None: context.mock_service.register_tool.side_effect = CleverAgentsError( "Registration failed" ) @given("the tool registry raises a CleverAgentsError on list") def step_tool_list_clever_error(context: Context) -> None: context.mock_service.list_tools.side_effect = CleverAgentsError("List failed") @given("the tool registry raises a CleverAgentsError on get") def step_tool_get_clever_error(context: Context) -> None: context.mock_service.get_tool.side_effect = CleverAgentsError("Get failed") @given("an invalid YAML tool config file") def step_invalid_tool_yaml(context: Context) -> None: fd, path = tempfile.mkstemp(suffix=".yaml") with os.fdopen(fd, "w") as fh: fh.write(_INVALID_YAML) context.invalid_yaml_path = path @given("there are mocked tools in the registry") def step_mocked_tools_exist(context: Context) -> None: context.mock_service.list_tools.return_value = [ _make_mock_tool("local/line-counter", "tool", "custom"), _make_mock_tool("local/docker-build", "tool", "mcp"), _make_mock_tool("local/coverage-check", "validation", "custom"), ] @given("there are no mocked tools") def step_no_mocked_tools(context: Context) -> None: context.mock_service.list_tools.return_value = [] @given("there is a mocked read-only tool in the registry") def step_mocked_read_only_tool(context: Context) -> None: tool = _make_mock_tool("local/read-only-tool", "tool", "custom") tool["capability"] = {"read_only": True, "writes": False, "checkpointable": False} context.mock_service.list_tools.return_value = [tool] @given('there is a mocked tool with name "{name}"') def step_mocked_tool_exists(context: Context, name: str) -> None: context.mock_service.get_tool.return_value = _make_mock_tool(name) context.mock_service.remove_tool.return_value = True @given('there is no tool with name "{name}"') def step_no_tool_exists(context: Context, name: str) -> None: context.mock_service.get_tool.return_value = None @given("the tool registry raises an error on remove") def step_tool_remove_error(context: Context) -> None: context.mock_service.remove_tool.side_effect = CleverAgentsError("Tool is in use") @given("the tool registry returns false on remove") def step_tool_remove_false(context: Context) -> None: context.mock_service.remove_tool.return_value = False # --------------------------------------------------------------------------- # Given steps - Validation # --------------------------------------------------------------------------- @given("a valid validation config YAML file") def step_valid_validation_yaml(context: Context) -> None: fd, path = tempfile.mkstemp(suffix=".yaml") with os.fdopen(fd, "w") as fh: fh.write(_VALIDATION_YAML) context.validation_yaml_path = path context.mock_service.register_tool.return_value = _make_mock_tool( "local/test-val", "validation" ) @given("the validation already exists in the registry") def step_validation_already_exists(context: Context) -> None: context.mock_service.get_tool.return_value = _make_mock_tool( "local/test-val", "validation" ) context.mock_service.update_tool.return_value = _make_mock_tool( "local/test-val", "validation" ) @given("the validation does not exist in the registry") def step_validation_does_not_exist(context: Context) -> None: context.mock_service.get_tool.return_value = None @given("an invalid YAML validation config file") def step_invalid_validation_yaml(context: Context) -> None: fd, path = tempfile.mkstemp(suffix=".yaml") with os.fdopen(fd, "w") as fh: fh.write(_INVALID_YAML) context.invalid_yaml_path = path @given("the validation registry raises a CleverAgentsError on register") def step_validation_register_clever_error(context: Context) -> None: context.mock_service.register_tool.side_effect = CleverAgentsError( "Registration failed" ) @given("the validation registry raises a validation error on register") def step_validation_register_validation_error(context: Context) -> None: context.mock_service.register_tool.side_effect = ValidationError( "Invalid validation config" ) @given("a mocked validation exists for attaching") def step_mock_validation_for_attach(context: Context) -> None: context.mock_service.attach_validation.return_value = _make_mock_attachment() @given("the validation is not found for attaching") def step_validation_not_found_for_attach(context: Context) -> None: context.mock_service.attach_validation.side_effect = NotFoundError( resource_type="validation", resource_id="local/missing-val", ) @given("the validation registry raises a CleverAgentsError on attach") def step_validation_attach_clever_error(context: Context) -> None: context.mock_service.attach_validation.side_effect = CleverAgentsError( "Attach failed" ) @given("the validation registry raises a validation error on attach") def step_validation_attach_validation_error(context: Context) -> None: context.mock_service.attach_validation.side_effect = ValidationError( "Invalid attachment" ) @given("a mocked attachment exists for detaching") def step_mock_attachment_for_detach(context: Context) -> None: context.mock_service.detach_validation.return_value = True @given("the attachment is not found for detaching") def step_attachment_not_found_for_detach(context: Context) -> None: context.mock_service.detach_validation.return_value = False @given("the validation registry raises a CleverAgentsError on detach") def step_validation_detach_clever_error(context: Context) -> None: context.mock_service.detach_validation.side_effect = CleverAgentsError( "Detach failed" ) # --------------------------------------------------------------------------- # When steps - Tool add # --------------------------------------------------------------------------- @when("I run tool CLI add with --config pointing to the YAML file") def step_run_tool_add(context: Context) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke( tool_app, ["add", "--config", context.tool_yaml_path] ) @when("I run tool CLI add with --config and --update") def step_run_tool_add_update(context: Context) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke( tool_app, ["add", "--config", context.tool_yaml_path, "--update"] ) @when("I run tool CLI add with --config pointing to a missing file") def step_run_tool_add_missing(context: Context) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke( tool_app, ["add", "--config", "/tmp/nonexistent-tool-config.yaml"] ) @when("I run tool CLI add with --config pointing to the invalid YAML file") def step_run_tool_add_invalid_yaml(context: Context) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke( tool_app, ["add", "--config", context.invalid_yaml_path] ) @when("I run tool CLI add with --config and --format json") def step_run_tool_add_json(context: Context) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke( tool_app, ["add", "--config", context.tool_yaml_path, "--format", "json"], ) # --------------------------------------------------------------------------- # When steps - Tool list # --------------------------------------------------------------------------- @when("I run tool CLI list") def step_run_tool_list(context: Context) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list"]) @when('I run tool CLI list with namespace filter "{ns}"') def step_run_tool_list_namespace(context: Context, ns: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list", "--namespace", ns]) @when('I run tool CLI list with source filter "{src}"') def step_run_tool_list_source(context: Context, src: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list", "--source", src]) @when('I run tool CLI list with type filter "{tt}"') def step_run_tool_list_type(context: Context, tt: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list", "--type", tt]) @when('I run tool CLI list with invalid type "{tt}"') def step_run_tool_list_invalid_type(context: Context, tt: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list", "--type", tt]) @when('I run tool CLI list with regex filter "{regex}"') def step_run_tool_list_regex(context: Context, regex: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list", regex]) @when('I run tool CLI list with invalid regex "{regex}"') def step_run_tool_list_invalid_regex(context: Context, regex: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list", regex]) @when('I run tool CLI list with format "{fmt}"') def step_run_tool_list_format(context: Context, fmt: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["list", "--format", fmt]) # --------------------------------------------------------------------------- # When steps - Tool show # --------------------------------------------------------------------------- @when('I run tool CLI show with name "{name}"') def step_run_tool_show(context: Context, name: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["show", name]) @when('I run tool CLI show for name "{name}" using format "{fmt}"') def step_run_tool_show_format(context: Context, name: str, fmt: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["show", name, "--format", fmt]) # --------------------------------------------------------------------------- # When steps - Tool remove # --------------------------------------------------------------------------- @when('I run tool CLI remove with name "{name}" and --yes') def step_run_tool_remove(context: Context, name: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke(tool_app, ["remove", name, "--yes"]) @when('I run tool CLI remove confirmed name "{name}" with fmt "{fmt}"') def step_run_tool_remove_format(context: Context, name: str, fmt: str) -> None: with _patch_tool_svc(context): context.tool_result = _runner.invoke( tool_app, ["remove", name, "--yes", "--format", fmt] ) # --------------------------------------------------------------------------- # When steps - Validation add # --------------------------------------------------------------------------- @when("I run validation CLI add with --config pointing to the YAML file") def step_run_validation_add(context: Context) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["add", "--config", context.validation_yaml_path] ) @when("I run validation CLI add with --config and --update") def step_run_validation_add_update(context: Context) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["add", "--config", context.validation_yaml_path, "--update"], ) @when("I run validation CLI add with --config pointing to a missing file") def step_run_validation_add_missing(context: Context) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["add", "--config", "/tmp/nonexistent-val-config.yaml"], ) @when("I run validation CLI add with --config pointing to the invalid YAML file") def step_run_validation_add_invalid_yaml(context: Context) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["add", "--config", context.invalid_yaml_path] ) @when("I run validation CLI add with --config and --format json") def step_run_validation_add_json(context: Context) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["add", "--config", context.validation_yaml_path, "--format", "json"], ) # --------------------------------------------------------------------------- # When steps - Validation attach # --------------------------------------------------------------------------- @when('I run validation CLI attach "{resource}" "{val_name}"') def step_run_validation_attach(context: Context, resource: str, val_name: str) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["attach", resource, val_name] ) @when( 'I run validation CLI attach scoped "{resource}" "{val_name}" ' 'with project "{project}"' ) def step_run_validation_attach_project( context: Context, resource: str, val_name: str, project: str ) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["attach", resource, val_name, "--project", project] ) @when( 'I run validation CLI attach with named option "{resource}" "{val_name}" ' 'option "{option}" value "{value}"' ) def step_run_validation_attach_named_option( context: Context, resource: str, val_name: str, option: str, value: str ) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["attach", resource, val_name, option, value] ) @when( 'I run validation CLI attach with extra args "{resource}" "{val_name}" ' 'arg is "{arg}"' ) def step_run_validation_attach_args( context: Context, resource: str, val_name: str, arg: str ) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["attach", resource, val_name, arg] ) @when('I run validation CLI attach formatted "{resource}" "{val_name}" fmt is "{fmt}"') def step_run_validation_attach_format( context: Context, resource: str, val_name: str, fmt: str ) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["attach", resource, val_name, "--format", fmt] ) # --------------------------------------------------------------------------- # When steps - Validation detach # --------------------------------------------------------------------------- @when('I run validation CLI detach "{att_id}" with --yes') def step_run_validation_detach(context: Context, att_id: str) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["detach", att_id, "--yes"] ) @when('I run validation CLI detach formatted "{att_id}" confirmed fmt "{fmt}"') def step_run_validation_detach_format(context: Context, att_id: str, fmt: str) -> None: with _patch_val_svc(context): context.validation_result = _runner.invoke( validation_app, ["detach", att_id, "--yes", "--format", fmt] ) # --------------------------------------------------------------------------- # Then assertions - Tool # --------------------------------------------------------------------------- @then("the tool CLI add should succeed") def step_tool_add_success(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0, ( f"Expected exit 0, got {context.tool_result.exit_code}. " f"Output: {context.tool_result.output}" ) @then('the tool CLI output should contain "{text}"') def step_tool_output_contains(context: Context, text: str) -> None: assert context.tool_result is not None assert text in context.tool_result.output, ( f"Expected '{text}' in output. Got: {context.tool_result.output}" ) @then("the tool CLI command should abort") def step_tool_command_aborts(context: Context) -> None: result = context.tool_result assert result is not None assert result.exit_code != 0, ( f"Expected non-zero exit, got {result.exit_code}. Output: {result.output}" ) @then("the tool CLI list should show all tools") def step_tool_list_shows_all(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0 assert ( "Tools" in context.tool_result.output or "local/" in context.tool_result.output ) @then("the tool CLI list should succeed with results") def step_tool_list_succeeds(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0 @then("the tool CLI should show no tools message") def step_tool_list_empty(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0 assert "No tools found" in context.tool_result.output @then("the Rich tool list output contains all five spec columns") def step_rich_tool_list_columns(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0 output = context.tool_result.output for col in ("Name", "Type", "Source", "Read-Only", "Writes"): assert col in output, ( f"Expected column '{col}' in Rich table output. Got: {output}" ) @then("the tool CLI should show the tool details") def step_tool_show_details(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0 assert "local/" in context.tool_result.output @then("the tool CLI show should succeed") def step_tool_show_succeeds(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0 @then("the tool CLI remove should succeed") def step_tool_remove_succeeds(context: Context) -> None: assert context.tool_result is not None assert context.tool_result.exit_code == 0 # --------------------------------------------------------------------------- # Then assertions - Validation # --------------------------------------------------------------------------- @then("the validation CLI add should succeed") def step_validation_add_success(context: Context) -> None: assert context.validation_result is not None assert context.validation_result.exit_code == 0, ( f"Expected exit 0, got {context.validation_result.exit_code}. " f"Output: {context.validation_result.output}" ) @then('the validation CLI output should contain "{text}"') def step_validation_output_contains(context: Context, text: str) -> None: assert context.validation_result is not None assert text in context.validation_result.output, ( f"Expected '{text}' in output. Got: {context.validation_result.output}" ) @then("the validation CLI command should abort") def step_validation_command_aborts(context: Context) -> None: result = context.validation_result assert result is not None assert result.exit_code != 0, ( f"Expected non-zero exit, got {result.exit_code}. Output: {result.output}" ) @then("the validation CLI attach should succeed") def step_validation_attach_succeeds(context: Context) -> None: assert context.validation_result is not None assert context.validation_result.exit_code == 0, ( f"Expected exit 0, got {context.validation_result.exit_code}. " f"Output: {context.validation_result.output}" ) @then("the validation CLI detach should succeed") def step_validation_detach_succeeds(context: Context) -> None: assert context.validation_result is not None assert context.validation_result.exit_code == 0, ( f"Expected exit 0, got {context.validation_result.exit_code}. " f"Output: {context.validation_result.output}" )