diff --git a/benchmarks/tool_wrapping_bench.py b/benchmarks/tool_wrapping_bench.py new file mode 100644 index 000000000..1beaf0d7b --- /dev/null +++ b/benchmarks/tool_wrapping_bench.py @@ -0,0 +1,185 @@ +"""ASV benchmarks for tool wrapping delegation overhead. + +Measures the performance of: +- ArgumentMapper.apply() with identity and configured mappings +- TransformExecutor.execute() with simple transform functions +- WrappedToolExecutor.execute() for single and chained delegation +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from typing import Any + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.domain.models.core.tool import ( # noqa: E402 + Tool, + ToolCapability, + ToolSource, + ToolType, + Validation, + ValidationMode, +) +from cleveragents.tool.wrapping import ( # noqa: E402 + ArgumentMapper, + TransformExecutor, + WrappedToolExecutor, +) + +_SIMPLE_TRANSFORM = """\ +def transform(tool_output): + passed = tool_output.get("returncode") == 0 + return {"passed": passed, "message": "done", "data": tool_output} +""" + +_PASSTHROUGH_TRANSFORM = """\ +def transform(tool_output): + return {"passed": True, "message": "ok", "data": tool_output} +""" + + +def _make_tool(name: str) -> Tool: + return Tool( + name=name, + description=f"Benchmark tool {name}", + source=ToolSource.BUILTIN, + capability=ToolCapability(read_only=True), + ) + + +def _make_validation( + name: str, + wraps: str, + transform_code: str, + argument_mapping: dict[str, Any] | None = None, +) -> Validation: + return Validation( + name=name, + description=f"Benchmark validation wrapping {wraps}", + source=ToolSource.WRAPPED, + tool_type=ToolType.VALIDATION, + mode=ValidationMode.REQUIRED, + wraps=wraps, + transform=transform_code, + argument_mapping=argument_mapping, + ) + + +class ArgumentMapperSuite: + """Benchmarks for ArgumentMapper.""" + + def time_identity_mapping(self) -> None: + """Measure identity (passthrough) mapping overhead.""" + mapper = ArgumentMapper(None) + for _ in range(1000): + mapper.apply({"path": "/src", "verbose": True, "count": 42}) + + def time_configured_mapping(self) -> None: + """Measure mapping with configured argument translation.""" + mapper = ArgumentMapper( + { + "test_directory": "source_dir", + "coverage_enabled": True, + "verbose": False, + } + ) + for _ in range(1000): + mapper.apply({"source_dir": "/tests", "extra": "ignored"}) + + +class TransformExecutorSuite: + """Benchmarks for TransformExecutor.""" + + def setup(self) -> None: + self._executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool") + self._output = {"returncode": 0, "tests_run": 100} + + def time_execute_transform(self) -> None: + """Measure transform function execution.""" + for _ in range(1000): + self._executor.execute(self._output) + + def time_create_and_execute(self) -> None: + """Measure creation + execution (cold path).""" + for _ in range(100): + executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool") + executor.execute(self._output) + + +class WrappedToolExecutorSuite: + """Benchmarks for WrappedToolExecutor delegation.""" + + def setup(self) -> None: + self._tool = _make_tool("local/base-tool") + self._output: dict[str, Any] = {"returncode": 0, "data": "ok"} + self._validation = _make_validation( + "local/bench-wrap", + "local/base-tool", + _SIMPLE_TRANSFORM, + ) + + tools: dict[str, tuple[Tool, dict[str, Any]]] = { + "local/base-tool": (self._tool, self._output), + } + + def lookup(name: str) -> Tool | Validation | None: + if name in tools: + return tools[name][0] + return None + + def executor(name: str, args: dict[str, Any]) -> dict[str, Any]: + if name in tools: + return tools[name][1] + raise RuntimeError(f"Not found: {name}") + + self._executor = WrappedToolExecutor(lookup, executor) + + def time_single_delegation(self) -> None: + """Measure single-level wrapping delegation.""" + for _ in range(1000): + self._executor.execute(self._validation, {"path": "/src"}) + + def time_chained_delegation(self) -> None: + """Measure two-level wrapping delegation chain.""" + inner_val = _make_validation( + "local/inner-wrap", + "local/base-tool", + _PASSTHROUGH_TRANSFORM, + ) + outer_val = _make_validation( + "local/outer-wrap", + "local/inner-wrap", + _SIMPLE_TRANSFORM, + ) + + tools: dict[str, tuple[Tool, dict[str, Any]]] = { + "local/base-tool": (self._tool, self._output), + } + + all_items: dict[str, Tool | Validation] = { + "local/inner-wrap": inner_val, + "local/base-tool": self._tool, + } + + def lookup(name: str) -> Tool | Validation | None: + return all_items.get(name) + + def executor(name: str, args: dict[str, Any]) -> dict[str, Any]: + if name in tools: + return tools[name][1] + raise RuntimeError(f"Not found: {name}") + + chained_executor = WrappedToolExecutor(lookup, executor) + for _ in range(500): + chained_executor.execute(outer_val, {"path": "/src"}) diff --git a/features/steps/tool_wrapping_runtime_steps.py b/features/steps/tool_wrapping_runtime_steps.py new file mode 100644 index 000000000..f55dc4100 --- /dev/null +++ b/features/steps/tool_wrapping_runtime_steps.py @@ -0,0 +1,913 @@ +"""Step definitions for tool wrapping runtime feature tests.""" + +from __future__ import annotations + +import json +from typing import Any + +from behave import given, then, when +from behave.runner import Context + +from cleveragents.domain.models.core.tool import ( + Tool, + ToolCapability, + ToolSource, + ToolType, + Validation, + ValidationMode, +) +from cleveragents.tool.wrapping import ( + ArgumentMapper, + TransformExecutionError, + TransformExecutor, + WrappedToolExecutor, + WrappedToolNotFoundError, + WrappingCycleError, + WrappingDepthExceededError, +) + +# ------------------------------------------------------------------- +# Helpers +# ------------------------------------------------------------------- + +_SIMPLE_TRANSFORM = """\ +def transform(tool_output): + passed = tool_output.get("returncode") == 0 + msg = "All tests passed" if passed else "Tests failed" + return {"passed": passed, "message": msg, "data": tool_output} +""" + +_PASSTHROUGH_TRANSFORM = """\ +def transform(tool_output): + return {"passed": True, "message": "ok", "data": tool_output} +""" + +_STATUS_TRANSFORM = """\ +def transform(tool_output): + passed = tool_output.get("passed", False) + return {"passed": passed, "message": "outer", "data": tool_output} +""" + +_DATA_TRANSFORM = """\ +def transform(tool_output): + return {"passed": True, "message": "data ok", "data": tool_output} +""" + +_RETURNCODE_TRANSFORM = """\ +def transform(tool_output): + passed = tool_output.get("returncode") == 0 + msg = "All tests passed" if passed else "Tests failed" + return {"passed": passed, "message": msg, "data": tool_output} +""" + + +def _make_validation( + name: str, + wraps: str, + transform_code: str, + argument_mapping: dict[str, Any] | None = None, +) -> Validation: + """Create a Validation with wraps set.""" + return Validation( + name=name, + description=f"Test validation wrapping {wraps}", + source=ToolSource.WRAPPED, + tool_type=ToolType.VALIDATION, + mode=ValidationMode.REQUIRED, + wraps=wraps, + transform=transform_code, + argument_mapping=argument_mapping, + ) + + +def _make_plain_validation(name: str) -> Validation: + """Create a Validation without wraps.""" + return Validation( + name=name, + description="Test validation no wraps", + source=ToolSource.CUSTOM, + tool_type=ToolType.VALIDATION, + mode=ValidationMode.REQUIRED, + code="pass", + ) + + +def _make_tool(name: str) -> Tool: + """Create a plain Tool.""" + return Tool( + name=name, + description=f"Test tool {name}", + source=ToolSource.BUILTIN, + capability=ToolCapability(read_only=True), + ) + + +# ------------------------------------------------------------------- +# ArgumentMapper steps +# ------------------------------------------------------------------- + + +@given("an argument mapper with no mapping") +def step_mapper_no_mapping(context: Context) -> None: + context.mapper = ArgumentMapper(None) + + +@given("an argument mapper with mapping {mapping_json}") +def step_mapper_with_mapping(context: Context, mapping_json: str) -> None: + mapping = json.loads(mapping_json) + context.mapper = ArgumentMapper(mapping) + + +@when("I apply the mapper to inputs {inputs_json}") +def step_apply_mapper(context: Context, inputs_json: str) -> None: + inputs = json.loads(inputs_json) + context.mapped_result = context.mapper.apply(inputs) + + +@then("the mapped arguments should be {expected_json}") +def step_check_mapped(context: Context, expected_json: str) -> None: + expected = json.loads(expected_json) + assert context.mapped_result == expected, ( + f"Expected {expected}, got {context.mapped_result}" + ) + + +@when('I try to apply the mapper to non-dict input "{value}"') +def step_apply_mapper_non_dict(context: Context, value: str) -> None: + context.mapper_error = None + try: + context.mapper.apply(value) # type: ignore[arg-type] + except TypeError as exc: + context.mapper_error = exc + + +@then("a TypeError should be raised from the argument mapper") +def step_check_mapper_type_error(context: Context) -> None: + assert context.mapper_error is not None, "Expected TypeError" + assert isinstance(context.mapper_error, TypeError) + + +@when("I try to create an argument mapper with a non-dict mapping") +def step_create_mapper_non_dict(context: Context) -> None: + context.mapper_construction_error = None + try: + ArgumentMapper("not_a_dict") # type: ignore[arg-type] + except TypeError as exc: + context.mapper_construction_error = exc + + +@then("a TypeError should be raised from argument mapper construction") +def step_check_mapper_construction_error(context: Context) -> None: + assert context.mapper_construction_error is not None, "Expected TypeError" + assert isinstance(context.mapper_construction_error, TypeError) + + +# ------------------------------------------------------------------- +# TransformExecutor steps +# ------------------------------------------------------------------- + + +@given("a transform executor with code that checks returncode equals zero") +def step_transform_returncode(context: Context) -> None: + context.transform_executor = TransformExecutor( + _RETURNCODE_TRANSFORM, "test/transform" + ) + + +@when("I execute the transform with tool output {output_json}") +def step_execute_transform(context: Context, output_json: str) -> None: + output = json.loads(output_json) + context.transform_result = context.transform_executor.execute(output) + + +@then("the transform result should have passed true") +def step_check_transform_passed_true(context: Context) -> None: + assert context.transform_result["passed"] is True + + +@then("the transform result should have passed false") +def step_check_transform_passed_false(context: Context) -> None: + assert context.transform_result["passed"] is False + + +@then('the transform result should have message "{expected_msg}"') +def step_check_transform_message(context: Context, expected_msg: str) -> None: + assert context.transform_result["message"] == expected_msg, ( + f"Expected '{expected_msg}', got '{context.transform_result['message']}'" + ) + + +@given("a transform executor with code that does not define a transform function") +def step_transform_no_func(context: Context) -> None: + context.transform_executor = TransformExecutor("x = 1 + 2\n", "test/no-func") + + +@when("I try to execute the transform with any output") +def step_try_execute_transform(context: Context) -> None: + context.transform_error = None + try: + context.transform_executor.execute({"data": "test"}) + except (TransformExecutionError, ValueError) as exc: + context.transform_error = exc + + +@then('a TransformExecutionError should be raised with message containing "{fragment}"') +def step_check_transform_error(context: Context, fragment: str) -> None: + assert context.transform_error is not None, "Expected TransformExecutionError" + assert isinstance(context.transform_error, TransformExecutionError) + assert fragment in str(context.transform_error), ( + f"Expected '{fragment}' in '{context.transform_error}'" + ) + + +@given("a transform executor with code that returns a non-dict value") +def step_transform_non_dict_return(context: Context) -> None: + context.transform_executor = TransformExecutor( + "def transform(x):\n return 'not a dict'\n", + "test/non-dict", + ) + + +@given("a transform executor with code that returns a dict without passed key") +def step_transform_no_passed(context: Context) -> None: + context.transform_executor = TransformExecutor( + 'def transform(x):\n return {"message": "no passed"}\n', + "test/no-passed", + ) + + +@when("I try to create a transform executor with empty code") +def step_create_transform_empty(context: Context) -> None: + context.transform_construction_error = None + try: + TransformExecutor(" ", "test/empty") + except ValueError as exc: + context.transform_construction_error = exc + + +@then("a ValueError should be raised from transform construction") +def step_check_transform_construction_error(context: Context) -> None: + assert context.transform_construction_error is not None, "Expected ValueError" + assert isinstance(context.transform_construction_error, ValueError) + + +@given("a transform executor with code that attempts to import os") +def step_transform_sandbox_import(context: Context) -> None: + context.transform_executor = TransformExecutor( + "def transform(x):\n import os\n return {'passed': True}\n", + "test/sandbox", + ) + + +@then("a TransformExecutionError should be raised from sandbox restriction") +def step_check_sandbox_error(context: Context) -> None: + assert context.transform_error is not None, ( + "Expected TransformExecutionError from sandbox" + ) + assert isinstance(context.transform_error, TransformExecutionError) + + +# ------------------------------------------------------------------- +# WrappedToolExecutor steps +# ------------------------------------------------------------------- + + +@given('a tool registry with a tool "{tool_name}" that returns {output_json}') +def step_tool_registry_with_tool( + context: Context, + tool_name: str, + output_json: str, +) -> None: + if not hasattr(context, "test_tools"): + context.test_tools = {} + if not hasattr(context, "test_validations"): + context.test_validations = {} + if not hasattr(context, "tool_call_log"): + context.tool_call_log = {} + + output = json.loads(output_json) + context.test_tools[tool_name] = (_make_tool(tool_name), output) + + +@given('a validation "{val_name}" that wraps "{wraps_name}" with a simple transform') +def step_validation_simple_wrap( + context: Context, + val_name: str, + wraps_name: str, +) -> None: + if not hasattr(context, "test_validations"): + context.test_validations = {} + v = _make_validation(val_name, wraps_name, _SIMPLE_TRANSFORM) + context.test_validations[val_name] = v + + +@given( + 'a validation "{val_name}" that wraps "{wraps_name}" ' + "with argument mapping {mapping_json}" +) +def step_validation_with_mapping( + context: Context, + val_name: str, + wraps_name: str, + mapping_json: str, +) -> None: + if not hasattr(context, "test_validations"): + context.test_validations = {} + mapping = json.loads(mapping_json) + v = _make_validation( + val_name, + wraps_name, + _SIMPLE_TRANSFORM, + argument_mapping=mapping, + ) + context.test_validations[val_name] = v + + +@given( + 'a validation "{val_name}" that wraps "{wraps_name}" with a passthrough transform' +) +def step_validation_passthrough_wrap( + context: Context, + val_name: str, + wraps_name: str, +) -> None: + if not hasattr(context, "test_validations"): + context.test_validations = {} + v = _make_validation(val_name, wraps_name, _PASSTHROUGH_TRANSFORM) + context.test_validations[val_name] = v + + +@given('a validation "{val_name}" that wraps "{wraps_name}" with a status transform') +def step_validation_status_wrap( + context: Context, + val_name: str, + wraps_name: str, +) -> None: + if not hasattr(context, "test_validations"): + context.test_validations = {} + v = _make_validation(val_name, wraps_name, _STATUS_TRANSFORM) + context.test_validations[val_name] = v + + +@given('a validation "{val_name}" that wraps "{wraps_name}" with a data transform') +def step_validation_data_wrap( + context: Context, + val_name: str, + wraps_name: str, +) -> None: + if not hasattr(context, "test_validations"): + context.test_validations = {} + v = _make_validation(val_name, wraps_name, _DATA_TRANSFORM) + context.test_validations[val_name] = v + + +@given("a wrapped tool executor using the test registry") +def step_create_executor(context: Context) -> None: + tools = getattr(context, "test_tools", {}) + validations = getattr(context, "test_validations", {}) + context.tool_call_log = {} + + def lookup(name: str) -> Tool | Validation | None: + if name in validations: + return validations[name] + if name in tools: + return tools[name][0] + return None + + def executor(name: str, args: dict[str, Any]) -> Any: + context.tool_call_log[name] = args + if name in tools: + return tools[name][1] + raise RuntimeError(f"Tool {name} not found in test registry") + + context.wrapped_executor = WrappedToolExecutor(lookup, executor) + + +@given("a wrapped tool executor with empty registry") +def step_create_empty_executor(context: Context) -> None: + context.tool_call_log = {} + + def lookup(name: str) -> None: + return None + + def executor(name: str, args: dict[str, Any]) -> Any: + raise RuntimeError(f"Tool {name} not found") + + context.wrapped_executor = WrappedToolExecutor(lookup, executor) + + +@given("a wrapped tool executor using the test registry with cycle") +def step_create_executor_with_cycle(context: Context) -> None: + validations = getattr(context, "test_validations", {}) + + def lookup(name: str) -> Validation | None: + return validations.get(name) + + def executor(name: str, args: dict[str, Any]) -> Any: + raise RuntimeError("Should not reach leaf executor in cycle") + + context.wrapped_executor = WrappedToolExecutor(lookup, executor) + + +@when("I execute the wrapping validation with inputs {inputs_json}") +def step_execute_wrapping(context: Context, inputs_json: str) -> None: + inputs = json.loads(inputs_json) + # Find the first validation with wraps + val = None + for v in context.test_validations.values(): + if v.wraps is not None: + val = v + break + assert val is not None, "No wrapping validation found" + context.wrapped_result = context.wrapped_executor.execute(val, inputs) + context.last_executed_validation = val + + +@when("I execute the outer wrapping validation with inputs {inputs_json}") +def step_execute_outer_wrapping(context: Context, inputs_json: str) -> None: + inputs = json.loads(inputs_json) + val = context.test_validations.get("local/outer-wrapper") + assert val is not None, "local/outer-wrapper not found" + context.wrapped_result = context.wrapped_executor.execute(val, inputs) + + +@when("I try to execute the wrapping validation with inputs {inputs_json}") +def step_try_execute_wrapping(context: Context, inputs_json: str) -> None: + inputs = json.loads(inputs_json) + context.wrapped_error = None + # Find first validation with wraps + val = None + for v in getattr(context, "test_validations", {}).values(): + if v.wraps is not None: + val = v + break + if val is None: + return + try: + context.wrapped_executor.execute(val, inputs) + except (WrappedToolNotFoundError, WrappingCycleError) as exc: + context.wrapped_error = exc + + +@when('I try to execute the wrapping validation with cycle from "{val_name}"') +def step_try_execute_cycle(context: Context, val_name: str) -> None: + context.wrapped_error = None + val = context.test_validations[val_name] + try: + context.wrapped_executor.execute(val, {}) + except WrappingCycleError as exc: + context.wrapped_error = exc + + +@then("the wrapped execution should succeed with passed true") +def step_check_wrapped_passed(context: Context) -> None: + assert context.wrapped_result["passed"] is True, ( + f"Expected passed=True, got {context.wrapped_result}" + ) + + +@then('the wrapped tool "{tool_name}" should have been called') +def step_check_tool_called(context: Context, tool_name: str) -> None: + assert tool_name in context.tool_call_log, ( + f"Tool '{tool_name}' was not called. " + f"Called: {list(context.tool_call_log.keys())}" + ) + + +@then( + 'the wrapped tool "{tool_name}" should have received argument ' + '"{arg_name}" with value "{arg_value}"' +) +def step_check_tool_arg_str( + context: Context, + tool_name: str, + arg_name: str, + arg_value: str, +) -> None: + call_args = context.tool_call_log[tool_name] + assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}" + assert call_args[arg_name] == arg_value, ( + f"Expected '{arg_value}', got '{call_args[arg_name]}'" + ) + + +@then( + 'the wrapped tool "{tool_name}" should have received argument ' + '"{arg_name}" with value true' +) +def step_check_tool_arg_true( + context: Context, + tool_name: str, + arg_name: str, +) -> None: + call_args = context.tool_call_log[tool_name] + assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}" + assert call_args[arg_name] is True, f"Expected True, got {call_args[arg_name]}" + + +@then('a WrappedToolNotFoundError should be raised for "{wraps_name}"') +def step_check_not_found_error(context: Context, wraps_name: str) -> None: + assert context.wrapped_error is not None, "Expected WrappedToolNotFoundError" + assert isinstance(context.wrapped_error, WrappedToolNotFoundError) + assert wraps_name in str(context.wrapped_error) + + +@then("a WrappingCycleError should be raised") +def step_check_cycle_error(context: Context) -> None: + assert context.wrapped_error is not None, "Expected WrappingCycleError" + assert isinstance(context.wrapped_error, WrappingCycleError) + + +@then("the wrapped tool should have received the validation inputs") +def step_check_inputs_passed(context: Context) -> None: + assert len(context.tool_call_log) > 0, "No tool was called" + + +@when("I try to execute with a non-Validation object") +def step_try_execute_non_validation(context: Context) -> None: + context.executor_type_error = None + try: + context.wrapped_executor.execute("not_a_validation", {}) # type: ignore[arg-type] + except TypeError as exc: + context.executor_type_error = exc + + +@then("a TypeError should be raised from the executor") +def step_check_executor_type_error(context: Context) -> None: + assert context.executor_type_error is not None, "Expected TypeError" + assert isinstance(context.executor_type_error, TypeError) + + +@given('a validation "{val_name}" without wraps set') +def step_validation_no_wraps(context: Context, val_name: str) -> None: + if not hasattr(context, "test_validations"): + context.test_validations = {} + v = _make_plain_validation(val_name) + context.test_validations[val_name] = v + context.no_wrap_validation = v + + +@when("I try to execute the non-wrapping validation") +def step_try_execute_no_wraps(context: Context) -> None: + context.no_wrap_error = None + try: + context.wrapped_executor.execute(context.no_wrap_validation, {}) + except ValueError as exc: + context.no_wrap_error = exc + + +@then("a ValueError should be raised indicating wraps is not set") +def step_check_no_wrap_error(context: Context) -> None: + assert context.no_wrap_error is not None, "Expected ValueError" + assert isinstance(context.no_wrap_error, ValueError) + assert "wraps" in str(context.no_wrap_error).lower() + + +# ------------------------------------------------------------------- +# Additional coverage: WrappingDepthExceededError +# ------------------------------------------------------------------- + + +@given("a deep wrapping chain of {count:d} validations") +def step_deep_chain(context: Context, count: int) -> None: + if not hasattr(context, "test_validations"): + context.test_validations = {} + if not hasattr(context, "test_tools"): + context.test_tools = {} + # Create a chain: v0 wraps v1, v1 wraps v2, ..., vN wraps leaf-tool + for i in range(count): + wraps_name = f"local/deep-v{i + 1}" if i < count - 1 else "local/deep-leaf" + v = _make_validation( + f"local/deep-v{i}", + wraps_name, + _PASSTHROUGH_TRANSFORM, + ) + context.test_validations[v.name] = v + # Create the leaf tool + context.test_tools["local/deep-leaf"] = ( + _make_tool("local/deep-leaf"), + {"status": "ok"}, + ) + + +@given("a wrapped tool executor using the deep chain registry") +def step_create_deep_chain_executor(context: Context) -> None: + tools = getattr(context, "test_tools", {}) + validations = getattr(context, "test_validations", {}) + + def lookup(name: str) -> Tool | Validation | None: + if name in validations: + return validations[name] + if name in tools: + return tools[name][0] + return None + + def executor(name: str, args: dict[str, Any]) -> Any: + if name in tools: + return tools[name][1] + raise RuntimeError(f"Tool {name} not found") + + context.wrapped_executor = WrappedToolExecutor(lookup, executor) + + +@when("I try to execute the deep chain wrapping validation") +def step_try_execute_deep_chain(context: Context) -> None: + context.wrapped_error = None + val = context.test_validations["local/deep-v0"] + try: + context.wrapped_executor.execute(val, {}) + except WrappingDepthExceededError as exc: + context.wrapped_error = exc + + +@then("a WrappingDepthExceededError should be raised with depth {depth:d}") +def step_check_depth_exceeded(context: Context, depth: int) -> None: + assert context.wrapped_error is not None, "Expected WrappingDepthExceededError" + assert isinstance(context.wrapped_error, WrappingDepthExceededError) + assert context.wrapped_error.depth == depth, ( + f"Expected depth {depth}, got {context.wrapped_error.depth}" + ) + + +# ------------------------------------------------------------------- +# Additional coverage: ArgumentMapper.mapping property +# ------------------------------------------------------------------- + + +@then("the mapper mapping property should return {expected_json}") +def step_check_mapper_property(context: Context, expected_json: str) -> None: + expected = json.loads(expected_json) + assert context.mapper.mapping == expected, ( + f"Expected {expected}, got {context.mapper.mapping}" + ) + + +@then("the mapper mapping property should be None") +def step_check_mapper_property_none(context: Context) -> None: + assert context.mapper.mapping is None, ( + f"Expected None, got {context.mapper.mapping}" + ) + + +# ------------------------------------------------------------------- +# Additional coverage: TransformExecutor type checks +# ------------------------------------------------------------------- + + +@when("I try to create a transform executor with non-string code") +def step_create_transform_non_string_code(context: Context) -> None: + context.transform_type_error = None + try: + TransformExecutor(12345, "test/non-string-code") # type: ignore[arg-type] + except TypeError as exc: + context.transform_type_error = exc + + +@then("a TypeError should be raised from transform code type check") +def step_check_transform_code_type_error(context: Context) -> None: + assert context.transform_type_error is not None, "Expected TypeError" + assert isinstance(context.transform_type_error, TypeError) + + +@when("I try to create a transform executor with non-string tool name") +def step_create_transform_non_string_name(context: Context) -> None: + context.transform_name_type_error = None + try: + TransformExecutor("def transform(x): return {'passed': True}", 42) # type: ignore[arg-type] + except TypeError as exc: + context.transform_name_type_error = exc + + +@then("a TypeError should be raised from transform tool name check") +def step_check_transform_name_type_error(context: Context) -> None: + assert context.transform_name_type_error is not None, "Expected TypeError" + assert isinstance(context.transform_name_type_error, TypeError) + + +@given("a transform executor with code that raises an error during exec") +def step_transform_exec_error(context: Context) -> None: + # This code has a top-level expression that raises during exec + context.transform_executor = TransformExecutor( + "raise RuntimeError('bad code')\ndef transform(x):\n return {'passed': True}\n", + "test/exec-error", + ) + + +# ------------------------------------------------------------------- +# Additional coverage: WrappedToolExecutor init checks +# ------------------------------------------------------------------- + + +@when("I try to create a wrapped tool executor with None tool_lookup") +def step_create_executor_none_lookup(context: Context) -> None: + context.executor_init_error = None + try: + WrappedToolExecutor(None, lambda n, a: {}) # type: ignore[arg-type] + except ValueError as exc: + context.executor_init_error = exc + + +@then("a ValueError should be raised from executor construction for tool_lookup") +def step_check_executor_none_lookup_error(context: Context) -> None: + assert context.executor_init_error is not None, "Expected ValueError" + assert isinstance(context.executor_init_error, ValueError) + + +@when("I try to create a wrapped tool executor with None tool_executor") +def step_create_executor_none_executor(context: Context) -> None: + context.executor_init_error = None + try: + WrappedToolExecutor(lambda n: None, None) # type: ignore[arg-type] + except ValueError as exc: + context.executor_init_error = exc + + +@then("a ValueError should be raised from executor construction for tool_executor") +def step_check_executor_none_executor_error(context: Context) -> None: + assert context.executor_init_error is not None, "Expected ValueError" + assert isinstance(context.executor_init_error, ValueError) + + +@when("I try to create a wrapped tool executor with non-callable tool_lookup") +def step_create_executor_non_callable_lookup(context: Context) -> None: + context.executor_type_init_error = None + try: + WrappedToolExecutor("not_callable", lambda n, a: {}) # type: ignore[arg-type] + except TypeError as exc: + context.executor_type_init_error = exc + + +@then("a TypeError should be raised from executor construction for tool_lookup") +def step_check_executor_non_callable_lookup_error(context: Context) -> None: + assert context.executor_type_init_error is not None, "Expected TypeError" + assert isinstance(context.executor_type_init_error, TypeError) + + +@when("I try to create a wrapped tool executor with non-callable tool_executor") +def step_create_executor_non_callable_executor(context: Context) -> None: + context.executor_type_init_error = None + try: + WrappedToolExecutor(lambda n: None, "not_callable") # type: ignore[arg-type] + except TypeError as exc: + context.executor_type_init_error = exc + + +@then("a TypeError should be raised from executor construction for tool_executor") +def step_check_executor_non_callable_executor_error(context: Context) -> None: + assert context.executor_type_init_error is not None, "Expected TypeError" + assert isinstance(context.executor_type_init_error, TypeError) + + +# ------------------------------------------------------------------- +# Additional coverage: execute with non-dict inputs +# ------------------------------------------------------------------- + + +@when("I try to execute wrapping validation with non-dict inputs") +def step_try_execute_non_dict_inputs(context: Context) -> None: + context.inputs_type_error = None + val = None + for v in context.test_validations.values(): + if v.wraps is not None: + val = v + break + assert val is not None + try: + context.wrapped_executor.execute(val, "not_a_dict") # type: ignore[arg-type] + except TypeError as exc: + context.inputs_type_error = exc + + +@then("a TypeError should be raised for non-dict inputs") +def step_check_non_dict_inputs_error(context: Context) -> None: + assert context.inputs_type_error is not None, "Expected TypeError" + assert isinstance(context.inputs_type_error, TypeError) + + +# ------------------------------------------------------------------- +# Additional coverage: innermost wrapper with no wraps target +# ------------------------------------------------------------------- + + +# ------------------------------------------------------------------- +# ToolRunner coverage steps +# ------------------------------------------------------------------- + + +@given("a ToolRunner with a mock registry") +def step_tool_runner_mock_registry(context: Context) -> None: + from cleveragents.tool.registry import ToolRegistry + from cleveragents.tool.runner import ToolRunner + from cleveragents.tool.runtime import ToolSpec + + registry = ToolRegistry() + registry.register( + ToolSpec( + name="local/test-tool", + description="test", + handler=lambda inputs: {"result": "ok"}, + ) + ) + context.tool_runner = ToolRunner(registry) + + +@when("I call resolve_execution_environment on the runner") +def step_call_resolve_env(context: Context) -> None: + context.resolved_env = context.tool_runner.resolve_execution_environment() + + +@then("the resolved environment should be local") +def step_check_env_local(context: Context) -> None: + from cleveragents.domain.models.core.plan import ExecutionEnvironment + + assert context.resolved_env == ExecutionEnvironment.HOST + + +@given("a ToolRunner with a value-error-raising env resolver") +def step_runner_value_error_resolver(context: Context) -> None: + from unittest.mock import MagicMock + + from cleveragents.tool.registry import ToolRegistry + from cleveragents.tool.runner import ToolRunner + from cleveragents.tool.runtime import ToolSpec + + registry = ToolRegistry() + registry.register( + ToolSpec( + name="local/test-tool", + description="test", + handler=lambda inputs: {"result": "ok"}, + ) + ) + runner = ToolRunner(registry) + # Mock the resolver to raise ValueError + mock_resolver = MagicMock() + mock_resolver.resolve_and_validate.side_effect = ValueError("bad env config") + runner._env_resolver = mock_resolver + context.tool_runner = runner + + +@when("I execute a tool through the runner with env error") +def step_execute_runner_env_error(context: Context) -> None: + context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1}) + + +@then("the tool result should have success false") +def step_check_runner_result_fail(context: Context) -> None: + assert context.runner_result.success is False + + +@then('the tool result error should contain "{fragment}"') +def step_check_runner_result_error(context: Context, fragment: str) -> None: + assert fragment in (context.runner_result.error or ""), ( + f"Expected '{fragment}' in '{context.runner_result.error}'" + ) + + +@given("a ToolRunner with a container-returning env resolver") +def step_runner_container_resolver(context: Context) -> None: + from unittest.mock import MagicMock + + from cleveragents.domain.models.core.plan import ExecutionEnvironment + from cleveragents.tool.registry import ToolRegistry + from cleveragents.tool.runner import ToolRunner + from cleveragents.tool.runtime import ToolSpec + + registry = ToolRegistry() + registry.register( + ToolSpec( + name="local/test-tool", + description="test", + handler=lambda inputs: {"result": "ok"}, + ) + ) + runner = ToolRunner(registry) + # Mock the resolver to return CONTAINER + mock_resolver = MagicMock() + mock_resolver.resolve_and_validate.return_value = ExecutionEnvironment.CONTAINER + runner._env_resolver = mock_resolver + context.tool_runner = runner + + +@when("I execute a tool through the runner with container env") +def step_execute_runner_container(context: Context) -> None: + context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1}) + + +@when("I directly call _execute_chain with a no-wraps leaf") +def step_direct_execute_chain_no_wraps(context: Context) -> None: + context.chain_error = None + # Create a plain validation with wraps=None + leaf_val = _make_plain_validation("local/no-wraps-leaf") + try: + # Call the private method directly to exercise lines 437-438 + context.wrapped_executor._execute_chain([leaf_val], {}) + except ValueError as exc: + context.chain_error = exc + + +@then("a ValueError should be raised for missing wraps target") +def step_check_missing_wraps_target(context: Context) -> None: + assert context.chain_error is not None, "Expected ValueError" + assert isinstance(context.chain_error, ValueError) + assert "wraps" in str(context.chain_error).lower() diff --git a/features/tool_wrapping_runtime.feature b/features/tool_wrapping_runtime.feature new file mode 100644 index 000000000..cad826bba --- /dev/null +++ b/features/tool_wrapping_runtime.feature @@ -0,0 +1,220 @@ +Feature: Tool wrapping runtime + As the validation execution engine + I need to delegate execution to wrapped tools via wraps + transform + So that validations can reuse existing tool implementations + + # ── ArgumentMapper ──────────────────────────────────────────────── + + Scenario: ArgumentMapper with None mapping passes arguments through + Given an argument mapper with no mapping + When I apply the mapper to inputs {"path": "/src", "verbose": true} + Then the mapped arguments should be {"path": "/src", "verbose": true} + + Scenario: ArgumentMapper with mapping translates argument names + Given an argument mapper with mapping {"test_directory": "source_dir", "coverage_enabled": true} + When I apply the mapper to inputs {"source_dir": "/tests"} + Then the mapped arguments should be {"test_directory": "/tests", "coverage_enabled": true} + + Scenario: ArgumentMapper with literal values injects fixed values + Given an argument mapper with mapping {"mode": "strict", "count": 42} + When I apply the mapper to inputs {"extra": "ignored"} + Then the mapped arguments should be {"mode": "strict", "count": 42} + + Scenario: ArgumentMapper rejects non-dict inputs + Given an argument mapper with no mapping + When I try to apply the mapper to non-dict input "not_a_dict" + Then a TypeError should be raised from the argument mapper + + Scenario: ArgumentMapper rejects non-dict mapping at construction + When I try to create an argument mapper with a non-dict mapping + Then a TypeError should be raised from argument mapper construction + + # ── TransformExecutor ───────────────────────────────────────────── + + Scenario: TransformExecutor runs a valid transform function + Given a transform executor with code that checks returncode equals zero + When I execute the transform with tool output {"returncode": 0, "tests_run": 10} + Then the transform result should have passed true + And the transform result should have message "All tests passed" + + Scenario: TransformExecutor handles failing transform output + Given a transform executor with code that checks returncode equals zero + When I execute the transform with tool output {"returncode": 1, "tests_run": 10} + Then the transform result should have passed false + + Scenario: TransformExecutor raises on missing transform function + Given a transform executor with code that does not define a transform function + When I try to execute the transform with any output + Then a TransformExecutionError should be raised with message containing "callable" + + Scenario: TransformExecutor raises on non-dict return value + Given a transform executor with code that returns a non-dict value + When I try to execute the transform with any output + Then a TransformExecutionError should be raised with message containing "dict" + + Scenario: TransformExecutor raises on missing passed key in result + Given a transform executor with code that returns a dict without passed key + When I try to execute the transform with any output + Then a TransformExecutionError should be raised with message containing "passed" + + Scenario: TransformExecutor raises on empty transform code + When I try to create a transform executor with empty code + Then a ValueError should be raised from transform construction + + Scenario: TransformExecutor sandboxes dangerous operations + Given a transform executor with code that attempts to import os + When I try to execute the transform with any output + Then a TransformExecutionError should be raised from sandbox restriction + + # ── WrappedToolExecutor ─────────────────────────────────────────── + + Scenario: Simple wrapping delegates to the wrapped tool + Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0} + And a validation "local/tests-pass" that wraps "local/run-tests" with a simple transform + And a wrapped tool executor using the test registry + When I execute the wrapping validation with inputs {"path": "/src"} + Then the wrapped execution should succeed with passed true + And the wrapped tool "local/run-tests" should have been called + + Scenario: Argument mapping translates arguments to the wrapped tool + Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0} + And a validation "local/mapped-check" that wraps "local/run-tests" with argument mapping {"test_directory": "source_dir", "coverage_enabled": true} + And a wrapped tool executor using the test registry + When I execute the wrapping validation with inputs {"source_dir": "/tests"} + Then the wrapped tool "local/run-tests" should have received argument "test_directory" with value "/tests" + And the wrapped tool "local/run-tests" should have received argument "coverage_enabled" with value true + + Scenario: Chained wrapping delegates through the chain + Given a tool registry with a tool "local/base-tool" that returns {"status": "ok"} + And a validation "local/mid-wrapper" that wraps "local/base-tool" with a passthrough transform + And a validation "local/outer-wrapper" that wraps "local/mid-wrapper" with a status transform + And a wrapped tool executor using the test registry + When I execute the outer wrapping validation with inputs {} + Then the wrapped execution should succeed with passed true + And the wrapped tool "local/base-tool" should have been called + + Scenario: Missing wrapped tool raises WrappedToolNotFoundError + Given a validation "local/broken-wrap" that wraps "local/nonexistent" with a simple transform + And a wrapped tool executor with empty registry + When I try to execute the wrapping validation with inputs {} + Then a WrappedToolNotFoundError should be raised for "local/nonexistent" + + Scenario: Circular wrapping chain raises WrappingCycleError + Given a validation "local/wrap-a" that wraps "local/wrap-b" with a simple transform + And a validation "local/wrap-b" that wraps "local/wrap-a" with a simple transform + And a wrapped tool executor using the test registry with cycle + When I try to execute the wrapping validation with cycle from "local/wrap-a" + Then a WrappingCycleError should be raised + + Scenario: Execution context is inherited from wrapping validation + Given a tool registry with a tool "local/context-tool" that returns {"data": "value"} + And a validation "local/context-wrap" that wraps "local/context-tool" with a data transform + And a wrapped tool executor using the test registry + When I execute the wrapping validation with inputs {"key": "value"} + Then the wrapped tool should have received the validation inputs + + Scenario: WrappedToolExecutor rejects non-Validation argument + Given a wrapped tool executor with empty registry + When I try to execute with a non-Validation object + Then a TypeError should be raised from the executor + + Scenario: WrappedToolExecutor rejects validation without wraps + Given a validation "local/no-wrap" without wraps set + And a wrapped tool executor with empty registry + When I try to execute the non-wrapping validation + Then a ValueError should be raised indicating wraps is not set + + # ── Additional coverage: WrappingDepthExceededError ────────────── + + Scenario: WrappingDepthExceededError is raised when chain is too deep + Given a deep wrapping chain of 11 validations + And a wrapped tool executor using the deep chain registry + When I try to execute the deep chain wrapping validation + Then a WrappingDepthExceededError should be raised with depth 10 + + # ── Additional coverage: ArgumentMapper.mapping property ───────── + + Scenario: ArgumentMapper exposes the raw mapping via property + Given an argument mapper with mapping {"x": "y"} + Then the mapper mapping property should return {"x": "y"} + + Scenario: ArgumentMapper mapping property returns None for identity mapper + Given an argument mapper with no mapping + Then the mapper mapping property should be None + + # ── Additional coverage: TransformExecutor type checks ─────────── + + Scenario: TransformExecutor rejects non-string transform code + When I try to create a transform executor with non-string code + Then a TypeError should be raised from transform code type check + + Scenario: TransformExecutor rejects non-string tool name + When I try to create a transform executor with non-string tool name + Then a TypeError should be raised from transform tool name check + + Scenario: TransformExecutor raises on code that throws during exec + Given a transform executor with code that raises an error during exec + When I try to execute the transform with any output + Then a TransformExecutionError should be raised with message containing "compile" + + # ── Additional coverage: WrappedToolExecutor init checks ───────── + + Scenario: WrappedToolExecutor rejects None tool_lookup + When I try to create a wrapped tool executor with None tool_lookup + Then a ValueError should be raised from executor construction for tool_lookup + + Scenario: WrappedToolExecutor rejects None tool_executor + When I try to create a wrapped tool executor with None tool_executor + Then a ValueError should be raised from executor construction for tool_executor + + Scenario: WrappedToolExecutor rejects non-callable tool_lookup + When I try to create a wrapped tool executor with non-callable tool_lookup + Then a TypeError should be raised from executor construction for tool_lookup + + Scenario: WrappedToolExecutor rejects non-callable tool_executor + When I try to create a wrapped tool executor with non-callable tool_executor + Then a TypeError should be raised from executor construction for tool_executor + + # ── Additional coverage: execute with non-dict inputs ──────────── + + Scenario: WrappedToolExecutor rejects non-dict inputs + Given a tool registry with a tool "local/some-tool" that returns {"ok": true} + And a validation "local/input-wrap" that wraps "local/some-tool" with a simple transform + And a wrapped tool executor using the test registry + When I try to execute wrapping validation with non-dict inputs + Then a TypeError should be raised for non-dict inputs + + # ── Additional coverage: chain with None wraps at end ──────────── + + Scenario: Chain resolution handles validation with wraps set to None in chain + Given a tool registry with a tool "local/leaf" that returns {"value": 1} + And a validation "local/wrapper-no-inner" that wraps "local/leaf" with a passthrough transform + And a wrapped tool executor using the test registry + When I execute the wrapping validation with inputs {} + Then the wrapped execution should succeed with passed true + + # ── Additional coverage: _execute_chain with None wraps target ──── + + Scenario: _execute_chain raises ValueError when leaf wraps is None + Given a wrapped tool executor with empty registry + When I directly call _execute_chain with a no-wraps leaf + Then a ValueError should be raised for missing wraps target + + # ── ToolRunner coverage: execution environment and error paths ─── + + Scenario: ToolRunner resolve_execution_environment delegates to resolver + Given a ToolRunner with a mock registry + When I call resolve_execution_environment on the runner + Then the resolved environment should be local + + Scenario: ToolRunner execute returns error when env resolver raises ValueError + Given a ToolRunner with a value-error-raising env resolver + When I execute a tool through the runner with env error + Then the tool result should have success false + And the tool result error should contain "Execution environment error" + + Scenario: ToolRunner execute returns error for container environment + Given a ToolRunner with a container-returning env resolver + When I execute a tool through the runner with container env + Then the tool result should have success false + And the tool result error should contain "Container execution is not yet implemented" diff --git a/robot/scripts/test_delegation.py b/robot/scripts/test_delegation.py new file mode 100644 index 000000000..6a83f23e2 --- /dev/null +++ b/robot/scripts/test_delegation.py @@ -0,0 +1,42 @@ +"""Robot helper: verify WrappedToolExecutor delegates to wrapped tool.""" + +from cleveragents.domain.models.core.tool import ( + Tool, + ToolCapability, + ToolSource, + ToolType, + Validation, + ValidationMode, +) +from cleveragents.tool.wrapping import WrappedToolExecutor + +tool = Tool( + name="local/run", + description="run", + source=ToolSource.BUILTIN, + capability=ToolCapability(read_only=True), +) +val = Validation( + name="local/check", + description="check", + source=ToolSource.WRAPPED, + tool_type=ToolType.VALIDATION, + mode=ValidationMode.REQUIRED, + wraps="local/run", + transform='def transform(x):\n return {"passed": True, "data": x}\n', +) +lookup = lambda n: tool if n == "local/run" else None # noqa: E731 +called: list[str] = [] + + +def executor_fn(n: str, a: dict) -> dict: # type: ignore[type-arg] + """Record tool call and return output.""" + called.append(n) + return {"result": "ok"} + + +ex = WrappedToolExecutor(lookup, executor_fn) +r = ex.execute(val, {"a": 1}) +assert r["passed"] is True +assert "local/run" in called +print("OK") diff --git a/robot/scripts/test_mapper_configured.py b/robot/scripts/test_mapper_configured.py new file mode 100644 index 000000000..4dd033e41 --- /dev/null +++ b/robot/scripts/test_mapper_configured.py @@ -0,0 +1,8 @@ +"""Robot helper: verify ArgumentMapper translates arguments.""" + +from cleveragents.tool.wrapping import ArgumentMapper + +m = ArgumentMapper({"dest": "src", "flag": True}) +r = m.apply({"src": "/path"}) +assert r == {"dest": "/path", "flag": True}, f"Got {r}" +print("OK") diff --git a/robot/scripts/test_mapper_identity.py b/robot/scripts/test_mapper_identity.py new file mode 100644 index 000000000..f4951b0bc --- /dev/null +++ b/robot/scripts/test_mapper_identity.py @@ -0,0 +1,8 @@ +"""Robot helper: verify ArgumentMapper with None mapping.""" + +from cleveragents.tool.wrapping import ArgumentMapper + +m = ArgumentMapper(None) +r = m.apply({"x": 1, "y": "two"}) +assert r == {"x": 1, "y": "two"}, f"Got {r}" +print("OK") diff --git a/robot/scripts/test_missing_tool.py b/robot/scripts/test_missing_tool.py new file mode 100644 index 000000000..3484f94c6 --- /dev/null +++ b/robot/scripts/test_missing_tool.py @@ -0,0 +1,28 @@ +"""Robot helper: verify WrappedToolExecutor raises on missing tool.""" + +from cleveragents.domain.models.core.tool import ( + ToolSource, + ToolType, + Validation, + ValidationMode, +) +from cleveragents.tool.wrapping import WrappedToolExecutor, WrappedToolNotFoundError + +val = Validation( + name="local/check", + description="check", + source=ToolSource.WRAPPED, + tool_type=ToolType.VALIDATION, + mode=ValidationMode.REQUIRED, + wraps="local/missing", + transform='def transform(x):\n return {"passed": True}\n', +) +ok = False +try: + WrappedToolExecutor( + lambda n: None, + lambda n, a: {}, + ).execute(val, {}) +except WrappedToolNotFoundError: + ok = True +print(f"error_raised={ok}") diff --git a/robot/scripts/test_transform_run.py b/robot/scripts/test_transform_run.py new file mode 100644 index 000000000..900f44b0d --- /dev/null +++ b/robot/scripts/test_transform_run.py @@ -0,0 +1,9 @@ +"""Robot helper: verify TransformExecutor runs a transform.""" + +from cleveragents.tool.wrapping import TransformExecutor + +code = 'def transform(x):\n return {"passed": x.get("ok"), "message": "done"}\n' +t = TransformExecutor(code, "test/t") +r = t.execute({"ok": True}) +assert r["passed"] is True +print("OK") diff --git a/robot/scripts/test_transform_sandbox.py b/robot/scripts/test_transform_sandbox.py new file mode 100644 index 000000000..513364900 --- /dev/null +++ b/robot/scripts/test_transform_sandbox.py @@ -0,0 +1,12 @@ +"""Robot helper: verify TransformExecutor blocks import in sandbox.""" + +from cleveragents.tool.wrapping import TransformExecutionError, TransformExecutor + +code = 'def transform(x):\n import os\n return {"passed": True}\n' +t = TransformExecutor(code, "test/s") +ok = False +try: + t.execute({}) +except TransformExecutionError: + ok = True +print(f"blocked={ok}") diff --git a/robot/tool_wrapping_runtime.robot b/robot/tool_wrapping_runtime.robot new file mode 100644 index 000000000..5046e20f1 --- /dev/null +++ b/robot/tool_wrapping_runtime.robot @@ -0,0 +1,58 @@ +*** Settings *** +Documentation Tool wrapping runtime smoke tests +Library Process +Library OperatingSystem + +*** Variables *** +${PYTHON} python + +*** Test Cases *** +Wrapping Package Is Importable + [Documentation] Verify tool wrapping module can be imported + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool.wrapping import ArgumentMapper, TransformExecutor, WrappedToolExecutor, WrappedToolNotFoundError, WrappingCycleError, WrappingDepthExceededError, TransformExecutionError; print('OK') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} OK + +Wrapping Exports In Tool Init + [Documentation] Verify wrapping types are exported from tool package + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool import ArgumentMapper, TransformExecutor, WrappedToolExecutor; print('OK') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} OK + +ArgumentMapper Identity Mapping + [Documentation] Verify ArgumentMapper with None mapping passes args through + ${result}= Run Process ${PYTHON} robot/scripts/test_mapper_identity.py + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + Should Contain ${result.stdout} OK + +ArgumentMapper With Mapping + [Documentation] Verify ArgumentMapper translates arguments + ${result}= Run Process ${PYTHON} robot/scripts/test_mapper_configured.py + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + Should Contain ${result.stdout} OK + +TransformExecutor Runs Transform + [Documentation] Verify TransformExecutor executes transform code + ${result}= Run Process ${PYTHON} robot/scripts/test_transform_run.py + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + Should Contain ${result.stdout} OK + +TransformExecutor Sandbox Blocks Import + [Documentation] Verify TransformExecutor blocks import in sandbox + ${result}= Run Process ${PYTHON} robot/scripts/test_transform_sandbox.py + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + Should Contain ${result.stdout} blocked=True + +WrappedToolExecutor Simple Delegation + [Documentation] Verify WrappedToolExecutor delegates to wrapped tool + ${result}= Run Process ${PYTHON} robot/scripts/test_delegation.py + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + Should Contain ${result.stdout} OK + +WrappedToolExecutor Missing Tool Error + [Documentation] Verify WrappedToolExecutor raises on missing wrapped tool + ${result}= Run Process ${PYTHON} robot/scripts/test_missing_tool.py + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + Should Contain ${result.stdout} error_raised=True diff --git a/src/cleveragents/tool/__init__.py b/src/cleveragents/tool/__init__.py index 5e5395d0b..b2c0d8082 100644 --- a/src/cleveragents/tool/__init__.py +++ b/src/cleveragents/tool/__init__.py @@ -69,8 +69,18 @@ from cleveragents.tool.schema_validator import ( validate_tool_input, validate_tool_output, ) +from cleveragents.tool.wrapping import ( + ArgumentMapper, + TransformExecutionError, + TransformExecutor, + WrappedToolExecutor, + WrappedToolNotFoundError, + WrappingCycleError, + WrappingDepthExceededError, +) __all__ = [ + "ArgumentMapper", "BoundResource", "CancellationToken", "Change", @@ -116,6 +126,12 @@ __all__ = [ "ToolSandboxRequiredError", "ToolSchemaValidationError", "ToolSpec", + "TransformExecutionError", + "TransformExecutor", + "WrappedToolExecutor", + "WrappedToolNotFoundError", + "WrappingCycleError", + "WrappingDepthExceededError", "classify_tool_error", "detect_provider_format", "generate_tool_call_id", diff --git a/src/cleveragents/tool/wrapping.py b/src/cleveragents/tool/wrapping.py new file mode 100644 index 000000000..0566a3a10 --- /dev/null +++ b/src/cleveragents/tool/wrapping.py @@ -0,0 +1,470 @@ +"""Tool wrapping runtime for CleverAgents v3. + +Implements the ``wraps`` + ``transform`` + ``argument_mapping`` delegation +mechanism described in the specification (§ Core Concepts > Validation > +Tool Wrapping). + +## Architecture + +``` +WrappedToolExecutor + |-- ArgumentMapper (translate wrapper args -> wrapped tool args) + |-- TransformExecutor (sandboxed transform function on tool output) + +-- ToolRuntime (resolve + invoke wrapped tools) +``` + +## Delegation Chain + +A validation wrapping another validation forms a delegation chain: + +``` +validation-A -> validation-B -> tool-C +(wraps B) (wraps C) (leaf) +``` + +``WrappedToolExecutor.execute()`` recursively resolves the chain, +detecting cycles and enforcing a maximum depth. + +## Sandbox + +``TransformExecutor`` compiles and executes the user-supplied +``transform`` function with a restricted set of built-in names. +No filesystem, network, or import access is available inside the +transform. + +Based on ``docs/specification.md`` § Tool Wrapping. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from cleveragents.domain.models.core.tool import Validation + +logger = logging.getLogger(__name__) + +# Maximum delegation chain depth to prevent infinite recursion. +MAX_WRAPPING_DEPTH = 10 + +# Safe built-ins available inside the ``transform`` sandbox. +_SAFE_BUILTINS: dict[str, Any] = { + "True": True, + "False": False, + "None": None, + "abs": abs, + "all": all, + "any": any, + "bool": bool, + "dict": dict, + "enumerate": enumerate, + "float": float, + "frozenset": frozenset, + "int": int, + "isinstance": isinstance, + "len": len, + "list": list, + "map": map, + "max": max, + "min": min, + "range": range, + "repr": repr, + "reversed": reversed, + "round": round, + "set": set, + "sorted": sorted, + "str": str, + "sum": sum, + "tuple": tuple, + "type": type, + "zip": zip, +} + + +# ------------------------------------------------------------------- +# Errors +# ------------------------------------------------------------------- + + +class WrappedToolNotFoundError(Exception): + """Raised when the tool referenced by ``wraps`` is not registered.""" + + def __init__(self, tool_name: str, wraps_name: str) -> None: + self.tool_name = tool_name + self.wraps_name = wraps_name + super().__init__( + f"Validation '{tool_name}' wraps '{wraps_name}' but " + f"'{wraps_name}' is not registered in the runtime" + ) + + +class WrappingCycleError(Exception): + """Raised when a circular wrapping chain is detected.""" + + def __init__(self, chain: list[str]) -> None: + self.chain = list(chain) + cycle_str = " -> ".join(chain) + super().__init__(f"Circular wrapping chain detected: {cycle_str}") + + +class WrappingDepthExceededError(Exception): + """Raised when the delegation chain exceeds ``MAX_WRAPPING_DEPTH``.""" + + def __init__(self, depth: int) -> None: + self.depth = depth + super().__init__( + f"Wrapping delegation chain depth ({depth}) exceeds " + f"the maximum ({MAX_WRAPPING_DEPTH})" + ) + + +class TransformExecutionError(Exception): + """Raised when the ``transform`` function fails during execution.""" + + def __init__(self, tool_name: str, detail: str) -> None: + self.tool_name = tool_name + self.detail = detail + super().__init__(f"Transform function for '{tool_name}' failed: {detail}") + + +# ------------------------------------------------------------------- +# ArgumentMapper +# ------------------------------------------------------------------- + + +class ArgumentMapper: + """Applies ``argument_mapping`` to translate arguments. + + When a validation wraps a tool, the validation may accept different + input arguments. The ``argument_mapping`` dictionary specifies how + the validation's inputs map to the wrapped tool's expected inputs. + + Keys are the wrapped tool's parameter names. Values are either: + - A string referencing a validation input parameter (forwarded). + - A literal value (string, number, boolean) always sent to the + wrapped tool. + + When ``argument_mapping`` is ``None``, input arguments are passed + through to the wrapped tool unchanged (identity mapping). + """ + + def __init__(self, mapping: dict[str, Any] | None) -> None: + if mapping is not None and not isinstance(mapping, dict): + raise TypeError( + f"argument_mapping must be a dict or None, got {type(mapping).__name__}" + ) + self._mapping = mapping + + @property + def mapping(self) -> dict[str, Any] | None: + """Return the raw mapping configuration.""" + return self._mapping + + def apply(self, validation_inputs: dict[str, Any]) -> dict[str, Any]: + """Translate *validation_inputs* into wrapped-tool arguments. + + Parameters + ---------- + validation_inputs: + The arguments the validation was invoked with. + + Returns + ------- + dict[str, Any]: + Arguments to forward to the wrapped tool. + """ + if not isinstance(validation_inputs, dict): + raise TypeError( + f"validation_inputs must be a dict, " + f"got {type(validation_inputs).__name__}" + ) + if self._mapping is None: + return dict(validation_inputs) + + mapped: dict[str, Any] = {} + for wrapped_param, source in self._mapping.items(): + if isinstance(source, str) and source in validation_inputs: + mapped[wrapped_param] = validation_inputs[source] + else: + # Literal value (or string not present as a key -- use + # as literal). + mapped[wrapped_param] = source + return mapped + + +# ------------------------------------------------------------------- +# TransformExecutor +# ------------------------------------------------------------------- + + +class TransformExecutor: + """Executes the ``transform`` function in a sandboxed context. + + The transform code must define a function called ``transform`` that + accepts one positional argument (the wrapped tool's output) and + returns a dict with at minimum ``{"passed": bool}``. + + The sandbox disallows imports, file I/O, and network access by + restricting available built-ins. + """ + + def __init__(self, transform_code: str, tool_name: str) -> None: + if not isinstance(transform_code, str): + raise TypeError( + f"transform_code must be a str, got {type(transform_code).__name__}" + ) + if not transform_code.strip(): + raise ValueError("transform_code must not be empty") + if not isinstance(tool_name, str): + raise TypeError(f"tool_name must be a str, got {type(tool_name).__name__}") + self._code = transform_code + self._tool_name = tool_name + self._compiled = compile(self._code, f"", "exec") + + def execute(self, tool_output: Any) -> dict[str, Any]: + """Run the transform on *tool_output*. + + Returns + ------- + dict[str, Any]: + The validation-format result (must contain ``passed``). + + Raises + ------ + TransformExecutionError: + If the transform function is missing, raises, or returns + an invalid result. + """ + sandbox: dict[str, Any] = {"__builtins__": dict(_SAFE_BUILTINS)} + + try: + exec(self._compiled, sandbox) # Controlled sandbox execution + except Exception as exc: + raise TransformExecutionError( + self._tool_name, + f"Failed to compile/execute transform code: {exc}", + ) from exc + + transform_fn = sandbox.get("transform") + if transform_fn is None or not callable(transform_fn): + raise TransformExecutionError( + self._tool_name, + "Transform code must define a callable 'transform' function", + ) + + try: + result = transform_fn(tool_output) + except Exception as exc: + raise TransformExecutionError( + self._tool_name, + f"Transform function raised: {type(exc).__name__}: {exc}", + ) from exc + + if not isinstance(result, dict): + raise TransformExecutionError( + self._tool_name, + f"Transform must return a dict, got {type(result).__name__}", + ) + + if "passed" not in result: + raise TransformExecutionError( + self._tool_name, + "Transform result must contain a 'passed' key", + ) + + return result + + +# ------------------------------------------------------------------- +# WrappedToolExecutor +# ------------------------------------------------------------------- + + +class WrappedToolExecutor: + """Resolves ``wraps`` references and delegates execution. + + Orchestrates the full wrapping flow: + + 1. Resolve the wrapped tool from the tool lookup. + 2. If the wrapped tool is itself a wrapping validation, recurse. + 3. Apply ``argument_mapping`` to translate arguments. + 4. Execute the leaf tool. + 5. Walk back up the chain applying ``transform`` at each level. + + Parameters + ---------- + tool_lookup: + A callable that accepts a tool name (str) and returns + the ``Validation`` or ``Tool`` domain model, or ``None`` + if not found. + tool_executor: + A callable ``(tool_name, mapped_args) -> dict`` that + actually runs the leaf tool and returns its raw output. + """ + + def __init__( + self, + tool_lookup: Any, + tool_executor: Any, + ) -> None: + if tool_lookup is None: + raise ValueError("tool_lookup must not be None") + if tool_executor is None: + raise ValueError("tool_executor must not be None") + if not callable(tool_lookup): + raise TypeError( + f"tool_lookup must be callable, got {type(tool_lookup).__name__}" + ) + if not callable(tool_executor): + raise TypeError( + f"tool_executor must be callable, got {type(tool_executor).__name__}" + ) + self._tool_lookup = tool_lookup + self._tool_executor = tool_executor + + def execute( + self, + validation: Validation, + inputs: dict[str, Any], + ) -> dict[str, Any]: + """Execute a wrapping validation. + + Parameters + ---------- + validation: + The wrapping ``Validation`` (must have ``wraps`` set). + inputs: + The arguments the validation was invoked with. + + Returns + ------- + dict[str, Any]: + The validation-format result with ``passed``, optionally + ``message`` and ``data``. + + Raises + ------ + WrappedToolNotFoundError: + If any tool in the chain is not registered. + WrappingCycleError: + If the chain contains a cycle. + WrappingDepthExceededError: + If the chain exceeds ``MAX_WRAPPING_DEPTH``. + TransformExecutionError: + If any transform function in the chain fails. + """ + if not isinstance(validation, Validation): + raise TypeError( + f"validation must be a Validation, got {type(validation).__name__}" + ) + if not isinstance(inputs, dict): + raise TypeError(f"inputs must be a dict, got {type(inputs).__name__}") + if validation.wraps is None: + raise ValueError( + f"Validation '{validation.name}' does not have " + f"'wraps' set -- use direct execution instead" + ) + + chain = self._resolve_chain(validation) + return self._execute_chain(chain, inputs) + + # -- Chain resolution -------------------------------------------------- + + def _resolve_chain( + self, + validation: Validation, + ) -> list[Validation]: + """Walk the wraps chain and return the ordered list. + + The first element is the outermost wrapper, the last is the + innermost wrapper (whose ``wraps`` target is the leaf tool). + """ + chain: list[Validation] = [] + visited: set[str] = set() + current: Validation = validation + + while True: + if len(chain) >= MAX_WRAPPING_DEPTH: + raise WrappingDepthExceededError(len(chain)) + + if current.name in visited: + cycle = [v.name for v in chain] + [current.name] + raise WrappingCycleError(cycle) + + visited.add(current.name) + chain.append(current) + + wraps_name = current.wraps + if wraps_name is None: + break + + wrapped = self._tool_lookup(wraps_name) + if wrapped is None: + raise WrappedToolNotFoundError(current.name, wraps_name) + + if isinstance(wrapped, Validation) and wrapped.wraps is not None: + current = wrapped + else: + # Leaf tool found -- stop the chain + break + + return chain + + # -- Chain execution --------------------------------------------------- + + def _execute_chain( + self, + chain: list[Validation], + inputs: dict[str, Any], + ) -> dict[str, Any]: + """Execute the delegation chain. + + 1. Walk from outermost to innermost, applying argument mappings. + 2. Execute the leaf tool. + 3. Walk back from innermost to outermost, applying transforms. + """ + # Build the fully-mapped arguments by applying each wrapper's + # argument_mapping in sequence. + mapped_args = dict(inputs) + for wrapper in chain: + mapper = ArgumentMapper(wrapper.argument_mapping) + mapped_args = mapper.apply(mapped_args) + + # Execute the leaf tool (the target of the innermost wraps) + leaf_wrapper = chain[-1] + leaf_tool_name = leaf_wrapper.wraps + if leaf_tool_name is None: + raise ValueError( + f"Innermost wrapper '{leaf_wrapper.name}' has no 'wraps' target" + ) + + logger.info( + "Executing wrapped tool", + extra={ + "chain": [v.name for v in chain], + "leaf_tool": leaf_tool_name, + }, + ) + + raw_output = self._tool_executor(leaf_tool_name, mapped_args) + + # Apply transforms in reverse (innermost first, outermost last) + result: dict[str, Any] = ( + raw_output if isinstance(raw_output, dict) else {"raw": raw_output} + ) + for wrapper in reversed(chain): + if wrapper.transform is not None: + transform_exec = TransformExecutor( + wrapper.transform, + wrapper.name, + ) + logger.debug( + "Applying transform", + extra={ + "wrapper": wrapper.name, + "input_type": type(result).__name__, + }, + ) + result = transform_exec.execute(result) + + return result