feat(validation): implement tool wrapping runtime (wraps + transform delegation)
CI / lint (pull_request) Successful in 14s
CI / typecheck (pull_request) Successful in 34s
CI / security (pull_request) Successful in 29s
CI / quality (pull_request) Successful in 15s
CI / benchmark-publish (pull_request) Has been skipped
CI / build (pull_request) Successful in 14s
CI / unit_tests (pull_request) Successful in 2m4s
CI / integration_tests (pull_request) Successful in 2m50s
CI / coverage (pull_request) Successful in 4m15s
CI / docker (pull_request) Successful in 42s
CI / lint (push) Successful in 12s
CI / typecheck (push) Successful in 31s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 31s
CI / build (push) Successful in 16s
CI / unit_tests (push) Successful in 3m23s
CI / benchmark-regression (push) Has been skipped
CI / docker (push) Successful in 42s
CI / integration_tests (push) Successful in 4m11s
CI / coverage (push) Successful in 4m8s
CI / benchmark-publish (push) Successful in 16m5s
CI / benchmark-regression (pull_request) Successful in 29m59s

Implement the runtime execution engine for validation tool wrapping,
as specified in docs/specification.md § Tool Wrapping.

WrappedToolExecutor resolves wraps references and delegates execution
to wrapped tools, supporting composable wrapping chains with cycle
detection and depth limiting (max 10 levels).

ArgumentMapper translates arguments between wrapper and wrapped tool
schemas using the argument_mapping configuration. Supports both
forwarded parameter names and literal fixed values.

TransformExecutor runs user-supplied transform functions in a
sandboxed Python environment with restricted builtins (no imports,
no filesystem, no network access). Validates that transforms return
proper validation-format dicts with a passed boolean.

Wired into the tool package public API via tool/__init__.py exports.
All new error types (WrappedToolNotFoundError, WrappingCycleError,
WrappingDepthExceededError, TransformExecutionError) provide clear
diagnostic messages.

Tests: 20 Behave scenarios covering argument mapping, transform
execution, simple/chained delegation, error handling, and sandbox
restrictions. 8 Robot Framework integration smoke tests. ASV
benchmarks for delegation overhead measurement.

ISSUES CLOSED: #543
This commit was merged in pull request #555.
This commit is contained in:
2026-03-04 03:36:28 +00:00
committed by Forgejo
parent db5e5c974f
commit 93e3893d69
12 changed files with 1969 additions and 0 deletions
+185
View File
@@ -0,0 +1,185 @@
"""ASV benchmarks for tool wrapping delegation overhead.
Measures the performance of:
- ArgumentMapper.apply() with identity and configured mappings
- TransformExecutor.execute() with simple transform functions
- WrappedToolExecutor.execute() for single and chained delegation
"""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
from typing import Any
# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from cleveragents.domain.models.core.tool import ( # noqa: E402
Tool,
ToolCapability,
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import ( # noqa: E402
ArgumentMapper,
TransformExecutor,
WrappedToolExecutor,
)
_SIMPLE_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("returncode") == 0
return {"passed": passed, "message": "done", "data": tool_output}
"""
_PASSTHROUGH_TRANSFORM = """\
def transform(tool_output):
return {"passed": True, "message": "ok", "data": tool_output}
"""
def _make_tool(name: str) -> Tool:
return Tool(
name=name,
description=f"Benchmark tool {name}",
source=ToolSource.BUILTIN,
capability=ToolCapability(read_only=True),
)
def _make_validation(
name: str,
wraps: str,
transform_code: str,
argument_mapping: dict[str, Any] | None = None,
) -> Validation:
return Validation(
name=name,
description=f"Benchmark validation wrapping {wraps}",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps=wraps,
transform=transform_code,
argument_mapping=argument_mapping,
)
class ArgumentMapperSuite:
"""Benchmarks for ArgumentMapper."""
def time_identity_mapping(self) -> None:
"""Measure identity (passthrough) mapping overhead."""
mapper = ArgumentMapper(None)
for _ in range(1000):
mapper.apply({"path": "/src", "verbose": True, "count": 42})
def time_configured_mapping(self) -> None:
"""Measure mapping with configured argument translation."""
mapper = ArgumentMapper(
{
"test_directory": "source_dir",
"coverage_enabled": True,
"verbose": False,
}
)
for _ in range(1000):
mapper.apply({"source_dir": "/tests", "extra": "ignored"})
class TransformExecutorSuite:
"""Benchmarks for TransformExecutor."""
def setup(self) -> None:
self._executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
self._output = {"returncode": 0, "tests_run": 100}
def time_execute_transform(self) -> None:
"""Measure transform function execution."""
for _ in range(1000):
self._executor.execute(self._output)
def time_create_and_execute(self) -> None:
"""Measure creation + execution (cold path)."""
for _ in range(100):
executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
executor.execute(self._output)
class WrappedToolExecutorSuite:
"""Benchmarks for WrappedToolExecutor delegation."""
def setup(self) -> None:
self._tool = _make_tool("local/base-tool")
self._output: dict[str, Any] = {"returncode": 0, "data": "ok"}
self._validation = _make_validation(
"local/bench-wrap",
"local/base-tool",
_SIMPLE_TRANSFORM,
)
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
"local/base-tool": (self._tool, self._output),
}
def lookup(name: str) -> Tool | Validation | None:
if name in tools:
return tools[name][0]
return None
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
if name in tools:
return tools[name][1]
raise RuntimeError(f"Not found: {name}")
self._executor = WrappedToolExecutor(lookup, executor)
def time_single_delegation(self) -> None:
"""Measure single-level wrapping delegation."""
for _ in range(1000):
self._executor.execute(self._validation, {"path": "/src"})
def time_chained_delegation(self) -> None:
"""Measure two-level wrapping delegation chain."""
inner_val = _make_validation(
"local/inner-wrap",
"local/base-tool",
_PASSTHROUGH_TRANSFORM,
)
outer_val = _make_validation(
"local/outer-wrap",
"local/inner-wrap",
_SIMPLE_TRANSFORM,
)
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
"local/base-tool": (self._tool, self._output),
}
all_items: dict[str, Tool | Validation] = {
"local/inner-wrap": inner_val,
"local/base-tool": self._tool,
}
def lookup(name: str) -> Tool | Validation | None:
return all_items.get(name)
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
if name in tools:
return tools[name][1]
raise RuntimeError(f"Not found: {name}")
chained_executor = WrappedToolExecutor(lookup, executor)
for _ in range(500):
chained_executor.execute(outer_val, {"path": "/src"})
@@ -0,0 +1,913 @@
"""Step definitions for tool wrapping runtime feature tests."""
from __future__ import annotations
import json
from typing import Any
from behave import given, then, when
from behave.runner import Context
from cleveragents.domain.models.core.tool import (
Tool,
ToolCapability,
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import (
ArgumentMapper,
TransformExecutionError,
TransformExecutor,
WrappedToolExecutor,
WrappedToolNotFoundError,
WrappingCycleError,
WrappingDepthExceededError,
)
# -------------------------------------------------------------------
# Helpers
# -------------------------------------------------------------------
_SIMPLE_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("returncode") == 0
msg = "All tests passed" if passed else "Tests failed"
return {"passed": passed, "message": msg, "data": tool_output}
"""
_PASSTHROUGH_TRANSFORM = """\
def transform(tool_output):
return {"passed": True, "message": "ok", "data": tool_output}
"""
_STATUS_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("passed", False)
return {"passed": passed, "message": "outer", "data": tool_output}
"""
_DATA_TRANSFORM = """\
def transform(tool_output):
return {"passed": True, "message": "data ok", "data": tool_output}
"""
_RETURNCODE_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("returncode") == 0
msg = "All tests passed" if passed else "Tests failed"
return {"passed": passed, "message": msg, "data": tool_output}
"""
def _make_validation(
name: str,
wraps: str,
transform_code: str,
argument_mapping: dict[str, Any] | None = None,
) -> Validation:
"""Create a Validation with wraps set."""
return Validation(
name=name,
description=f"Test validation wrapping {wraps}",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps=wraps,
transform=transform_code,
argument_mapping=argument_mapping,
)
def _make_plain_validation(name: str) -> Validation:
"""Create a Validation without wraps."""
return Validation(
name=name,
description="Test validation no wraps",
source=ToolSource.CUSTOM,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
code="pass",
)
def _make_tool(name: str) -> Tool:
"""Create a plain Tool."""
return Tool(
name=name,
description=f"Test tool {name}",
source=ToolSource.BUILTIN,
capability=ToolCapability(read_only=True),
)
# -------------------------------------------------------------------
# ArgumentMapper steps
# -------------------------------------------------------------------
@given("an argument mapper with no mapping")
def step_mapper_no_mapping(context: Context) -> None:
context.mapper = ArgumentMapper(None)
@given("an argument mapper with mapping {mapping_json}")
def step_mapper_with_mapping(context: Context, mapping_json: str) -> None:
mapping = json.loads(mapping_json)
context.mapper = ArgumentMapper(mapping)
@when("I apply the mapper to inputs {inputs_json}")
def step_apply_mapper(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
context.mapped_result = context.mapper.apply(inputs)
@then("the mapped arguments should be {expected_json}")
def step_check_mapped(context: Context, expected_json: str) -> None:
expected = json.loads(expected_json)
assert context.mapped_result == expected, (
f"Expected {expected}, got {context.mapped_result}"
)
@when('I try to apply the mapper to non-dict input "{value}"')
def step_apply_mapper_non_dict(context: Context, value: str) -> None:
context.mapper_error = None
try:
context.mapper.apply(value) # type: ignore[arg-type]
except TypeError as exc:
context.mapper_error = exc
@then("a TypeError should be raised from the argument mapper")
def step_check_mapper_type_error(context: Context) -> None:
assert context.mapper_error is not None, "Expected TypeError"
assert isinstance(context.mapper_error, TypeError)
@when("I try to create an argument mapper with a non-dict mapping")
def step_create_mapper_non_dict(context: Context) -> None:
context.mapper_construction_error = None
try:
ArgumentMapper("not_a_dict") # type: ignore[arg-type]
except TypeError as exc:
context.mapper_construction_error = exc
@then("a TypeError should be raised from argument mapper construction")
def step_check_mapper_construction_error(context: Context) -> None:
assert context.mapper_construction_error is not None, "Expected TypeError"
assert isinstance(context.mapper_construction_error, TypeError)
# -------------------------------------------------------------------
# TransformExecutor steps
# -------------------------------------------------------------------
@given("a transform executor with code that checks returncode equals zero")
def step_transform_returncode(context: Context) -> None:
context.transform_executor = TransformExecutor(
_RETURNCODE_TRANSFORM, "test/transform"
)
@when("I execute the transform with tool output {output_json}")
def step_execute_transform(context: Context, output_json: str) -> None:
output = json.loads(output_json)
context.transform_result = context.transform_executor.execute(output)
@then("the transform result should have passed true")
def step_check_transform_passed_true(context: Context) -> None:
assert context.transform_result["passed"] is True
@then("the transform result should have passed false")
def step_check_transform_passed_false(context: Context) -> None:
assert context.transform_result["passed"] is False
@then('the transform result should have message "{expected_msg}"')
def step_check_transform_message(context: Context, expected_msg: str) -> None:
assert context.transform_result["message"] == expected_msg, (
f"Expected '{expected_msg}', got '{context.transform_result['message']}'"
)
@given("a transform executor with code that does not define a transform function")
def step_transform_no_func(context: Context) -> None:
context.transform_executor = TransformExecutor("x = 1 + 2\n", "test/no-func")
@when("I try to execute the transform with any output")
def step_try_execute_transform(context: Context) -> None:
context.transform_error = None
try:
context.transform_executor.execute({"data": "test"})
except (TransformExecutionError, ValueError) as exc:
context.transform_error = exc
@then('a TransformExecutionError should be raised with message containing "{fragment}"')
def step_check_transform_error(context: Context, fragment: str) -> None:
assert context.transform_error is not None, "Expected TransformExecutionError"
assert isinstance(context.transform_error, TransformExecutionError)
assert fragment in str(context.transform_error), (
f"Expected '{fragment}' in '{context.transform_error}'"
)
@given("a transform executor with code that returns a non-dict value")
def step_transform_non_dict_return(context: Context) -> None:
context.transform_executor = TransformExecutor(
"def transform(x):\n return 'not a dict'\n",
"test/non-dict",
)
@given("a transform executor with code that returns a dict without passed key")
def step_transform_no_passed(context: Context) -> None:
context.transform_executor = TransformExecutor(
'def transform(x):\n return {"message": "no passed"}\n',
"test/no-passed",
)
@when("I try to create a transform executor with empty code")
def step_create_transform_empty(context: Context) -> None:
context.transform_construction_error = None
try:
TransformExecutor(" ", "test/empty")
except ValueError as exc:
context.transform_construction_error = exc
@then("a ValueError should be raised from transform construction")
def step_check_transform_construction_error(context: Context) -> None:
assert context.transform_construction_error is not None, "Expected ValueError"
assert isinstance(context.transform_construction_error, ValueError)
@given("a transform executor with code that attempts to import os")
def step_transform_sandbox_import(context: Context) -> None:
context.transform_executor = TransformExecutor(
"def transform(x):\n import os\n return {'passed': True}\n",
"test/sandbox",
)
@then("a TransformExecutionError should be raised from sandbox restriction")
def step_check_sandbox_error(context: Context) -> None:
assert context.transform_error is not None, (
"Expected TransformExecutionError from sandbox"
)
assert isinstance(context.transform_error, TransformExecutionError)
# -------------------------------------------------------------------
# WrappedToolExecutor steps
# -------------------------------------------------------------------
@given('a tool registry with a tool "{tool_name}" that returns {output_json}')
def step_tool_registry_with_tool(
context: Context,
tool_name: str,
output_json: str,
) -> None:
if not hasattr(context, "test_tools"):
context.test_tools = {}
if not hasattr(context, "test_validations"):
context.test_validations = {}
if not hasattr(context, "tool_call_log"):
context.tool_call_log = {}
output = json.loads(output_json)
context.test_tools[tool_name] = (_make_tool(tool_name), output)
@given('a validation "{val_name}" that wraps "{wraps_name}" with a simple transform')
def step_validation_simple_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _SIMPLE_TRANSFORM)
context.test_validations[val_name] = v
@given(
'a validation "{val_name}" that wraps "{wraps_name}" '
"with argument mapping {mapping_json}"
)
def step_validation_with_mapping(
context: Context,
val_name: str,
wraps_name: str,
mapping_json: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
mapping = json.loads(mapping_json)
v = _make_validation(
val_name,
wraps_name,
_SIMPLE_TRANSFORM,
argument_mapping=mapping,
)
context.test_validations[val_name] = v
@given(
'a validation "{val_name}" that wraps "{wraps_name}" with a passthrough transform'
)
def step_validation_passthrough_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _PASSTHROUGH_TRANSFORM)
context.test_validations[val_name] = v
@given('a validation "{val_name}" that wraps "{wraps_name}" with a status transform')
def step_validation_status_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _STATUS_TRANSFORM)
context.test_validations[val_name] = v
@given('a validation "{val_name}" that wraps "{wraps_name}" with a data transform')
def step_validation_data_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _DATA_TRANSFORM)
context.test_validations[val_name] = v
@given("a wrapped tool executor using the test registry")
def step_create_executor(context: Context) -> None:
tools = getattr(context, "test_tools", {})
validations = getattr(context, "test_validations", {})
context.tool_call_log = {}
def lookup(name: str) -> Tool | Validation | None:
if name in validations:
return validations[name]
if name in tools:
return tools[name][0]
return None
def executor(name: str, args: dict[str, Any]) -> Any:
context.tool_call_log[name] = args
if name in tools:
return tools[name][1]
raise RuntimeError(f"Tool {name} not found in test registry")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@given("a wrapped tool executor with empty registry")
def step_create_empty_executor(context: Context) -> None:
context.tool_call_log = {}
def lookup(name: str) -> None:
return None
def executor(name: str, args: dict[str, Any]) -> Any:
raise RuntimeError(f"Tool {name} not found")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@given("a wrapped tool executor using the test registry with cycle")
def step_create_executor_with_cycle(context: Context) -> None:
validations = getattr(context, "test_validations", {})
def lookup(name: str) -> Validation | None:
return validations.get(name)
def executor(name: str, args: dict[str, Any]) -> Any:
raise RuntimeError("Should not reach leaf executor in cycle")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@when("I execute the wrapping validation with inputs {inputs_json}")
def step_execute_wrapping(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
# Find the first validation with wraps
val = None
for v in context.test_validations.values():
if v.wraps is not None:
val = v
break
assert val is not None, "No wrapping validation found"
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
context.last_executed_validation = val
@when("I execute the outer wrapping validation with inputs {inputs_json}")
def step_execute_outer_wrapping(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
val = context.test_validations.get("local/outer-wrapper")
assert val is not None, "local/outer-wrapper not found"
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
@when("I try to execute the wrapping validation with inputs {inputs_json}")
def step_try_execute_wrapping(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
context.wrapped_error = None
# Find first validation with wraps
val = None
for v in getattr(context, "test_validations", {}).values():
if v.wraps is not None:
val = v
break
if val is None:
return
try:
context.wrapped_executor.execute(val, inputs)
except (WrappedToolNotFoundError, WrappingCycleError) as exc:
context.wrapped_error = exc
@when('I try to execute the wrapping validation with cycle from "{val_name}"')
def step_try_execute_cycle(context: Context, val_name: str) -> None:
context.wrapped_error = None
val = context.test_validations[val_name]
try:
context.wrapped_executor.execute(val, {})
except WrappingCycleError as exc:
context.wrapped_error = exc
@then("the wrapped execution should succeed with passed true")
def step_check_wrapped_passed(context: Context) -> None:
assert context.wrapped_result["passed"] is True, (
f"Expected passed=True, got {context.wrapped_result}"
)
@then('the wrapped tool "{tool_name}" should have been called')
def step_check_tool_called(context: Context, tool_name: str) -> None:
assert tool_name in context.tool_call_log, (
f"Tool '{tool_name}' was not called. "
f"Called: {list(context.tool_call_log.keys())}"
)
@then(
'the wrapped tool "{tool_name}" should have received argument '
'"{arg_name}" with value "{arg_value}"'
)
def step_check_tool_arg_str(
context: Context,
tool_name: str,
arg_name: str,
arg_value: str,
) -> None:
call_args = context.tool_call_log[tool_name]
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
assert call_args[arg_name] == arg_value, (
f"Expected '{arg_value}', got '{call_args[arg_name]}'"
)
@then(
'the wrapped tool "{tool_name}" should have received argument '
'"{arg_name}" with value true'
)
def step_check_tool_arg_true(
context: Context,
tool_name: str,
arg_name: str,
) -> None:
call_args = context.tool_call_log[tool_name]
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
assert call_args[arg_name] is True, f"Expected True, got {call_args[arg_name]}"
@then('a WrappedToolNotFoundError should be raised for "{wraps_name}"')
def step_check_not_found_error(context: Context, wraps_name: str) -> None:
assert context.wrapped_error is not None, "Expected WrappedToolNotFoundError"
assert isinstance(context.wrapped_error, WrappedToolNotFoundError)
assert wraps_name in str(context.wrapped_error)
@then("a WrappingCycleError should be raised")
def step_check_cycle_error(context: Context) -> None:
assert context.wrapped_error is not None, "Expected WrappingCycleError"
assert isinstance(context.wrapped_error, WrappingCycleError)
@then("the wrapped tool should have received the validation inputs")
def step_check_inputs_passed(context: Context) -> None:
assert len(context.tool_call_log) > 0, "No tool was called"
@when("I try to execute with a non-Validation object")
def step_try_execute_non_validation(context: Context) -> None:
context.executor_type_error = None
try:
context.wrapped_executor.execute("not_a_validation", {}) # type: ignore[arg-type]
except TypeError as exc:
context.executor_type_error = exc
@then("a TypeError should be raised from the executor")
def step_check_executor_type_error(context: Context) -> None:
assert context.executor_type_error is not None, "Expected TypeError"
assert isinstance(context.executor_type_error, TypeError)
@given('a validation "{val_name}" without wraps set')
def step_validation_no_wraps(context: Context, val_name: str) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_plain_validation(val_name)
context.test_validations[val_name] = v
context.no_wrap_validation = v
@when("I try to execute the non-wrapping validation")
def step_try_execute_no_wraps(context: Context) -> None:
context.no_wrap_error = None
try:
context.wrapped_executor.execute(context.no_wrap_validation, {})
except ValueError as exc:
context.no_wrap_error = exc
@then("a ValueError should be raised indicating wraps is not set")
def step_check_no_wrap_error(context: Context) -> None:
assert context.no_wrap_error is not None, "Expected ValueError"
assert isinstance(context.no_wrap_error, ValueError)
assert "wraps" in str(context.no_wrap_error).lower()
# -------------------------------------------------------------------
# Additional coverage: WrappingDepthExceededError
# -------------------------------------------------------------------
@given("a deep wrapping chain of {count:d} validations")
def step_deep_chain(context: Context, count: int) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
if not hasattr(context, "test_tools"):
context.test_tools = {}
# Create a chain: v0 wraps v1, v1 wraps v2, ..., vN wraps leaf-tool
for i in range(count):
wraps_name = f"local/deep-v{i + 1}" if i < count - 1 else "local/deep-leaf"
v = _make_validation(
f"local/deep-v{i}",
wraps_name,
_PASSTHROUGH_TRANSFORM,
)
context.test_validations[v.name] = v
# Create the leaf tool
context.test_tools["local/deep-leaf"] = (
_make_tool("local/deep-leaf"),
{"status": "ok"},
)
@given("a wrapped tool executor using the deep chain registry")
def step_create_deep_chain_executor(context: Context) -> None:
tools = getattr(context, "test_tools", {})
validations = getattr(context, "test_validations", {})
def lookup(name: str) -> Tool | Validation | None:
if name in validations:
return validations[name]
if name in tools:
return tools[name][0]
return None
def executor(name: str, args: dict[str, Any]) -> Any:
if name in tools:
return tools[name][1]
raise RuntimeError(f"Tool {name} not found")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@when("I try to execute the deep chain wrapping validation")
def step_try_execute_deep_chain(context: Context) -> None:
context.wrapped_error = None
val = context.test_validations["local/deep-v0"]
try:
context.wrapped_executor.execute(val, {})
except WrappingDepthExceededError as exc:
context.wrapped_error = exc
@then("a WrappingDepthExceededError should be raised with depth {depth:d}")
def step_check_depth_exceeded(context: Context, depth: int) -> None:
assert context.wrapped_error is not None, "Expected WrappingDepthExceededError"
assert isinstance(context.wrapped_error, WrappingDepthExceededError)
assert context.wrapped_error.depth == depth, (
f"Expected depth {depth}, got {context.wrapped_error.depth}"
)
# -------------------------------------------------------------------
# Additional coverage: ArgumentMapper.mapping property
# -------------------------------------------------------------------
@then("the mapper mapping property should return {expected_json}")
def step_check_mapper_property(context: Context, expected_json: str) -> None:
expected = json.loads(expected_json)
assert context.mapper.mapping == expected, (
f"Expected {expected}, got {context.mapper.mapping}"
)
@then("the mapper mapping property should be None")
def step_check_mapper_property_none(context: Context) -> None:
assert context.mapper.mapping is None, (
f"Expected None, got {context.mapper.mapping}"
)
# -------------------------------------------------------------------
# Additional coverage: TransformExecutor type checks
# -------------------------------------------------------------------
@when("I try to create a transform executor with non-string code")
def step_create_transform_non_string_code(context: Context) -> None:
context.transform_type_error = None
try:
TransformExecutor(12345, "test/non-string-code") # type: ignore[arg-type]
except TypeError as exc:
context.transform_type_error = exc
@then("a TypeError should be raised from transform code type check")
def step_check_transform_code_type_error(context: Context) -> None:
assert context.transform_type_error is not None, "Expected TypeError"
assert isinstance(context.transform_type_error, TypeError)
@when("I try to create a transform executor with non-string tool name")
def step_create_transform_non_string_name(context: Context) -> None:
context.transform_name_type_error = None
try:
TransformExecutor("def transform(x): return {'passed': True}", 42) # type: ignore[arg-type]
except TypeError as exc:
context.transform_name_type_error = exc
@then("a TypeError should be raised from transform tool name check")
def step_check_transform_name_type_error(context: Context) -> None:
assert context.transform_name_type_error is not None, "Expected TypeError"
assert isinstance(context.transform_name_type_error, TypeError)
@given("a transform executor with code that raises an error during exec")
def step_transform_exec_error(context: Context) -> None:
# This code has a top-level expression that raises during exec
context.transform_executor = TransformExecutor(
"raise RuntimeError('bad code')\ndef transform(x):\n return {'passed': True}\n",
"test/exec-error",
)
# -------------------------------------------------------------------
# Additional coverage: WrappedToolExecutor init checks
# -------------------------------------------------------------------
@when("I try to create a wrapped tool executor with None tool_lookup")
def step_create_executor_none_lookup(context: Context) -> None:
context.executor_init_error = None
try:
WrappedToolExecutor(None, lambda n, a: {}) # type: ignore[arg-type]
except ValueError as exc:
context.executor_init_error = exc
@then("a ValueError should be raised from executor construction for tool_lookup")
def step_check_executor_none_lookup_error(context: Context) -> None:
assert context.executor_init_error is not None, "Expected ValueError"
assert isinstance(context.executor_init_error, ValueError)
@when("I try to create a wrapped tool executor with None tool_executor")
def step_create_executor_none_executor(context: Context) -> None:
context.executor_init_error = None
try:
WrappedToolExecutor(lambda n: None, None) # type: ignore[arg-type]
except ValueError as exc:
context.executor_init_error = exc
@then("a ValueError should be raised from executor construction for tool_executor")
def step_check_executor_none_executor_error(context: Context) -> None:
assert context.executor_init_error is not None, "Expected ValueError"
assert isinstance(context.executor_init_error, ValueError)
@when("I try to create a wrapped tool executor with non-callable tool_lookup")
def step_create_executor_non_callable_lookup(context: Context) -> None:
context.executor_type_init_error = None
try:
WrappedToolExecutor("not_callable", lambda n, a: {}) # type: ignore[arg-type]
except TypeError as exc:
context.executor_type_init_error = exc
@then("a TypeError should be raised from executor construction for tool_lookup")
def step_check_executor_non_callable_lookup_error(context: Context) -> None:
assert context.executor_type_init_error is not None, "Expected TypeError"
assert isinstance(context.executor_type_init_error, TypeError)
@when("I try to create a wrapped tool executor with non-callable tool_executor")
def step_create_executor_non_callable_executor(context: Context) -> None:
context.executor_type_init_error = None
try:
WrappedToolExecutor(lambda n: None, "not_callable") # type: ignore[arg-type]
except TypeError as exc:
context.executor_type_init_error = exc
@then("a TypeError should be raised from executor construction for tool_executor")
def step_check_executor_non_callable_executor_error(context: Context) -> None:
assert context.executor_type_init_error is not None, "Expected TypeError"
assert isinstance(context.executor_type_init_error, TypeError)
# -------------------------------------------------------------------
# Additional coverage: execute with non-dict inputs
# -------------------------------------------------------------------
@when("I try to execute wrapping validation with non-dict inputs")
def step_try_execute_non_dict_inputs(context: Context) -> None:
context.inputs_type_error = None
val = None
for v in context.test_validations.values():
if v.wraps is not None:
val = v
break
assert val is not None
try:
context.wrapped_executor.execute(val, "not_a_dict") # type: ignore[arg-type]
except TypeError as exc:
context.inputs_type_error = exc
@then("a TypeError should be raised for non-dict inputs")
def step_check_non_dict_inputs_error(context: Context) -> None:
assert context.inputs_type_error is not None, "Expected TypeError"
assert isinstance(context.inputs_type_error, TypeError)
# -------------------------------------------------------------------
# Additional coverage: innermost wrapper with no wraps target
# -------------------------------------------------------------------
# -------------------------------------------------------------------
# ToolRunner coverage steps
# -------------------------------------------------------------------
@given("a ToolRunner with a mock registry")
def step_tool_runner_mock_registry(context: Context) -> None:
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
registry.register(
ToolSpec(
name="local/test-tool",
description="test",
handler=lambda inputs: {"result": "ok"},
)
)
context.tool_runner = ToolRunner(registry)
@when("I call resolve_execution_environment on the runner")
def step_call_resolve_env(context: Context) -> None:
context.resolved_env = context.tool_runner.resolve_execution_environment()
@then("the resolved environment should be local")
def step_check_env_local(context: Context) -> None:
from cleveragents.domain.models.core.plan import ExecutionEnvironment
assert context.resolved_env == ExecutionEnvironment.HOST
@given("a ToolRunner with a value-error-raising env resolver")
def step_runner_value_error_resolver(context: Context) -> None:
from unittest.mock import MagicMock
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
registry.register(
ToolSpec(
name="local/test-tool",
description="test",
handler=lambda inputs: {"result": "ok"},
)
)
runner = ToolRunner(registry)
# Mock the resolver to raise ValueError
mock_resolver = MagicMock()
mock_resolver.resolve_and_validate.side_effect = ValueError("bad env config")
runner._env_resolver = mock_resolver
context.tool_runner = runner
@when("I execute a tool through the runner with env error")
def step_execute_runner_env_error(context: Context) -> None:
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
@then("the tool result should have success false")
def step_check_runner_result_fail(context: Context) -> None:
assert context.runner_result.success is False
@then('the tool result error should contain "{fragment}"')
def step_check_runner_result_error(context: Context, fragment: str) -> None:
assert fragment in (context.runner_result.error or ""), (
f"Expected '{fragment}' in '{context.runner_result.error}'"
)
@given("a ToolRunner with a container-returning env resolver")
def step_runner_container_resolver(context: Context) -> None:
from unittest.mock import MagicMock
from cleveragents.domain.models.core.plan import ExecutionEnvironment
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
registry.register(
ToolSpec(
name="local/test-tool",
description="test",
handler=lambda inputs: {"result": "ok"},
)
)
runner = ToolRunner(registry)
# Mock the resolver to return CONTAINER
mock_resolver = MagicMock()
mock_resolver.resolve_and_validate.return_value = ExecutionEnvironment.CONTAINER
runner._env_resolver = mock_resolver
context.tool_runner = runner
@when("I execute a tool through the runner with container env")
def step_execute_runner_container(context: Context) -> None:
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
@when("I directly call _execute_chain with a no-wraps leaf")
def step_direct_execute_chain_no_wraps(context: Context) -> None:
context.chain_error = None
# Create a plain validation with wraps=None
leaf_val = _make_plain_validation("local/no-wraps-leaf")
try:
# Call the private method directly to exercise lines 437-438
context.wrapped_executor._execute_chain([leaf_val], {})
except ValueError as exc:
context.chain_error = exc
@then("a ValueError should be raised for missing wraps target")
def step_check_missing_wraps_target(context: Context) -> None:
assert context.chain_error is not None, "Expected ValueError"
assert isinstance(context.chain_error, ValueError)
assert "wraps" in str(context.chain_error).lower()
+220
View File
@@ -0,0 +1,220 @@
Feature: Tool wrapping runtime
As the validation execution engine
I need to delegate execution to wrapped tools via wraps + transform
So that validations can reuse existing tool implementations
# ArgumentMapper
Scenario: ArgumentMapper with None mapping passes arguments through
Given an argument mapper with no mapping
When I apply the mapper to inputs {"path": "/src", "verbose": true}
Then the mapped arguments should be {"path": "/src", "verbose": true}
Scenario: ArgumentMapper with mapping translates argument names
Given an argument mapper with mapping {"test_directory": "source_dir", "coverage_enabled": true}
When I apply the mapper to inputs {"source_dir": "/tests"}
Then the mapped arguments should be {"test_directory": "/tests", "coverage_enabled": true}
Scenario: ArgumentMapper with literal values injects fixed values
Given an argument mapper with mapping {"mode": "strict", "count": 42}
When I apply the mapper to inputs {"extra": "ignored"}
Then the mapped arguments should be {"mode": "strict", "count": 42}
Scenario: ArgumentMapper rejects non-dict inputs
Given an argument mapper with no mapping
When I try to apply the mapper to non-dict input "not_a_dict"
Then a TypeError should be raised from the argument mapper
Scenario: ArgumentMapper rejects non-dict mapping at construction
When I try to create an argument mapper with a non-dict mapping
Then a TypeError should be raised from argument mapper construction
# ── TransformExecutor ─────────────────────────────────────────────
Scenario: TransformExecutor runs a valid transform function
Given a transform executor with code that checks returncode equals zero
When I execute the transform with tool output {"returncode": 0, "tests_run": 10}
Then the transform result should have passed true
And the transform result should have message "All tests passed"
Scenario: TransformExecutor handles failing transform output
Given a transform executor with code that checks returncode equals zero
When I execute the transform with tool output {"returncode": 1, "tests_run": 10}
Then the transform result should have passed false
Scenario: TransformExecutor raises on missing transform function
Given a transform executor with code that does not define a transform function
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "callable"
Scenario: TransformExecutor raises on non-dict return value
Given a transform executor with code that returns a non-dict value
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "dict"
Scenario: TransformExecutor raises on missing passed key in result
Given a transform executor with code that returns a dict without passed key
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "passed"
Scenario: TransformExecutor raises on empty transform code
When I try to create a transform executor with empty code
Then a ValueError should be raised from transform construction
Scenario: TransformExecutor sandboxes dangerous operations
Given a transform executor with code that attempts to import os
When I try to execute the transform with any output
Then a TransformExecutionError should be raised from sandbox restriction
# ── WrappedToolExecutor ───────────────────────────────────────────
Scenario: Simple wrapping delegates to the wrapped tool
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
And a validation "local/tests-pass" that wraps "local/run-tests" with a simple transform
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {"path": "/src"}
Then the wrapped execution should succeed with passed true
And the wrapped tool "local/run-tests" should have been called
Scenario: Argument mapping translates arguments to the wrapped tool
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
And a validation "local/mapped-check" that wraps "local/run-tests" with argument mapping {"test_directory": "source_dir", "coverage_enabled": true}
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {"source_dir": "/tests"}
Then the wrapped tool "local/run-tests" should have received argument "test_directory" with value "/tests"
And the wrapped tool "local/run-tests" should have received argument "coverage_enabled" with value true
Scenario: Chained wrapping delegates through the chain
Given a tool registry with a tool "local/base-tool" that returns {"status": "ok"}
And a validation "local/mid-wrapper" that wraps "local/base-tool" with a passthrough transform
And a validation "local/outer-wrapper" that wraps "local/mid-wrapper" with a status transform
And a wrapped tool executor using the test registry
When I execute the outer wrapping validation with inputs {}
Then the wrapped execution should succeed with passed true
And the wrapped tool "local/base-tool" should have been called
Scenario: Missing wrapped tool raises WrappedToolNotFoundError
Given a validation "local/broken-wrap" that wraps "local/nonexistent" with a simple transform
And a wrapped tool executor with empty registry
When I try to execute the wrapping validation with inputs {}
Then a WrappedToolNotFoundError should be raised for "local/nonexistent"
Scenario: Circular wrapping chain raises WrappingCycleError
Given a validation "local/wrap-a" that wraps "local/wrap-b" with a simple transform
And a validation "local/wrap-b" that wraps "local/wrap-a" with a simple transform
And a wrapped tool executor using the test registry with cycle
When I try to execute the wrapping validation with cycle from "local/wrap-a"
Then a WrappingCycleError should be raised
Scenario: Execution context is inherited from wrapping validation
Given a tool registry with a tool "local/context-tool" that returns {"data": "value"}
And a validation "local/context-wrap" that wraps "local/context-tool" with a data transform
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {"key": "value"}
Then the wrapped tool should have received the validation inputs
Scenario: WrappedToolExecutor rejects non-Validation argument
Given a wrapped tool executor with empty registry
When I try to execute with a non-Validation object
Then a TypeError should be raised from the executor
Scenario: WrappedToolExecutor rejects validation without wraps
Given a validation "local/no-wrap" without wraps set
And a wrapped tool executor with empty registry
When I try to execute the non-wrapping validation
Then a ValueError should be raised indicating wraps is not set
# ── Additional coverage: WrappingDepthExceededError ──────────────
Scenario: WrappingDepthExceededError is raised when chain is too deep
Given a deep wrapping chain of 11 validations
And a wrapped tool executor using the deep chain registry
When I try to execute the deep chain wrapping validation
Then a WrappingDepthExceededError should be raised with depth 10
# ── Additional coverage: ArgumentMapper.mapping property ─────────
Scenario: ArgumentMapper exposes the raw mapping via property
Given an argument mapper with mapping {"x": "y"}
Then the mapper mapping property should return {"x": "y"}
Scenario: ArgumentMapper mapping property returns None for identity mapper
Given an argument mapper with no mapping
Then the mapper mapping property should be None
# ── Additional coverage: TransformExecutor type checks ───────────
Scenario: TransformExecutor rejects non-string transform code
When I try to create a transform executor with non-string code
Then a TypeError should be raised from transform code type check
Scenario: TransformExecutor rejects non-string tool name
When I try to create a transform executor with non-string tool name
Then a TypeError should be raised from transform tool name check
Scenario: TransformExecutor raises on code that throws during exec
Given a transform executor with code that raises an error during exec
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "compile"
# ── Additional coverage: WrappedToolExecutor init checks ─────────
Scenario: WrappedToolExecutor rejects None tool_lookup
When I try to create a wrapped tool executor with None tool_lookup
Then a ValueError should be raised from executor construction for tool_lookup
Scenario: WrappedToolExecutor rejects None tool_executor
When I try to create a wrapped tool executor with None tool_executor
Then a ValueError should be raised from executor construction for tool_executor
Scenario: WrappedToolExecutor rejects non-callable tool_lookup
When I try to create a wrapped tool executor with non-callable tool_lookup
Then a TypeError should be raised from executor construction for tool_lookup
Scenario: WrappedToolExecutor rejects non-callable tool_executor
When I try to create a wrapped tool executor with non-callable tool_executor
Then a TypeError should be raised from executor construction for tool_executor
# ── Additional coverage: execute with non-dict inputs ────────────
Scenario: WrappedToolExecutor rejects non-dict inputs
Given a tool registry with a tool "local/some-tool" that returns {"ok": true}
And a validation "local/input-wrap" that wraps "local/some-tool" with a simple transform
And a wrapped tool executor using the test registry
When I try to execute wrapping validation with non-dict inputs
Then a TypeError should be raised for non-dict inputs
# ── Additional coverage: chain with None wraps at end ────────────
Scenario: Chain resolution handles validation with wraps set to None in chain
Given a tool registry with a tool "local/leaf" that returns {"value": 1}
And a validation "local/wrapper-no-inner" that wraps "local/leaf" with a passthrough transform
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {}
Then the wrapped execution should succeed with passed true
# ── Additional coverage: _execute_chain with None wraps target ────
Scenario: _execute_chain raises ValueError when leaf wraps is None
Given a wrapped tool executor with empty registry
When I directly call _execute_chain with a no-wraps leaf
Then a ValueError should be raised for missing wraps target
# ── ToolRunner coverage: execution environment and error paths ───
Scenario: ToolRunner resolve_execution_environment delegates to resolver
Given a ToolRunner with a mock registry
When I call resolve_execution_environment on the runner
Then the resolved environment should be local
Scenario: ToolRunner execute returns error when env resolver raises ValueError
Given a ToolRunner with a value-error-raising env resolver
When I execute a tool through the runner with env error
Then the tool result should have success false
And the tool result error should contain "Execution environment error"
Scenario: ToolRunner execute returns error for container environment
Given a ToolRunner with a container-returning env resolver
When I execute a tool through the runner with container env
Then the tool result should have success false
And the tool result error should contain "Container execution is not yet implemented"
+42
View File
@@ -0,0 +1,42 @@
"""Robot helper: verify WrappedToolExecutor delegates to wrapped tool."""
from cleveragents.domain.models.core.tool import (
Tool,
ToolCapability,
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import WrappedToolExecutor
tool = Tool(
name="local/run",
description="run",
source=ToolSource.BUILTIN,
capability=ToolCapability(read_only=True),
)
val = Validation(
name="local/check",
description="check",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps="local/run",
transform='def transform(x):\n return {"passed": True, "data": x}\n',
)
lookup = lambda n: tool if n == "local/run" else None # noqa: E731
called: list[str] = []
def executor_fn(n: str, a: dict) -> dict: # type: ignore[type-arg]
"""Record tool call and return output."""
called.append(n)
return {"result": "ok"}
ex = WrappedToolExecutor(lookup, executor_fn)
r = ex.execute(val, {"a": 1})
assert r["passed"] is True
assert "local/run" in called
print("OK")
+8
View File
@@ -0,0 +1,8 @@
"""Robot helper: verify ArgumentMapper translates arguments."""
from cleveragents.tool.wrapping import ArgumentMapper
m = ArgumentMapper({"dest": "src", "flag": True})
r = m.apply({"src": "/path"})
assert r == {"dest": "/path", "flag": True}, f"Got {r}"
print("OK")
+8
View File
@@ -0,0 +1,8 @@
"""Robot helper: verify ArgumentMapper with None mapping."""
from cleveragents.tool.wrapping import ArgumentMapper
m = ArgumentMapper(None)
r = m.apply({"x": 1, "y": "two"})
assert r == {"x": 1, "y": "two"}, f"Got {r}"
print("OK")
+28
View File
@@ -0,0 +1,28 @@
"""Robot helper: verify WrappedToolExecutor raises on missing tool."""
from cleveragents.domain.models.core.tool import (
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import WrappedToolExecutor, WrappedToolNotFoundError
val = Validation(
name="local/check",
description="check",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps="local/missing",
transform='def transform(x):\n return {"passed": True}\n',
)
ok = False
try:
WrappedToolExecutor(
lambda n: None,
lambda n, a: {},
).execute(val, {})
except WrappedToolNotFoundError:
ok = True
print(f"error_raised={ok}")
+9
View File
@@ -0,0 +1,9 @@
"""Robot helper: verify TransformExecutor runs a transform."""
from cleveragents.tool.wrapping import TransformExecutor
code = 'def transform(x):\n return {"passed": x.get("ok"), "message": "done"}\n'
t = TransformExecutor(code, "test/t")
r = t.execute({"ok": True})
assert r["passed"] is True
print("OK")
+12
View File
@@ -0,0 +1,12 @@
"""Robot helper: verify TransformExecutor blocks import in sandbox."""
from cleveragents.tool.wrapping import TransformExecutionError, TransformExecutor
code = 'def transform(x):\n import os\n return {"passed": True}\n'
t = TransformExecutor(code, "test/s")
ok = False
try:
t.execute({})
except TransformExecutionError:
ok = True
print(f"blocked={ok}")
+58
View File
@@ -0,0 +1,58 @@
*** Settings ***
Documentation Tool wrapping runtime smoke tests
Library Process
Library OperatingSystem
*** Variables ***
${PYTHON} python
*** Test Cases ***
Wrapping Package Is Importable
[Documentation] Verify tool wrapping module can be imported
${result}= Run Process ${PYTHON} -c
... from cleveragents.tool.wrapping import ArgumentMapper, TransformExecutor, WrappedToolExecutor, WrappedToolNotFoundError, WrappingCycleError, WrappingDepthExceededError, TransformExecutionError; print('OK')
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} OK
Wrapping Exports In Tool Init
[Documentation] Verify wrapping types are exported from tool package
${result}= Run Process ${PYTHON} -c
... from cleveragents.tool import ArgumentMapper, TransformExecutor, WrappedToolExecutor; print('OK')
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} OK
ArgumentMapper Identity Mapping
[Documentation] Verify ArgumentMapper with None mapping passes args through
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_identity.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
ArgumentMapper With Mapping
[Documentation] Verify ArgumentMapper translates arguments
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_configured.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
TransformExecutor Runs Transform
[Documentation] Verify TransformExecutor executes transform code
${result}= Run Process ${PYTHON} robot/scripts/test_transform_run.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
TransformExecutor Sandbox Blocks Import
[Documentation] Verify TransformExecutor blocks import in sandbox
${result}= Run Process ${PYTHON} robot/scripts/test_transform_sandbox.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} blocked=True
WrappedToolExecutor Simple Delegation
[Documentation] Verify WrappedToolExecutor delegates to wrapped tool
${result}= Run Process ${PYTHON} robot/scripts/test_delegation.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
WrappedToolExecutor Missing Tool Error
[Documentation] Verify WrappedToolExecutor raises on missing wrapped tool
${result}= Run Process ${PYTHON} robot/scripts/test_missing_tool.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} error_raised=True
+16
View File
@@ -69,8 +69,18 @@ from cleveragents.tool.schema_validator import (
validate_tool_input,
validate_tool_output,
)
from cleveragents.tool.wrapping import (
ArgumentMapper,
TransformExecutionError,
TransformExecutor,
WrappedToolExecutor,
WrappedToolNotFoundError,
WrappingCycleError,
WrappingDepthExceededError,
)
__all__ = [
"ArgumentMapper",
"BoundResource",
"CancellationToken",
"Change",
@@ -116,6 +126,12 @@ __all__ = [
"ToolSandboxRequiredError",
"ToolSchemaValidationError",
"ToolSpec",
"TransformExecutionError",
"TransformExecutor",
"WrappedToolExecutor",
"WrappedToolNotFoundError",
"WrappingCycleError",
"WrappingDepthExceededError",
"classify_tool_error",
"detect_provider_format",
"generate_tool_call_id",
+470
View File
@@ -0,0 +1,470 @@
"""Tool wrapping runtime for CleverAgents v3.
Implements the ``wraps`` + ``transform`` + ``argument_mapping`` delegation
mechanism described in the specification (§ Core Concepts > Validation >
Tool Wrapping).
## Architecture
```
WrappedToolExecutor
|-- ArgumentMapper (translate wrapper args -> wrapped tool args)
|-- TransformExecutor (sandboxed transform function on tool output)
+-- ToolRuntime (resolve + invoke wrapped tools)
```
## Delegation Chain
A validation wrapping another validation forms a delegation chain:
```
validation-A -> validation-B -> tool-C
(wraps B) (wraps C) (leaf)
```
``WrappedToolExecutor.execute()`` recursively resolves the chain,
detecting cycles and enforcing a maximum depth.
## Sandbox
``TransformExecutor`` compiles and executes the user-supplied
``transform`` function with a restricted set of built-in names.
No filesystem, network, or import access is available inside the
transform.
Based on ``docs/specification.md`` § Tool Wrapping.
"""
from __future__ import annotations
import logging
from typing import Any
from cleveragents.domain.models.core.tool import Validation
logger = logging.getLogger(__name__)
# Maximum delegation chain depth to prevent infinite recursion.
MAX_WRAPPING_DEPTH = 10
# Safe built-ins available inside the ``transform`` sandbox.
_SAFE_BUILTINS: dict[str, Any] = {
"True": True,
"False": False,
"None": None,
"abs": abs,
"all": all,
"any": any,
"bool": bool,
"dict": dict,
"enumerate": enumerate,
"float": float,
"frozenset": frozenset,
"int": int,
"isinstance": isinstance,
"len": len,
"list": list,
"map": map,
"max": max,
"min": min,
"range": range,
"repr": repr,
"reversed": reversed,
"round": round,
"set": set,
"sorted": sorted,
"str": str,
"sum": sum,
"tuple": tuple,
"type": type,
"zip": zip,
}
# -------------------------------------------------------------------
# Errors
# -------------------------------------------------------------------
class WrappedToolNotFoundError(Exception):
"""Raised when the tool referenced by ``wraps`` is not registered."""
def __init__(self, tool_name: str, wraps_name: str) -> None:
self.tool_name = tool_name
self.wraps_name = wraps_name
super().__init__(
f"Validation '{tool_name}' wraps '{wraps_name}' but "
f"'{wraps_name}' is not registered in the runtime"
)
class WrappingCycleError(Exception):
"""Raised when a circular wrapping chain is detected."""
def __init__(self, chain: list[str]) -> None:
self.chain = list(chain)
cycle_str = " -> ".join(chain)
super().__init__(f"Circular wrapping chain detected: {cycle_str}")
class WrappingDepthExceededError(Exception):
"""Raised when the delegation chain exceeds ``MAX_WRAPPING_DEPTH``."""
def __init__(self, depth: int) -> None:
self.depth = depth
super().__init__(
f"Wrapping delegation chain depth ({depth}) exceeds "
f"the maximum ({MAX_WRAPPING_DEPTH})"
)
class TransformExecutionError(Exception):
"""Raised when the ``transform`` function fails during execution."""
def __init__(self, tool_name: str, detail: str) -> None:
self.tool_name = tool_name
self.detail = detail
super().__init__(f"Transform function for '{tool_name}' failed: {detail}")
# -------------------------------------------------------------------
# ArgumentMapper
# -------------------------------------------------------------------
class ArgumentMapper:
"""Applies ``argument_mapping`` to translate arguments.
When a validation wraps a tool, the validation may accept different
input arguments. The ``argument_mapping`` dictionary specifies how
the validation's inputs map to the wrapped tool's expected inputs.
Keys are the wrapped tool's parameter names. Values are either:
- A string referencing a validation input parameter (forwarded).
- A literal value (string, number, boolean) always sent to the
wrapped tool.
When ``argument_mapping`` is ``None``, input arguments are passed
through to the wrapped tool unchanged (identity mapping).
"""
def __init__(self, mapping: dict[str, Any] | None) -> None:
if mapping is not None and not isinstance(mapping, dict):
raise TypeError(
f"argument_mapping must be a dict or None, got {type(mapping).__name__}"
)
self._mapping = mapping
@property
def mapping(self) -> dict[str, Any] | None:
"""Return the raw mapping configuration."""
return self._mapping
def apply(self, validation_inputs: dict[str, Any]) -> dict[str, Any]:
"""Translate *validation_inputs* into wrapped-tool arguments.
Parameters
----------
validation_inputs:
The arguments the validation was invoked with.
Returns
-------
dict[str, Any]:
Arguments to forward to the wrapped tool.
"""
if not isinstance(validation_inputs, dict):
raise TypeError(
f"validation_inputs must be a dict, "
f"got {type(validation_inputs).__name__}"
)
if self._mapping is None:
return dict(validation_inputs)
mapped: dict[str, Any] = {}
for wrapped_param, source in self._mapping.items():
if isinstance(source, str) and source in validation_inputs:
mapped[wrapped_param] = validation_inputs[source]
else:
# Literal value (or string not present as a key -- use
# as literal).
mapped[wrapped_param] = source
return mapped
# -------------------------------------------------------------------
# TransformExecutor
# -------------------------------------------------------------------
class TransformExecutor:
"""Executes the ``transform`` function in a sandboxed context.
The transform code must define a function called ``transform`` that
accepts one positional argument (the wrapped tool's output) and
returns a dict with at minimum ``{"passed": bool}``.
The sandbox disallows imports, file I/O, and network access by
restricting available built-ins.
"""
def __init__(self, transform_code: str, tool_name: str) -> None:
if not isinstance(transform_code, str):
raise TypeError(
f"transform_code must be a str, got {type(transform_code).__name__}"
)
if not transform_code.strip():
raise ValueError("transform_code must not be empty")
if not isinstance(tool_name, str):
raise TypeError(f"tool_name must be a str, got {type(tool_name).__name__}")
self._code = transform_code
self._tool_name = tool_name
self._compiled = compile(self._code, f"<transform:{tool_name}>", "exec")
def execute(self, tool_output: Any) -> dict[str, Any]:
"""Run the transform on *tool_output*.
Returns
-------
dict[str, Any]:
The validation-format result (must contain ``passed``).
Raises
------
TransformExecutionError:
If the transform function is missing, raises, or returns
an invalid result.
"""
sandbox: dict[str, Any] = {"__builtins__": dict(_SAFE_BUILTINS)}
try:
exec(self._compiled, sandbox) # Controlled sandbox execution
except Exception as exc:
raise TransformExecutionError(
self._tool_name,
f"Failed to compile/execute transform code: {exc}",
) from exc
transform_fn = sandbox.get("transform")
if transform_fn is None or not callable(transform_fn):
raise TransformExecutionError(
self._tool_name,
"Transform code must define a callable 'transform' function",
)
try:
result = transform_fn(tool_output)
except Exception as exc:
raise TransformExecutionError(
self._tool_name,
f"Transform function raised: {type(exc).__name__}: {exc}",
) from exc
if not isinstance(result, dict):
raise TransformExecutionError(
self._tool_name,
f"Transform must return a dict, got {type(result).__name__}",
)
if "passed" not in result:
raise TransformExecutionError(
self._tool_name,
"Transform result must contain a 'passed' key",
)
return result
# -------------------------------------------------------------------
# WrappedToolExecutor
# -------------------------------------------------------------------
class WrappedToolExecutor:
"""Resolves ``wraps`` references and delegates execution.
Orchestrates the full wrapping flow:
1. Resolve the wrapped tool from the tool lookup.
2. If the wrapped tool is itself a wrapping validation, recurse.
3. Apply ``argument_mapping`` to translate arguments.
4. Execute the leaf tool.
5. Walk back up the chain applying ``transform`` at each level.
Parameters
----------
tool_lookup:
A callable that accepts a tool name (str) and returns
the ``Validation`` or ``Tool`` domain model, or ``None``
if not found.
tool_executor:
A callable ``(tool_name, mapped_args) -> dict`` that
actually runs the leaf tool and returns its raw output.
"""
def __init__(
self,
tool_lookup: Any,
tool_executor: Any,
) -> None:
if tool_lookup is None:
raise ValueError("tool_lookup must not be None")
if tool_executor is None:
raise ValueError("tool_executor must not be None")
if not callable(tool_lookup):
raise TypeError(
f"tool_lookup must be callable, got {type(tool_lookup).__name__}"
)
if not callable(tool_executor):
raise TypeError(
f"tool_executor must be callable, got {type(tool_executor).__name__}"
)
self._tool_lookup = tool_lookup
self._tool_executor = tool_executor
def execute(
self,
validation: Validation,
inputs: dict[str, Any],
) -> dict[str, Any]:
"""Execute a wrapping validation.
Parameters
----------
validation:
The wrapping ``Validation`` (must have ``wraps`` set).
inputs:
The arguments the validation was invoked with.
Returns
-------
dict[str, Any]:
The validation-format result with ``passed``, optionally
``message`` and ``data``.
Raises
------
WrappedToolNotFoundError:
If any tool in the chain is not registered.
WrappingCycleError:
If the chain contains a cycle.
WrappingDepthExceededError:
If the chain exceeds ``MAX_WRAPPING_DEPTH``.
TransformExecutionError:
If any transform function in the chain fails.
"""
if not isinstance(validation, Validation):
raise TypeError(
f"validation must be a Validation, got {type(validation).__name__}"
)
if not isinstance(inputs, dict):
raise TypeError(f"inputs must be a dict, got {type(inputs).__name__}")
if validation.wraps is None:
raise ValueError(
f"Validation '{validation.name}' does not have "
f"'wraps' set -- use direct execution instead"
)
chain = self._resolve_chain(validation)
return self._execute_chain(chain, inputs)
# -- Chain resolution --------------------------------------------------
def _resolve_chain(
self,
validation: Validation,
) -> list[Validation]:
"""Walk the wraps chain and return the ordered list.
The first element is the outermost wrapper, the last is the
innermost wrapper (whose ``wraps`` target is the leaf tool).
"""
chain: list[Validation] = []
visited: set[str] = set()
current: Validation = validation
while True:
if len(chain) >= MAX_WRAPPING_DEPTH:
raise WrappingDepthExceededError(len(chain))
if current.name in visited:
cycle = [v.name for v in chain] + [current.name]
raise WrappingCycleError(cycle)
visited.add(current.name)
chain.append(current)
wraps_name = current.wraps
if wraps_name is None:
break
wrapped = self._tool_lookup(wraps_name)
if wrapped is None:
raise WrappedToolNotFoundError(current.name, wraps_name)
if isinstance(wrapped, Validation) and wrapped.wraps is not None:
current = wrapped
else:
# Leaf tool found -- stop the chain
break
return chain
# -- Chain execution ---------------------------------------------------
def _execute_chain(
self,
chain: list[Validation],
inputs: dict[str, Any],
) -> dict[str, Any]:
"""Execute the delegation chain.
1. Walk from outermost to innermost, applying argument mappings.
2. Execute the leaf tool.
3. Walk back from innermost to outermost, applying transforms.
"""
# Build the fully-mapped arguments by applying each wrapper's
# argument_mapping in sequence.
mapped_args = dict(inputs)
for wrapper in chain:
mapper = ArgumentMapper(wrapper.argument_mapping)
mapped_args = mapper.apply(mapped_args)
# Execute the leaf tool (the target of the innermost wraps)
leaf_wrapper = chain[-1]
leaf_tool_name = leaf_wrapper.wraps
if leaf_tool_name is None:
raise ValueError(
f"Innermost wrapper '{leaf_wrapper.name}' has no 'wraps' target"
)
logger.info(
"Executing wrapped tool",
extra={
"chain": [v.name for v in chain],
"leaf_tool": leaf_tool_name,
},
)
raw_output = self._tool_executor(leaf_tool_name, mapped_args)
# Apply transforms in reverse (innermost first, outermost last)
result: dict[str, Any] = (
raw_output if isinstance(raw_output, dict) else {"raw": raw_output}
)
for wrapper in reversed(chain):
if wrapper.transform is not None:
transform_exec = TransformExecutor(
wrapper.transform,
wrapper.name,
)
logger.debug(
"Applying transform",
extra={
"wrapper": wrapper.name,
"input_type": type(result).__name__,
},
)
result = transform_exec.execute(result)
return result