feat(validation): implement tool wrapping runtime (wraps + transform delegation) #555

Merged
freemo merged 1 commits from feature/m5-tool-wrapping-runtime into master 2026-03-04 18:42:41 +00:00
12 changed files with 1969 additions and 0 deletions
+185
View File
@@ -0,0 +1,185 @@
"""ASV benchmarks for tool wrapping delegation overhead.
Measures the performance of:
- ArgumentMapper.apply() with identity and configured mappings
- TransformExecutor.execute() with simple transform functions
- WrappedToolExecutor.execute() for single and chained delegation
"""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
from typing import Any
# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from cleveragents.domain.models.core.tool import ( # noqa: E402
Tool,
ToolCapability,
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import ( # noqa: E402
ArgumentMapper,
TransformExecutor,
WrappedToolExecutor,
)
_SIMPLE_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("returncode") == 0
return {"passed": passed, "message": "done", "data": tool_output}
"""
_PASSTHROUGH_TRANSFORM = """\
def transform(tool_output):
return {"passed": True, "message": "ok", "data": tool_output}
"""
def _make_tool(name: str) -> Tool:
return Tool(
name=name,
description=f"Benchmark tool {name}",
source=ToolSource.BUILTIN,
capability=ToolCapability(read_only=True),
)
def _make_validation(
name: str,
wraps: str,
transform_code: str,
argument_mapping: dict[str, Any] | None = None,
) -> Validation:
return Validation(
name=name,
description=f"Benchmark validation wrapping {wraps}",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps=wraps,
transform=transform_code,
argument_mapping=argument_mapping,
)
class ArgumentMapperSuite:
"""Benchmarks for ArgumentMapper."""
def time_identity_mapping(self) -> None:
"""Measure identity (passthrough) mapping overhead."""
mapper = ArgumentMapper(None)
for _ in range(1000):
mapper.apply({"path": "/src", "verbose": True, "count": 42})
def time_configured_mapping(self) -> None:
"""Measure mapping with configured argument translation."""
mapper = ArgumentMapper(
{
"test_directory": "source_dir",
"coverage_enabled": True,
"verbose": False,
}
)
for _ in range(1000):
mapper.apply({"source_dir": "/tests", "extra": "ignored"})
class TransformExecutorSuite:
"""Benchmarks for TransformExecutor."""
def setup(self) -> None:
self._executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
self._output = {"returncode": 0, "tests_run": 100}
def time_execute_transform(self) -> None:
"""Measure transform function execution."""
for _ in range(1000):
self._executor.execute(self._output)
def time_create_and_execute(self) -> None:
"""Measure creation + execution (cold path)."""
for _ in range(100):
executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
executor.execute(self._output)
class WrappedToolExecutorSuite:
"""Benchmarks for WrappedToolExecutor delegation."""
def setup(self) -> None:
self._tool = _make_tool("local/base-tool")
self._output: dict[str, Any] = {"returncode": 0, "data": "ok"}
self._validation = _make_validation(
"local/bench-wrap",
"local/base-tool",
_SIMPLE_TRANSFORM,
)
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
"local/base-tool": (self._tool, self._output),
}
def lookup(name: str) -> Tool | Validation | None:
if name in tools:
return tools[name][0]
return None
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
if name in tools:
return tools[name][1]
raise RuntimeError(f"Not found: {name}")
self._executor = WrappedToolExecutor(lookup, executor)
def time_single_delegation(self) -> None:
"""Measure single-level wrapping delegation."""
for _ in range(1000):
self._executor.execute(self._validation, {"path": "/src"})
def time_chained_delegation(self) -> None:
"""Measure two-level wrapping delegation chain."""
inner_val = _make_validation(
"local/inner-wrap",
"local/base-tool",
_PASSTHROUGH_TRANSFORM,
)
outer_val = _make_validation(
"local/outer-wrap",
"local/inner-wrap",
_SIMPLE_TRANSFORM,
)
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
"local/base-tool": (self._tool, self._output),
}
all_items: dict[str, Tool | Validation] = {
"local/inner-wrap": inner_val,
"local/base-tool": self._tool,
}
def lookup(name: str) -> Tool | Validation | None:
return all_items.get(name)
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
if name in tools:
return tools[name][1]
raise RuntimeError(f"Not found: {name}")
chained_executor = WrappedToolExecutor(lookup, executor)
for _ in range(500):
chained_executor.execute(outer_val, {"path": "/src"})
@@ -0,0 +1,913 @@
"""Step definitions for tool wrapping runtime feature tests."""
from __future__ import annotations
import json
from typing import Any
from behave import given, then, when
from behave.runner import Context
from cleveragents.domain.models.core.tool import (
Tool,
ToolCapability,
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import (
ArgumentMapper,
TransformExecutionError,
TransformExecutor,
WrappedToolExecutor,
WrappedToolNotFoundError,
WrappingCycleError,
WrappingDepthExceededError,
)
# -------------------------------------------------------------------
# Helpers
# -------------------------------------------------------------------
_SIMPLE_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("returncode") == 0
msg = "All tests passed" if passed else "Tests failed"
return {"passed": passed, "message": msg, "data": tool_output}
"""
_PASSTHROUGH_TRANSFORM = """\
def transform(tool_output):
return {"passed": True, "message": "ok", "data": tool_output}
"""
_STATUS_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("passed", False)
return {"passed": passed, "message": "outer", "data": tool_output}
"""
_DATA_TRANSFORM = """\
def transform(tool_output):
return {"passed": True, "message": "data ok", "data": tool_output}
"""
_RETURNCODE_TRANSFORM = """\
def transform(tool_output):
passed = tool_output.get("returncode") == 0
msg = "All tests passed" if passed else "Tests failed"
return {"passed": passed, "message": msg, "data": tool_output}
"""
def _make_validation(
name: str,
wraps: str,
transform_code: str,
argument_mapping: dict[str, Any] | None = None,
) -> Validation:
"""Create a Validation with wraps set."""
return Validation(
name=name,
description=f"Test validation wrapping {wraps}",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps=wraps,
transform=transform_code,
argument_mapping=argument_mapping,
)
def _make_plain_validation(name: str) -> Validation:
"""Create a Validation without wraps."""
return Validation(
name=name,
description="Test validation no wraps",
source=ToolSource.CUSTOM,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
code="pass",
)
def _make_tool(name: str) -> Tool:
"""Create a plain Tool."""
return Tool(
name=name,
description=f"Test tool {name}",
source=ToolSource.BUILTIN,
capability=ToolCapability(read_only=True),
)
# -------------------------------------------------------------------
# ArgumentMapper steps
# -------------------------------------------------------------------
@given("an argument mapper with no mapping")
def step_mapper_no_mapping(context: Context) -> None:
context.mapper = ArgumentMapper(None)
@given("an argument mapper with mapping {mapping_json}")
def step_mapper_with_mapping(context: Context, mapping_json: str) -> None:
mapping = json.loads(mapping_json)
context.mapper = ArgumentMapper(mapping)
@when("I apply the mapper to inputs {inputs_json}")
def step_apply_mapper(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
context.mapped_result = context.mapper.apply(inputs)
@then("the mapped arguments should be {expected_json}")
def step_check_mapped(context: Context, expected_json: str) -> None:
expected = json.loads(expected_json)
assert context.mapped_result == expected, (
f"Expected {expected}, got {context.mapped_result}"
)
@when('I try to apply the mapper to non-dict input "{value}"')
def step_apply_mapper_non_dict(context: Context, value: str) -> None:
context.mapper_error = None
try:
context.mapper.apply(value) # type: ignore[arg-type]
except TypeError as exc:
context.mapper_error = exc
@then("a TypeError should be raised from the argument mapper")
def step_check_mapper_type_error(context: Context) -> None:
assert context.mapper_error is not None, "Expected TypeError"
assert isinstance(context.mapper_error, TypeError)
@when("I try to create an argument mapper with a non-dict mapping")
def step_create_mapper_non_dict(context: Context) -> None:
context.mapper_construction_error = None
try:
ArgumentMapper("not_a_dict") # type: ignore[arg-type]
except TypeError as exc:
context.mapper_construction_error = exc
@then("a TypeError should be raised from argument mapper construction")
def step_check_mapper_construction_error(context: Context) -> None:
assert context.mapper_construction_error is not None, "Expected TypeError"
assert isinstance(context.mapper_construction_error, TypeError)
# -------------------------------------------------------------------
# TransformExecutor steps
# -------------------------------------------------------------------
@given("a transform executor with code that checks returncode equals zero")
def step_transform_returncode(context: Context) -> None:
context.transform_executor = TransformExecutor(
_RETURNCODE_TRANSFORM, "test/transform"
)
@when("I execute the transform with tool output {output_json}")
def step_execute_transform(context: Context, output_json: str) -> None:
output = json.loads(output_json)
context.transform_result = context.transform_executor.execute(output)
@then("the transform result should have passed true")
def step_check_transform_passed_true(context: Context) -> None:
assert context.transform_result["passed"] is True
@then("the transform result should have passed false")
def step_check_transform_passed_false(context: Context) -> None:
assert context.transform_result["passed"] is False
@then('the transform result should have message "{expected_msg}"')
def step_check_transform_message(context: Context, expected_msg: str) -> None:
assert context.transform_result["message"] == expected_msg, (
f"Expected '{expected_msg}', got '{context.transform_result['message']}'"
)
@given("a transform executor with code that does not define a transform function")
def step_transform_no_func(context: Context) -> None:
context.transform_executor = TransformExecutor("x = 1 + 2\n", "test/no-func")
@when("I try to execute the transform with any output")
def step_try_execute_transform(context: Context) -> None:
context.transform_error = None
try:
context.transform_executor.execute({"data": "test"})
except (TransformExecutionError, ValueError) as exc:
context.transform_error = exc
@then('a TransformExecutionError should be raised with message containing "{fragment}"')
def step_check_transform_error(context: Context, fragment: str) -> None:
assert context.transform_error is not None, "Expected TransformExecutionError"
assert isinstance(context.transform_error, TransformExecutionError)
assert fragment in str(context.transform_error), (
f"Expected '{fragment}' in '{context.transform_error}'"
)
@given("a transform executor with code that returns a non-dict value")
def step_transform_non_dict_return(context: Context) -> None:
context.transform_executor = TransformExecutor(
"def transform(x):\n return 'not a dict'\n",
"test/non-dict",
)
@given("a transform executor with code that returns a dict without passed key")
def step_transform_no_passed(context: Context) -> None:
context.transform_executor = TransformExecutor(
'def transform(x):\n return {"message": "no passed"}\n',
"test/no-passed",
)
@when("I try to create a transform executor with empty code")
def step_create_transform_empty(context: Context) -> None:
context.transform_construction_error = None
try:
TransformExecutor(" ", "test/empty")
except ValueError as exc:
context.transform_construction_error = exc
@then("a ValueError should be raised from transform construction")
def step_check_transform_construction_error(context: Context) -> None:
assert context.transform_construction_error is not None, "Expected ValueError"
assert isinstance(context.transform_construction_error, ValueError)
@given("a transform executor with code that attempts to import os")
def step_transform_sandbox_import(context: Context) -> None:
context.transform_executor = TransformExecutor(
"def transform(x):\n import os\n return {'passed': True}\n",
"test/sandbox",
)
@then("a TransformExecutionError should be raised from sandbox restriction")
def step_check_sandbox_error(context: Context) -> None:
assert context.transform_error is not None, (
"Expected TransformExecutionError from sandbox"
)
assert isinstance(context.transform_error, TransformExecutionError)
# -------------------------------------------------------------------
# WrappedToolExecutor steps
# -------------------------------------------------------------------
@given('a tool registry with a tool "{tool_name}" that returns {output_json}')
def step_tool_registry_with_tool(
context: Context,
tool_name: str,
output_json: str,
) -> None:
if not hasattr(context, "test_tools"):
context.test_tools = {}
if not hasattr(context, "test_validations"):
context.test_validations = {}
if not hasattr(context, "tool_call_log"):
context.tool_call_log = {}
output = json.loads(output_json)
context.test_tools[tool_name] = (_make_tool(tool_name), output)
@given('a validation "{val_name}" that wraps "{wraps_name}" with a simple transform')
def step_validation_simple_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _SIMPLE_TRANSFORM)
context.test_validations[val_name] = v
@given(
'a validation "{val_name}" that wraps "{wraps_name}" '
"with argument mapping {mapping_json}"
)
def step_validation_with_mapping(
context: Context,
val_name: str,
wraps_name: str,
mapping_json: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
mapping = json.loads(mapping_json)
v = _make_validation(
val_name,
wraps_name,
_SIMPLE_TRANSFORM,
argument_mapping=mapping,
)
context.test_validations[val_name] = v
@given(
'a validation "{val_name}" that wraps "{wraps_name}" with a passthrough transform'
)
def step_validation_passthrough_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _PASSTHROUGH_TRANSFORM)
context.test_validations[val_name] = v
@given('a validation "{val_name}" that wraps "{wraps_name}" with a status transform')
def step_validation_status_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _STATUS_TRANSFORM)
context.test_validations[val_name] = v
@given('a validation "{val_name}" that wraps "{wraps_name}" with a data transform')
def step_validation_data_wrap(
context: Context,
val_name: str,
wraps_name: str,
) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_validation(val_name, wraps_name, _DATA_TRANSFORM)
context.test_validations[val_name] = v
@given("a wrapped tool executor using the test registry")
def step_create_executor(context: Context) -> None:
tools = getattr(context, "test_tools", {})
validations = getattr(context, "test_validations", {})
context.tool_call_log = {}
def lookup(name: str) -> Tool | Validation | None:
if name in validations:
return validations[name]
if name in tools:
return tools[name][0]
return None
def executor(name: str, args: dict[str, Any]) -> Any:
context.tool_call_log[name] = args
if name in tools:
return tools[name][1]
raise RuntimeError(f"Tool {name} not found in test registry")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@given("a wrapped tool executor with empty registry")
def step_create_empty_executor(context: Context) -> None:
context.tool_call_log = {}
def lookup(name: str) -> None:
return None
def executor(name: str, args: dict[str, Any]) -> Any:
raise RuntimeError(f"Tool {name} not found")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@given("a wrapped tool executor using the test registry with cycle")
def step_create_executor_with_cycle(context: Context) -> None:
validations = getattr(context, "test_validations", {})
def lookup(name: str) -> Validation | None:
return validations.get(name)
def executor(name: str, args: dict[str, Any]) -> Any:
raise RuntimeError("Should not reach leaf executor in cycle")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@when("I execute the wrapping validation with inputs {inputs_json}")
def step_execute_wrapping(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
# Find the first validation with wraps
val = None
for v in context.test_validations.values():
if v.wraps is not None:
val = v
break
assert val is not None, "No wrapping validation found"
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
context.last_executed_validation = val
@when("I execute the outer wrapping validation with inputs {inputs_json}")
def step_execute_outer_wrapping(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
val = context.test_validations.get("local/outer-wrapper")
assert val is not None, "local/outer-wrapper not found"
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
@when("I try to execute the wrapping validation with inputs {inputs_json}")
def step_try_execute_wrapping(context: Context, inputs_json: str) -> None:
inputs = json.loads(inputs_json)
context.wrapped_error = None
# Find first validation with wraps
val = None
for v in getattr(context, "test_validations", {}).values():
if v.wraps is not None:
val = v
break
if val is None:
return
try:
context.wrapped_executor.execute(val, inputs)
except (WrappedToolNotFoundError, WrappingCycleError) as exc:
context.wrapped_error = exc
@when('I try to execute the wrapping validation with cycle from "{val_name}"')
def step_try_execute_cycle(context: Context, val_name: str) -> None:
context.wrapped_error = None
val = context.test_validations[val_name]
try:
context.wrapped_executor.execute(val, {})
except WrappingCycleError as exc:
context.wrapped_error = exc
@then("the wrapped execution should succeed with passed true")
def step_check_wrapped_passed(context: Context) -> None:
assert context.wrapped_result["passed"] is True, (
f"Expected passed=True, got {context.wrapped_result}"
)
@then('the wrapped tool "{tool_name}" should have been called')
def step_check_tool_called(context: Context, tool_name: str) -> None:
assert tool_name in context.tool_call_log, (
f"Tool '{tool_name}' was not called. "
f"Called: {list(context.tool_call_log.keys())}"
)
@then(
'the wrapped tool "{tool_name}" should have received argument '
'"{arg_name}" with value "{arg_value}"'
)
def step_check_tool_arg_str(
context: Context,
tool_name: str,
arg_name: str,
arg_value: str,
) -> None:
call_args = context.tool_call_log[tool_name]
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
assert call_args[arg_name] == arg_value, (
f"Expected '{arg_value}', got '{call_args[arg_name]}'"
)
@then(
'the wrapped tool "{tool_name}" should have received argument '
'"{arg_name}" with value true'
)
def step_check_tool_arg_true(
context: Context,
tool_name: str,
arg_name: str,
) -> None:
call_args = context.tool_call_log[tool_name]
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
assert call_args[arg_name] is True, f"Expected True, got {call_args[arg_name]}"
@then('a WrappedToolNotFoundError should be raised for "{wraps_name}"')
def step_check_not_found_error(context: Context, wraps_name: str) -> None:
assert context.wrapped_error is not None, "Expected WrappedToolNotFoundError"
assert isinstance(context.wrapped_error, WrappedToolNotFoundError)
assert wraps_name in str(context.wrapped_error)
@then("a WrappingCycleError should be raised")
def step_check_cycle_error(context: Context) -> None:
assert context.wrapped_error is not None, "Expected WrappingCycleError"
assert isinstance(context.wrapped_error, WrappingCycleError)
@then("the wrapped tool should have received the validation inputs")
def step_check_inputs_passed(context: Context) -> None:
assert len(context.tool_call_log) > 0, "No tool was called"
@when("I try to execute with a non-Validation object")
def step_try_execute_non_validation(context: Context) -> None:
context.executor_type_error = None
try:
context.wrapped_executor.execute("not_a_validation", {}) # type: ignore[arg-type]
except TypeError as exc:
context.executor_type_error = exc
@then("a TypeError should be raised from the executor")
def step_check_executor_type_error(context: Context) -> None:
assert context.executor_type_error is not None, "Expected TypeError"
assert isinstance(context.executor_type_error, TypeError)
@given('a validation "{val_name}" without wraps set')
def step_validation_no_wraps(context: Context, val_name: str) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
v = _make_plain_validation(val_name)
context.test_validations[val_name] = v
context.no_wrap_validation = v
@when("I try to execute the non-wrapping validation")
def step_try_execute_no_wraps(context: Context) -> None:
context.no_wrap_error = None
try:
context.wrapped_executor.execute(context.no_wrap_validation, {})
except ValueError as exc:
context.no_wrap_error = exc
@then("a ValueError should be raised indicating wraps is not set")
def step_check_no_wrap_error(context: Context) -> None:
assert context.no_wrap_error is not None, "Expected ValueError"
assert isinstance(context.no_wrap_error, ValueError)
assert "wraps" in str(context.no_wrap_error).lower()
# -------------------------------------------------------------------
# Additional coverage: WrappingDepthExceededError
# -------------------------------------------------------------------
@given("a deep wrapping chain of {count:d} validations")
def step_deep_chain(context: Context, count: int) -> None:
if not hasattr(context, "test_validations"):
context.test_validations = {}
if not hasattr(context, "test_tools"):
context.test_tools = {}
# Create a chain: v0 wraps v1, v1 wraps v2, ..., vN wraps leaf-tool
for i in range(count):
wraps_name = f"local/deep-v{i + 1}" if i < count - 1 else "local/deep-leaf"
v = _make_validation(
f"local/deep-v{i}",
wraps_name,
_PASSTHROUGH_TRANSFORM,
)
context.test_validations[v.name] = v
# Create the leaf tool
context.test_tools["local/deep-leaf"] = (
_make_tool("local/deep-leaf"),
{"status": "ok"},
)
@given("a wrapped tool executor using the deep chain registry")
def step_create_deep_chain_executor(context: Context) -> None:
tools = getattr(context, "test_tools", {})
validations = getattr(context, "test_validations", {})
def lookup(name: str) -> Tool | Validation | None:
if name in validations:
return validations[name]
if name in tools:
return tools[name][0]
return None
def executor(name: str, args: dict[str, Any]) -> Any:
if name in tools:
return tools[name][1]
raise RuntimeError(f"Tool {name} not found")
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
@when("I try to execute the deep chain wrapping validation")
def step_try_execute_deep_chain(context: Context) -> None:
context.wrapped_error = None
val = context.test_validations["local/deep-v0"]
try:
context.wrapped_executor.execute(val, {})
except WrappingDepthExceededError as exc:
context.wrapped_error = exc
@then("a WrappingDepthExceededError should be raised with depth {depth:d}")
def step_check_depth_exceeded(context: Context, depth: int) -> None:
assert context.wrapped_error is not None, "Expected WrappingDepthExceededError"
assert isinstance(context.wrapped_error, WrappingDepthExceededError)
assert context.wrapped_error.depth == depth, (
f"Expected depth {depth}, got {context.wrapped_error.depth}"
)
# -------------------------------------------------------------------
# Additional coverage: ArgumentMapper.mapping property
# -------------------------------------------------------------------
@then("the mapper mapping property should return {expected_json}")
def step_check_mapper_property(context: Context, expected_json: str) -> None:
expected = json.loads(expected_json)
assert context.mapper.mapping == expected, (
f"Expected {expected}, got {context.mapper.mapping}"
)
@then("the mapper mapping property should be None")
def step_check_mapper_property_none(context: Context) -> None:
assert context.mapper.mapping is None, (
f"Expected None, got {context.mapper.mapping}"
)
# -------------------------------------------------------------------
# Additional coverage: TransformExecutor type checks
# -------------------------------------------------------------------
@when("I try to create a transform executor with non-string code")
def step_create_transform_non_string_code(context: Context) -> None:
context.transform_type_error = None
try:
TransformExecutor(12345, "test/non-string-code") # type: ignore[arg-type]
except TypeError as exc:
context.transform_type_error = exc
@then("a TypeError should be raised from transform code type check")
def step_check_transform_code_type_error(context: Context) -> None:
assert context.transform_type_error is not None, "Expected TypeError"
assert isinstance(context.transform_type_error, TypeError)
@when("I try to create a transform executor with non-string tool name")
def step_create_transform_non_string_name(context: Context) -> None:
context.transform_name_type_error = None
try:
TransformExecutor("def transform(x): return {'passed': True}", 42) # type: ignore[arg-type]
except TypeError as exc:
context.transform_name_type_error = exc
@then("a TypeError should be raised from transform tool name check")
def step_check_transform_name_type_error(context: Context) -> None:
assert context.transform_name_type_error is not None, "Expected TypeError"
assert isinstance(context.transform_name_type_error, TypeError)
@given("a transform executor with code that raises an error during exec")
def step_transform_exec_error(context: Context) -> None:
# This code has a top-level expression that raises during exec
context.transform_executor = TransformExecutor(
"raise RuntimeError('bad code')\ndef transform(x):\n return {'passed': True}\n",
"test/exec-error",
)
# -------------------------------------------------------------------
# Additional coverage: WrappedToolExecutor init checks
# -------------------------------------------------------------------
@when("I try to create a wrapped tool executor with None tool_lookup")
def step_create_executor_none_lookup(context: Context) -> None:
context.executor_init_error = None
try:
WrappedToolExecutor(None, lambda n, a: {}) # type: ignore[arg-type]
except ValueError as exc:
context.executor_init_error = exc
@then("a ValueError should be raised from executor construction for tool_lookup")
def step_check_executor_none_lookup_error(context: Context) -> None:
assert context.executor_init_error is not None, "Expected ValueError"
assert isinstance(context.executor_init_error, ValueError)
@when("I try to create a wrapped tool executor with None tool_executor")
def step_create_executor_none_executor(context: Context) -> None:
context.executor_init_error = None
try:
WrappedToolExecutor(lambda n: None, None) # type: ignore[arg-type]
except ValueError as exc:
context.executor_init_error = exc
@then("a ValueError should be raised from executor construction for tool_executor")
def step_check_executor_none_executor_error(context: Context) -> None:
assert context.executor_init_error is not None, "Expected ValueError"
assert isinstance(context.executor_init_error, ValueError)
@when("I try to create a wrapped tool executor with non-callable tool_lookup")
def step_create_executor_non_callable_lookup(context: Context) -> None:
context.executor_type_init_error = None
try:
WrappedToolExecutor("not_callable", lambda n, a: {}) # type: ignore[arg-type]
except TypeError as exc:
context.executor_type_init_error = exc
@then("a TypeError should be raised from executor construction for tool_lookup")
def step_check_executor_non_callable_lookup_error(context: Context) -> None:
assert context.executor_type_init_error is not None, "Expected TypeError"
assert isinstance(context.executor_type_init_error, TypeError)
@when("I try to create a wrapped tool executor with non-callable tool_executor")
def step_create_executor_non_callable_executor(context: Context) -> None:
context.executor_type_init_error = None
try:
WrappedToolExecutor(lambda n: None, "not_callable") # type: ignore[arg-type]
except TypeError as exc:
context.executor_type_init_error = exc
@then("a TypeError should be raised from executor construction for tool_executor")
def step_check_executor_non_callable_executor_error(context: Context) -> None:
assert context.executor_type_init_error is not None, "Expected TypeError"
assert isinstance(context.executor_type_init_error, TypeError)
# -------------------------------------------------------------------
# Additional coverage: execute with non-dict inputs
# -------------------------------------------------------------------
@when("I try to execute wrapping validation with non-dict inputs")
def step_try_execute_non_dict_inputs(context: Context) -> None:
context.inputs_type_error = None
val = None
for v in context.test_validations.values():
if v.wraps is not None:
val = v
break
assert val is not None
try:
context.wrapped_executor.execute(val, "not_a_dict") # type: ignore[arg-type]
except TypeError as exc:
context.inputs_type_error = exc
@then("a TypeError should be raised for non-dict inputs")
def step_check_non_dict_inputs_error(context: Context) -> None:
assert context.inputs_type_error is not None, "Expected TypeError"
assert isinstance(context.inputs_type_error, TypeError)
# -------------------------------------------------------------------
# Additional coverage: innermost wrapper with no wraps target
# -------------------------------------------------------------------
# -------------------------------------------------------------------
# ToolRunner coverage steps
# -------------------------------------------------------------------
@given("a ToolRunner with a mock registry")
def step_tool_runner_mock_registry(context: Context) -> None:
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
registry.register(
ToolSpec(
name="local/test-tool",
description="test",
handler=lambda inputs: {"result": "ok"},
)
)
context.tool_runner = ToolRunner(registry)
@when("I call resolve_execution_environment on the runner")
def step_call_resolve_env(context: Context) -> None:
context.resolved_env = context.tool_runner.resolve_execution_environment()
@then("the resolved environment should be local")
def step_check_env_local(context: Context) -> None:
from cleveragents.domain.models.core.plan import ExecutionEnvironment
assert context.resolved_env == ExecutionEnvironment.HOST
@given("a ToolRunner with a value-error-raising env resolver")
def step_runner_value_error_resolver(context: Context) -> None:
from unittest.mock import MagicMock
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
registry.register(
ToolSpec(
name="local/test-tool",
description="test",
handler=lambda inputs: {"result": "ok"},
)
)
runner = ToolRunner(registry)
# Mock the resolver to raise ValueError
mock_resolver = MagicMock()
mock_resolver.resolve_and_validate.side_effect = ValueError("bad env config")
runner._env_resolver = mock_resolver
context.tool_runner = runner
@when("I execute a tool through the runner with env error")
def step_execute_runner_env_error(context: Context) -> None:
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
@then("the tool result should have success false")
def step_check_runner_result_fail(context: Context) -> None:
assert context.runner_result.success is False
@then('the tool result error should contain "{fragment}"')
def step_check_runner_result_error(context: Context, fragment: str) -> None:
assert fragment in (context.runner_result.error or ""), (
f"Expected '{fragment}' in '{context.runner_result.error}'"
)
@given("a ToolRunner with a container-returning env resolver")
def step_runner_container_resolver(context: Context) -> None:
from unittest.mock import MagicMock
from cleveragents.domain.models.core.plan import ExecutionEnvironment
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
registry.register(
ToolSpec(
name="local/test-tool",
description="test",
handler=lambda inputs: {"result": "ok"},
)
)
runner = ToolRunner(registry)
# Mock the resolver to return CONTAINER
mock_resolver = MagicMock()
mock_resolver.resolve_and_validate.return_value = ExecutionEnvironment.CONTAINER
runner._env_resolver = mock_resolver
context.tool_runner = runner
@when("I execute a tool through the runner with container env")
def step_execute_runner_container(context: Context) -> None:
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
@when("I directly call _execute_chain with a no-wraps leaf")
def step_direct_execute_chain_no_wraps(context: Context) -> None:
context.chain_error = None
# Create a plain validation with wraps=None
leaf_val = _make_plain_validation("local/no-wraps-leaf")
try:
# Call the private method directly to exercise lines 437-438
context.wrapped_executor._execute_chain([leaf_val], {})
except ValueError as exc:
context.chain_error = exc
@then("a ValueError should be raised for missing wraps target")
def step_check_missing_wraps_target(context: Context) -> None:
assert context.chain_error is not None, "Expected ValueError"
assert isinstance(context.chain_error, ValueError)
assert "wraps" in str(context.chain_error).lower()
+220
View File
@@ -0,0 +1,220 @@
Feature: Tool wrapping runtime
As the validation execution engine
I need to delegate execution to wrapped tools via wraps + transform
So that validations can reuse existing tool implementations
# ArgumentMapper
Scenario: ArgumentMapper with None mapping passes arguments through
Given an argument mapper with no mapping
When I apply the mapper to inputs {"path": "/src", "verbose": true}
Then the mapped arguments should be {"path": "/src", "verbose": true}
Scenario: ArgumentMapper with mapping translates argument names
Given an argument mapper with mapping {"test_directory": "source_dir", "coverage_enabled": true}
When I apply the mapper to inputs {"source_dir": "/tests"}
Then the mapped arguments should be {"test_directory": "/tests", "coverage_enabled": true}
Scenario: ArgumentMapper with literal values injects fixed values
Given an argument mapper with mapping {"mode": "strict", "count": 42}
When I apply the mapper to inputs {"extra": "ignored"}
Then the mapped arguments should be {"mode": "strict", "count": 42}
Scenario: ArgumentMapper rejects non-dict inputs
Given an argument mapper with no mapping
When I try to apply the mapper to non-dict input "not_a_dict"
Then a TypeError should be raised from the argument mapper
Scenario: ArgumentMapper rejects non-dict mapping at construction
When I try to create an argument mapper with a non-dict mapping
Then a TypeError should be raised from argument mapper construction
# ── TransformExecutor ─────────────────────────────────────────────
Scenario: TransformExecutor runs a valid transform function
Given a transform executor with code that checks returncode equals zero
When I execute the transform with tool output {"returncode": 0, "tests_run": 10}
Then the transform result should have passed true
And the transform result should have message "All tests passed"
Scenario: TransformExecutor handles failing transform output
Given a transform executor with code that checks returncode equals zero
When I execute the transform with tool output {"returncode": 1, "tests_run": 10}
Then the transform result should have passed false
Scenario: TransformExecutor raises on missing transform function
Given a transform executor with code that does not define a transform function
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "callable"
Scenario: TransformExecutor raises on non-dict return value
Given a transform executor with code that returns a non-dict value
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "dict"
Scenario: TransformExecutor raises on missing passed key in result
Given a transform executor with code that returns a dict without passed key
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "passed"
Scenario: TransformExecutor raises on empty transform code
When I try to create a transform executor with empty code
Then a ValueError should be raised from transform construction
Scenario: TransformExecutor sandboxes dangerous operations
Given a transform executor with code that attempts to import os
When I try to execute the transform with any output
Then a TransformExecutionError should be raised from sandbox restriction
# ── WrappedToolExecutor ───────────────────────────────────────────
Scenario: Simple wrapping delegates to the wrapped tool
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
And a validation "local/tests-pass" that wraps "local/run-tests" with a simple transform
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {"path": "/src"}
Then the wrapped execution should succeed with passed true
And the wrapped tool "local/run-tests" should have been called
Scenario: Argument mapping translates arguments to the wrapped tool
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
And a validation "local/mapped-check" that wraps "local/run-tests" with argument mapping {"test_directory": "source_dir", "coverage_enabled": true}
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {"source_dir": "/tests"}
Then the wrapped tool "local/run-tests" should have received argument "test_directory" with value "/tests"
And the wrapped tool "local/run-tests" should have received argument "coverage_enabled" with value true
Scenario: Chained wrapping delegates through the chain
Given a tool registry with a tool "local/base-tool" that returns {"status": "ok"}
And a validation "local/mid-wrapper" that wraps "local/base-tool" with a passthrough transform
And a validation "local/outer-wrapper" that wraps "local/mid-wrapper" with a status transform
And a wrapped tool executor using the test registry
When I execute the outer wrapping validation with inputs {}
Then the wrapped execution should succeed with passed true
And the wrapped tool "local/base-tool" should have been called
Scenario: Missing wrapped tool raises WrappedToolNotFoundError
Given a validation "local/broken-wrap" that wraps "local/nonexistent" with a simple transform
And a wrapped tool executor with empty registry
When I try to execute the wrapping validation with inputs {}
Then a WrappedToolNotFoundError should be raised for "local/nonexistent"
Scenario: Circular wrapping chain raises WrappingCycleError
Given a validation "local/wrap-a" that wraps "local/wrap-b" with a simple transform
And a validation "local/wrap-b" that wraps "local/wrap-a" with a simple transform
And a wrapped tool executor using the test registry with cycle
When I try to execute the wrapping validation with cycle from "local/wrap-a"
Then a WrappingCycleError should be raised
Scenario: Execution context is inherited from wrapping validation
Given a tool registry with a tool "local/context-tool" that returns {"data": "value"}
And a validation "local/context-wrap" that wraps "local/context-tool" with a data transform
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {"key": "value"}
Then the wrapped tool should have received the validation inputs
Scenario: WrappedToolExecutor rejects non-Validation argument
Given a wrapped tool executor with empty registry
When I try to execute with a non-Validation object
Then a TypeError should be raised from the executor
Scenario: WrappedToolExecutor rejects validation without wraps
Given a validation "local/no-wrap" without wraps set
And a wrapped tool executor with empty registry
When I try to execute the non-wrapping validation
Then a ValueError should be raised indicating wraps is not set
# ── Additional coverage: WrappingDepthExceededError ──────────────
Scenario: WrappingDepthExceededError is raised when chain is too deep
Given a deep wrapping chain of 11 validations
And a wrapped tool executor using the deep chain registry
When I try to execute the deep chain wrapping validation
Then a WrappingDepthExceededError should be raised with depth 10
# ── Additional coverage: ArgumentMapper.mapping property ─────────
Scenario: ArgumentMapper exposes the raw mapping via property
Given an argument mapper with mapping {"x": "y"}
Then the mapper mapping property should return {"x": "y"}
Scenario: ArgumentMapper mapping property returns None for identity mapper
Given an argument mapper with no mapping
Then the mapper mapping property should be None
# ── Additional coverage: TransformExecutor type checks ───────────
Scenario: TransformExecutor rejects non-string transform code
When I try to create a transform executor with non-string code
Then a TypeError should be raised from transform code type check
Scenario: TransformExecutor rejects non-string tool name
When I try to create a transform executor with non-string tool name
Then a TypeError should be raised from transform tool name check
Scenario: TransformExecutor raises on code that throws during exec
Given a transform executor with code that raises an error during exec
When I try to execute the transform with any output
Then a TransformExecutionError should be raised with message containing "compile"
# ── Additional coverage: WrappedToolExecutor init checks ─────────
Scenario: WrappedToolExecutor rejects None tool_lookup
When I try to create a wrapped tool executor with None tool_lookup
Then a ValueError should be raised from executor construction for tool_lookup
Scenario: WrappedToolExecutor rejects None tool_executor
When I try to create a wrapped tool executor with None tool_executor
Then a ValueError should be raised from executor construction for tool_executor
Scenario: WrappedToolExecutor rejects non-callable tool_lookup
When I try to create a wrapped tool executor with non-callable tool_lookup
Then a TypeError should be raised from executor construction for tool_lookup
Scenario: WrappedToolExecutor rejects non-callable tool_executor
When I try to create a wrapped tool executor with non-callable tool_executor
Then a TypeError should be raised from executor construction for tool_executor
# ── Additional coverage: execute with non-dict inputs ────────────
Scenario: WrappedToolExecutor rejects non-dict inputs
Given a tool registry with a tool "local/some-tool" that returns {"ok": true}
And a validation "local/input-wrap" that wraps "local/some-tool" with a simple transform
And a wrapped tool executor using the test registry
When I try to execute wrapping validation with non-dict inputs
Then a TypeError should be raised for non-dict inputs
# ── Additional coverage: chain with None wraps at end ────────────
Scenario: Chain resolution handles validation with wraps set to None in chain
Given a tool registry with a tool "local/leaf" that returns {"value": 1}
And a validation "local/wrapper-no-inner" that wraps "local/leaf" with a passthrough transform
And a wrapped tool executor using the test registry
When I execute the wrapping validation with inputs {}
Then the wrapped execution should succeed with passed true
# ── Additional coverage: _execute_chain with None wraps target ────
Scenario: _execute_chain raises ValueError when leaf wraps is None
Given a wrapped tool executor with empty registry
When I directly call _execute_chain with a no-wraps leaf
Then a ValueError should be raised for missing wraps target
# ── ToolRunner coverage: execution environment and error paths ───
Scenario: ToolRunner resolve_execution_environment delegates to resolver
Given a ToolRunner with a mock registry
When I call resolve_execution_environment on the runner
Then the resolved environment should be local
Scenario: ToolRunner execute returns error when env resolver raises ValueError
Given a ToolRunner with a value-error-raising env resolver
When I execute a tool through the runner with env error
Then the tool result should have success false
And the tool result error should contain "Execution environment error"
Scenario: ToolRunner execute returns error for container environment
Given a ToolRunner with a container-returning env resolver
When I execute a tool through the runner with container env
Then the tool result should have success false
And the tool result error should contain "Container execution is not yet implemented"
+42
View File
@@ -0,0 +1,42 @@
"""Robot helper: verify WrappedToolExecutor delegates to wrapped tool."""
from cleveragents.domain.models.core.tool import (
Tool,
ToolCapability,
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import WrappedToolExecutor
tool = Tool(
name="local/run",
description="run",
source=ToolSource.BUILTIN,
capability=ToolCapability(read_only=True),
)
val = Validation(
name="local/check",
description="check",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps="local/run",
transform='def transform(x):\n return {"passed": True, "data": x}\n',
)
lookup = lambda n: tool if n == "local/run" else None # noqa: E731
called: list[str] = []
def executor_fn(n: str, a: dict) -> dict: # type: ignore[type-arg]
"""Record tool call and return output."""
called.append(n)
return {"result": "ok"}
ex = WrappedToolExecutor(lookup, executor_fn)
r = ex.execute(val, {"a": 1})
assert r["passed"] is True
assert "local/run" in called
print("OK")
+8
View File
@@ -0,0 +1,8 @@
"""Robot helper: verify ArgumentMapper translates arguments."""
from cleveragents.tool.wrapping import ArgumentMapper
m = ArgumentMapper({"dest": "src", "flag": True})
r = m.apply({"src": "/path"})
assert r == {"dest": "/path", "flag": True}, f"Got {r}"
print("OK")
+8
View File
@@ -0,0 +1,8 @@
"""Robot helper: verify ArgumentMapper with None mapping."""
from cleveragents.tool.wrapping import ArgumentMapper
m = ArgumentMapper(None)
r = m.apply({"x": 1, "y": "two"})
assert r == {"x": 1, "y": "two"}, f"Got {r}"
print("OK")
+28
View File
@@ -0,0 +1,28 @@
"""Robot helper: verify WrappedToolExecutor raises on missing tool."""
from cleveragents.domain.models.core.tool import (
ToolSource,
ToolType,
Validation,
ValidationMode,
)
from cleveragents.tool.wrapping import WrappedToolExecutor, WrappedToolNotFoundError
val = Validation(
name="local/check",
description="check",
source=ToolSource.WRAPPED,
tool_type=ToolType.VALIDATION,
mode=ValidationMode.REQUIRED,
wraps="local/missing",
transform='def transform(x):\n return {"passed": True}\n',
)
ok = False
try:
WrappedToolExecutor(
lambda n: None,
lambda n, a: {},
).execute(val, {})
except WrappedToolNotFoundError:
ok = True
print(f"error_raised={ok}")
+9
View File
@@ -0,0 +1,9 @@
"""Robot helper: verify TransformExecutor runs a transform."""
from cleveragents.tool.wrapping import TransformExecutor
code = 'def transform(x):\n return {"passed": x.get("ok"), "message": "done"}\n'
t = TransformExecutor(code, "test/t")
r = t.execute({"ok": True})
assert r["passed"] is True
print("OK")
+12
View File
@@ -0,0 +1,12 @@
"""Robot helper: verify TransformExecutor blocks import in sandbox."""
from cleveragents.tool.wrapping import TransformExecutionError, TransformExecutor
code = 'def transform(x):\n import os\n return {"passed": True}\n'
t = TransformExecutor(code, "test/s")
ok = False
try:
t.execute({})
except TransformExecutionError:
ok = True
print(f"blocked={ok}")
+58
View File
@@ -0,0 +1,58 @@
*** Settings ***
Documentation Tool wrapping runtime smoke tests
Library Process
Library OperatingSystem
*** Variables ***
${PYTHON} python
*** Test Cases ***
Wrapping Package Is Importable
[Documentation] Verify tool wrapping module can be imported
${result}= Run Process ${PYTHON} -c
... from cleveragents.tool.wrapping import ArgumentMapper, TransformExecutor, WrappedToolExecutor, WrappedToolNotFoundError, WrappingCycleError, WrappingDepthExceededError, TransformExecutionError; print('OK')
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} OK
Wrapping Exports In Tool Init
[Documentation] Verify wrapping types are exported from tool package
${result}= Run Process ${PYTHON} -c
... from cleveragents.tool import ArgumentMapper, TransformExecutor, WrappedToolExecutor; print('OK')
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} OK
ArgumentMapper Identity Mapping
[Documentation] Verify ArgumentMapper with None mapping passes args through
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_identity.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
ArgumentMapper With Mapping
[Documentation] Verify ArgumentMapper translates arguments
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_configured.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
TransformExecutor Runs Transform
[Documentation] Verify TransformExecutor executes transform code
${result}= Run Process ${PYTHON} robot/scripts/test_transform_run.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
TransformExecutor Sandbox Blocks Import
[Documentation] Verify TransformExecutor blocks import in sandbox
${result}= Run Process ${PYTHON} robot/scripts/test_transform_sandbox.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} blocked=True
WrappedToolExecutor Simple Delegation
[Documentation] Verify WrappedToolExecutor delegates to wrapped tool
${result}= Run Process ${PYTHON} robot/scripts/test_delegation.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} OK
WrappedToolExecutor Missing Tool Error
[Documentation] Verify WrappedToolExecutor raises on missing wrapped tool
${result}= Run Process ${PYTHON} robot/scripts/test_missing_tool.py
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
Should Contain ${result.stdout} error_raised=True
+16
View File
@@ -69,8 +69,18 @@ from cleveragents.tool.schema_validator import (
validate_tool_input,
validate_tool_output,
)
from cleveragents.tool.wrapping import (
ArgumentMapper,
TransformExecutionError,
TransformExecutor,
WrappedToolExecutor,
WrappedToolNotFoundError,
WrappingCycleError,
WrappingDepthExceededError,
)
__all__ = [
"ArgumentMapper",
"BoundResource",
"CancellationToken",
"Change",
@@ -116,6 +126,12 @@ __all__ = [
"ToolSandboxRequiredError",
"ToolSchemaValidationError",
"ToolSpec",
"TransformExecutionError",
"TransformExecutor",
"WrappedToolExecutor",
"WrappedToolNotFoundError",
"WrappingCycleError",
"WrappingDepthExceededError",
"classify_tool_error",
"detect_provider_format",
"generate_tool_call_id",
+470
View File
@@ -0,0 +1,470 @@
"""Tool wrapping runtime for CleverAgents v3.
Implements the ``wraps`` + ``transform`` + ``argument_mapping`` delegation
mechanism described in the specification (§ Core Concepts > Validation >
Tool Wrapping).
## Architecture
```
WrappedToolExecutor
|-- ArgumentMapper (translate wrapper args -> wrapped tool args)
|-- TransformExecutor (sandboxed transform function on tool output)
+-- ToolRuntime (resolve + invoke wrapped tools)
```
## Delegation Chain
A validation wrapping another validation forms a delegation chain:
```
validation-A -> validation-B -> tool-C
(wraps B) (wraps C) (leaf)
```
``WrappedToolExecutor.execute()`` recursively resolves the chain,
detecting cycles and enforcing a maximum depth.
## Sandbox
``TransformExecutor`` compiles and executes the user-supplied
``transform`` function with a restricted set of built-in names.
No filesystem, network, or import access is available inside the
transform.
Based on ``docs/specification.md`` § Tool Wrapping.
"""
from __future__ import annotations
import logging
from typing import Any
from cleveragents.domain.models.core.tool import Validation
logger = logging.getLogger(__name__)
# Maximum delegation chain depth to prevent infinite recursion.
MAX_WRAPPING_DEPTH = 10
# Safe built-ins available inside the ``transform`` sandbox.
_SAFE_BUILTINS: dict[str, Any] = {
"True": True,
"False": False,
"None": None,
"abs": abs,
"all": all,
"any": any,
"bool": bool,
"dict": dict,
"enumerate": enumerate,
"float": float,
"frozenset": frozenset,
"int": int,
"isinstance": isinstance,
"len": len,
"list": list,
"map": map,
"max": max,
"min": min,
"range": range,
"repr": repr,
"reversed": reversed,
"round": round,
"set": set,
"sorted": sorted,
"str": str,
"sum": sum,
"tuple": tuple,
"type": type,
"zip": zip,
}
# -------------------------------------------------------------------
# Errors
# -------------------------------------------------------------------
class WrappedToolNotFoundError(Exception):
"""Raised when the tool referenced by ``wraps`` is not registered."""
def __init__(self, tool_name: str, wraps_name: str) -> None:
self.tool_name = tool_name
self.wraps_name = wraps_name
super().__init__(
f"Validation '{tool_name}' wraps '{wraps_name}' but "
f"'{wraps_name}' is not registered in the runtime"
)
class WrappingCycleError(Exception):
"""Raised when a circular wrapping chain is detected."""
def __init__(self, chain: list[str]) -> None:
self.chain = list(chain)
cycle_str = " -> ".join(chain)
super().__init__(f"Circular wrapping chain detected: {cycle_str}")
class WrappingDepthExceededError(Exception):
"""Raised when the delegation chain exceeds ``MAX_WRAPPING_DEPTH``."""
def __init__(self, depth: int) -> None:
self.depth = depth
super().__init__(
f"Wrapping delegation chain depth ({depth}) exceeds "
f"the maximum ({MAX_WRAPPING_DEPTH})"
)
class TransformExecutionError(Exception):
"""Raised when the ``transform`` function fails during execution."""
def __init__(self, tool_name: str, detail: str) -> None:
self.tool_name = tool_name
self.detail = detail
super().__init__(f"Transform function for '{tool_name}' failed: {detail}")
# -------------------------------------------------------------------
# ArgumentMapper
# -------------------------------------------------------------------
class ArgumentMapper:
"""Applies ``argument_mapping`` to translate arguments.
When a validation wraps a tool, the validation may accept different
input arguments. The ``argument_mapping`` dictionary specifies how
the validation's inputs map to the wrapped tool's expected inputs.
Keys are the wrapped tool's parameter names. Values are either:
- A string referencing a validation input parameter (forwarded).
- A literal value (string, number, boolean) always sent to the
wrapped tool.
When ``argument_mapping`` is ``None``, input arguments are passed
through to the wrapped tool unchanged (identity mapping).
"""
def __init__(self, mapping: dict[str, Any] | None) -> None:
if mapping is not None and not isinstance(mapping, dict):
raise TypeError(
f"argument_mapping must be a dict or None, got {type(mapping).__name__}"
)
self._mapping = mapping
@property
def mapping(self) -> dict[str, Any] | None:
"""Return the raw mapping configuration."""
return self._mapping
def apply(self, validation_inputs: dict[str, Any]) -> dict[str, Any]:
"""Translate *validation_inputs* into wrapped-tool arguments.
Parameters
----------
validation_inputs:
The arguments the validation was invoked with.
Returns
-------
dict[str, Any]:
Arguments to forward to the wrapped tool.
"""
if not isinstance(validation_inputs, dict):
raise TypeError(
f"validation_inputs must be a dict, "
f"got {type(validation_inputs).__name__}"
)
if self._mapping is None:
return dict(validation_inputs)
mapped: dict[str, Any] = {}
for wrapped_param, source in self._mapping.items():
if isinstance(source, str) and source in validation_inputs:
mapped[wrapped_param] = validation_inputs[source]
else:
# Literal value (or string not present as a key -- use
# as literal).
mapped[wrapped_param] = source
return mapped
# -------------------------------------------------------------------
# TransformExecutor
# -------------------------------------------------------------------
class TransformExecutor:
"""Executes the ``transform`` function in a sandboxed context.
The transform code must define a function called ``transform`` that
accepts one positional argument (the wrapped tool's output) and
returns a dict with at minimum ``{"passed": bool}``.
The sandbox disallows imports, file I/O, and network access by
restricting available built-ins.
"""
def __init__(self, transform_code: str, tool_name: str) -> None:
if not isinstance(transform_code, str):
raise TypeError(
f"transform_code must be a str, got {type(transform_code).__name__}"
)
if not transform_code.strip():
raise ValueError("transform_code must not be empty")
if not isinstance(tool_name, str):
raise TypeError(f"tool_name must be a str, got {type(tool_name).__name__}")
self._code = transform_code
self._tool_name = tool_name
self._compiled = compile(self._code, f"<transform:{tool_name}>", "exec")
def execute(self, tool_output: Any) -> dict[str, Any]:
"""Run the transform on *tool_output*.
Returns
-------
dict[str, Any]:
The validation-format result (must contain ``passed``).
Raises
------
TransformExecutionError:
If the transform function is missing, raises, or returns
an invalid result.
"""
sandbox: dict[str, Any] = {"__builtins__": dict(_SAFE_BUILTINS)}
try:
exec(self._compiled, sandbox) # Controlled sandbox execution
except Exception as exc:
raise TransformExecutionError(
self._tool_name,
f"Failed to compile/execute transform code: {exc}",
) from exc
transform_fn = sandbox.get("transform")
if transform_fn is None or not callable(transform_fn):
raise TransformExecutionError(
self._tool_name,
"Transform code must define a callable 'transform' function",
)
try:
result = transform_fn(tool_output)
except Exception as exc:
raise TransformExecutionError(
self._tool_name,
f"Transform function raised: {type(exc).__name__}: {exc}",
) from exc
if not isinstance(result, dict):
raise TransformExecutionError(
self._tool_name,
f"Transform must return a dict, got {type(result).__name__}",
)
if "passed" not in result:
raise TransformExecutionError(
self._tool_name,
"Transform result must contain a 'passed' key",
)
return result
# -------------------------------------------------------------------
# WrappedToolExecutor
# -------------------------------------------------------------------
class WrappedToolExecutor:
"""Resolves ``wraps`` references and delegates execution.
Orchestrates the full wrapping flow:
1. Resolve the wrapped tool from the tool lookup.
2. If the wrapped tool is itself a wrapping validation, recurse.
3. Apply ``argument_mapping`` to translate arguments.
4. Execute the leaf tool.
5. Walk back up the chain applying ``transform`` at each level.
Parameters
----------
tool_lookup:
A callable that accepts a tool name (str) and returns
the ``Validation`` or ``Tool`` domain model, or ``None``
if not found.
tool_executor:
A callable ``(tool_name, mapped_args) -> dict`` that
actually runs the leaf tool and returns its raw output.
"""
def __init__(
self,
tool_lookup: Any,
tool_executor: Any,
) -> None:
if tool_lookup is None:
raise ValueError("tool_lookup must not be None")
if tool_executor is None:
raise ValueError("tool_executor must not be None")
if not callable(tool_lookup):
raise TypeError(
f"tool_lookup must be callable, got {type(tool_lookup).__name__}"
)
if not callable(tool_executor):
raise TypeError(
f"tool_executor must be callable, got {type(tool_executor).__name__}"
)
self._tool_lookup = tool_lookup
self._tool_executor = tool_executor
def execute(
self,
validation: Validation,
inputs: dict[str, Any],
) -> dict[str, Any]:
"""Execute a wrapping validation.
Parameters
----------
validation:
The wrapping ``Validation`` (must have ``wraps`` set).
inputs:
The arguments the validation was invoked with.
Returns
-------
dict[str, Any]:
The validation-format result with ``passed``, optionally
``message`` and ``data``.
Raises
------
WrappedToolNotFoundError:
If any tool in the chain is not registered.
WrappingCycleError:
If the chain contains a cycle.
WrappingDepthExceededError:
If the chain exceeds ``MAX_WRAPPING_DEPTH``.
TransformExecutionError:
If any transform function in the chain fails.
"""
if not isinstance(validation, Validation):
raise TypeError(
f"validation must be a Validation, got {type(validation).__name__}"
)
if not isinstance(inputs, dict):
raise TypeError(f"inputs must be a dict, got {type(inputs).__name__}")
if validation.wraps is None:
raise ValueError(
f"Validation '{validation.name}' does not have "
f"'wraps' set -- use direct execution instead"
)
chain = self._resolve_chain(validation)
return self._execute_chain(chain, inputs)
# -- Chain resolution --------------------------------------------------
def _resolve_chain(
self,
validation: Validation,
) -> list[Validation]:
"""Walk the wraps chain and return the ordered list.
The first element is the outermost wrapper, the last is the
innermost wrapper (whose ``wraps`` target is the leaf tool).
"""
chain: list[Validation] = []
visited: set[str] = set()
current: Validation = validation
while True:
if len(chain) >= MAX_WRAPPING_DEPTH:
raise WrappingDepthExceededError(len(chain))
if current.name in visited:
cycle = [v.name for v in chain] + [current.name]
raise WrappingCycleError(cycle)
visited.add(current.name)
chain.append(current)
wraps_name = current.wraps
if wraps_name is None:
break
wrapped = self._tool_lookup(wraps_name)
if wrapped is None:
raise WrappedToolNotFoundError(current.name, wraps_name)
if isinstance(wrapped, Validation) and wrapped.wraps is not None:
current = wrapped
else:
# Leaf tool found -- stop the chain
break
return chain
# -- Chain execution ---------------------------------------------------
def _execute_chain(
self,
chain: list[Validation],
inputs: dict[str, Any],
) -> dict[str, Any]:
"""Execute the delegation chain.
1. Walk from outermost to innermost, applying argument mappings.
2. Execute the leaf tool.
3. Walk back from innermost to outermost, applying transforms.
"""
# Build the fully-mapped arguments by applying each wrapper's
# argument_mapping in sequence.
mapped_args = dict(inputs)
for wrapper in chain:
mapper = ArgumentMapper(wrapper.argument_mapping)
mapped_args = mapper.apply(mapped_args)
# Execute the leaf tool (the target of the innermost wraps)
leaf_wrapper = chain[-1]
leaf_tool_name = leaf_wrapper.wraps
if leaf_tool_name is None:
raise ValueError(
f"Innermost wrapper '{leaf_wrapper.name}' has no 'wraps' target"
)
logger.info(
"Executing wrapped tool",
extra={
"chain": [v.name for v in chain],
"leaf_tool": leaf_tool_name,
},
)
raw_output = self._tool_executor(leaf_tool_name, mapped_args)
# Apply transforms in reverse (innermost first, outermost last)
result: dict[str, Any] = (
raw_output if isinstance(raw_output, dict) else {"raw": raw_output}
)
for wrapper in reversed(chain):
if wrapper.transform is not None:
transform_exec = TransformExecutor(
wrapper.transform,
wrapper.name,
)
logger.debug(
"Applying transform",
extra={
"wrapper": wrapper.name,
"input_type": type(result).__name__,
},
)
result = transform_exec.execute(result)
return result