feat(validation): implement tool wrapping runtime (wraps + transform delegation) #555
@@ -0,0 +1,185 @@
|
||||
"""ASV benchmarks for tool wrapping delegation overhead.
|
||||
|
||||
Measures the performance of:
|
||||
- ArgumentMapper.apply() with identity and configured mappings
|
||||
- TransformExecutor.execute() with simple transform functions
|
||||
- WrappedToolExecutor.execute() for single and chained delegation
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.domain.models.core.tool import ( # noqa: E402
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import ( # noqa: E402
|
||||
ArgumentMapper,
|
||||
TransformExecutor,
|
||||
WrappedToolExecutor,
|
||||
)
|
||||
|
||||
_SIMPLE_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("returncode") == 0
|
||||
return {"passed": passed, "message": "done", "data": tool_output}
|
||||
"""
|
||||
|
||||
_PASSTHROUGH_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
return {"passed": True, "message": "ok", "data": tool_output}
|
||||
"""
|
||||
|
||||
|
||||
def _make_tool(name: str) -> Tool:
|
||||
return Tool(
|
||||
name=name,
|
||||
description=f"Benchmark tool {name}",
|
||||
source=ToolSource.BUILTIN,
|
||||
capability=ToolCapability(read_only=True),
|
||||
)
|
||||
|
||||
|
||||
def _make_validation(
|
||||
name: str,
|
||||
wraps: str,
|
||||
transform_code: str,
|
||||
argument_mapping: dict[str, Any] | None = None,
|
||||
) -> Validation:
|
||||
return Validation(
|
||||
name=name,
|
||||
description=f"Benchmark validation wrapping {wraps}",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps=wraps,
|
||||
transform=transform_code,
|
||||
argument_mapping=argument_mapping,
|
||||
)
|
||||
|
||||
|
||||
class ArgumentMapperSuite:
|
||||
"""Benchmarks for ArgumentMapper."""
|
||||
|
||||
def time_identity_mapping(self) -> None:
|
||||
"""Measure identity (passthrough) mapping overhead."""
|
||||
mapper = ArgumentMapper(None)
|
||||
for _ in range(1000):
|
||||
mapper.apply({"path": "/src", "verbose": True, "count": 42})
|
||||
|
||||
def time_configured_mapping(self) -> None:
|
||||
"""Measure mapping with configured argument translation."""
|
||||
mapper = ArgumentMapper(
|
||||
{
|
||||
"test_directory": "source_dir",
|
||||
"coverage_enabled": True,
|
||||
"verbose": False,
|
||||
}
|
||||
)
|
||||
for _ in range(1000):
|
||||
mapper.apply({"source_dir": "/tests", "extra": "ignored"})
|
||||
|
||||
|
||||
class TransformExecutorSuite:
|
||||
"""Benchmarks for TransformExecutor."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
|
||||
self._output = {"returncode": 0, "tests_run": 100}
|
||||
|
||||
def time_execute_transform(self) -> None:
|
||||
"""Measure transform function execution."""
|
||||
for _ in range(1000):
|
||||
self._executor.execute(self._output)
|
||||
|
||||
def time_create_and_execute(self) -> None:
|
||||
"""Measure creation + execution (cold path)."""
|
||||
for _ in range(100):
|
||||
executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
|
||||
executor.execute(self._output)
|
||||
|
||||
|
||||
class WrappedToolExecutorSuite:
|
||||
"""Benchmarks for WrappedToolExecutor delegation."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._tool = _make_tool("local/base-tool")
|
||||
self._output: dict[str, Any] = {"returncode": 0, "data": "ok"}
|
||||
self._validation = _make_validation(
|
||||
"local/bench-wrap",
|
||||
"local/base-tool",
|
||||
_SIMPLE_TRANSFORM,
|
||||
)
|
||||
|
||||
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
|
||||
"local/base-tool": (self._tool, self._output),
|
||||
}
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
if name in tools:
|
||||
return tools[name][0]
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Not found: {name}")
|
||||
|
||||
self._executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
def time_single_delegation(self) -> None:
|
||||
"""Measure single-level wrapping delegation."""
|
||||
for _ in range(1000):
|
||||
self._executor.execute(self._validation, {"path": "/src"})
|
||||
|
||||
def time_chained_delegation(self) -> None:
|
||||
"""Measure two-level wrapping delegation chain."""
|
||||
inner_val = _make_validation(
|
||||
"local/inner-wrap",
|
||||
"local/base-tool",
|
||||
_PASSTHROUGH_TRANSFORM,
|
||||
)
|
||||
outer_val = _make_validation(
|
||||
"local/outer-wrap",
|
||||
"local/inner-wrap",
|
||||
_SIMPLE_TRANSFORM,
|
||||
)
|
||||
|
||||
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
|
||||
"local/base-tool": (self._tool, self._output),
|
||||
}
|
||||
|
||||
all_items: dict[str, Tool | Validation] = {
|
||||
"local/inner-wrap": inner_val,
|
||||
"local/base-tool": self._tool,
|
||||
}
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
return all_items.get(name)
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Not found: {name}")
|
||||
|
||||
chained_executor = WrappedToolExecutor(lookup, executor)
|
||||
for _ in range(500):
|
||||
chained_executor.execute(outer_val, {"path": "/src"})
|
||||
@@ -0,0 +1,913 @@
|
||||
"""Step definitions for tool wrapping runtime feature tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import (
|
||||
ArgumentMapper,
|
||||
TransformExecutionError,
|
||||
TransformExecutor,
|
||||
WrappedToolExecutor,
|
||||
WrappedToolNotFoundError,
|
||||
WrappingCycleError,
|
||||
WrappingDepthExceededError,
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Helpers
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
_SIMPLE_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("returncode") == 0
|
||||
msg = "All tests passed" if passed else "Tests failed"
|
||||
return {"passed": passed, "message": msg, "data": tool_output}
|
||||
"""
|
||||
|
||||
_PASSTHROUGH_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
return {"passed": True, "message": "ok", "data": tool_output}
|
||||
"""
|
||||
|
||||
_STATUS_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("passed", False)
|
||||
return {"passed": passed, "message": "outer", "data": tool_output}
|
||||
"""
|
||||
|
||||
_DATA_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
return {"passed": True, "message": "data ok", "data": tool_output}
|
||||
"""
|
||||
|
||||
_RETURNCODE_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("returncode") == 0
|
||||
msg = "All tests passed" if passed else "Tests failed"
|
||||
return {"passed": passed, "message": msg, "data": tool_output}
|
||||
"""
|
||||
|
||||
|
||||
def _make_validation(
|
||||
name: str,
|
||||
wraps: str,
|
||||
transform_code: str,
|
||||
argument_mapping: dict[str, Any] | None = None,
|
||||
) -> Validation:
|
||||
"""Create a Validation with wraps set."""
|
||||
return Validation(
|
||||
name=name,
|
||||
description=f"Test validation wrapping {wraps}",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps=wraps,
|
||||
transform=transform_code,
|
||||
argument_mapping=argument_mapping,
|
||||
)
|
||||
|
||||
|
||||
def _make_plain_validation(name: str) -> Validation:
|
||||
"""Create a Validation without wraps."""
|
||||
return Validation(
|
||||
name=name,
|
||||
description="Test validation no wraps",
|
||||
source=ToolSource.CUSTOM,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
code="pass",
|
||||
)
|
||||
|
||||
|
||||
def _make_tool(name: str) -> Tool:
|
||||
"""Create a plain Tool."""
|
||||
return Tool(
|
||||
name=name,
|
||||
description=f"Test tool {name}",
|
||||
source=ToolSource.BUILTIN,
|
||||
capability=ToolCapability(read_only=True),
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ArgumentMapper steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("an argument mapper with no mapping")
|
||||
def step_mapper_no_mapping(context: Context) -> None:
|
||||
context.mapper = ArgumentMapper(None)
|
||||
|
||||
|
||||
@given("an argument mapper with mapping {mapping_json}")
|
||||
def step_mapper_with_mapping(context: Context, mapping_json: str) -> None:
|
||||
mapping = json.loads(mapping_json)
|
||||
context.mapper = ArgumentMapper(mapping)
|
||||
|
||||
|
||||
@when("I apply the mapper to inputs {inputs_json}")
|
||||
def step_apply_mapper(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
context.mapped_result = context.mapper.apply(inputs)
|
||||
|
||||
|
||||
@then("the mapped arguments should be {expected_json}")
|
||||
def step_check_mapped(context: Context, expected_json: str) -> None:
|
||||
expected = json.loads(expected_json)
|
||||
assert context.mapped_result == expected, (
|
||||
f"Expected {expected}, got {context.mapped_result}"
|
||||
)
|
||||
|
||||
|
||||
@when('I try to apply the mapper to non-dict input "{value}"')
|
||||
def step_apply_mapper_non_dict(context: Context, value: str) -> None:
|
||||
context.mapper_error = None
|
||||
try:
|
||||
context.mapper.apply(value) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.mapper_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from the argument mapper")
|
||||
def step_check_mapper_type_error(context: Context) -> None:
|
||||
assert context.mapper_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.mapper_error, TypeError)
|
||||
|
||||
|
||||
@when("I try to create an argument mapper with a non-dict mapping")
|
||||
def step_create_mapper_non_dict(context: Context) -> None:
|
||||
context.mapper_construction_error = None
|
||||
try:
|
||||
ArgumentMapper("not_a_dict") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.mapper_construction_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from argument mapper construction")
|
||||
def step_check_mapper_construction_error(context: Context) -> None:
|
||||
assert context.mapper_construction_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.mapper_construction_error, TypeError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# TransformExecutor steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a transform executor with code that checks returncode equals zero")
|
||||
def step_transform_returncode(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
_RETURNCODE_TRANSFORM, "test/transform"
|
||||
)
|
||||
|
||||
|
||||
@when("I execute the transform with tool output {output_json}")
|
||||
def step_execute_transform(context: Context, output_json: str) -> None:
|
||||
output = json.loads(output_json)
|
||||
context.transform_result = context.transform_executor.execute(output)
|
||||
|
||||
|
||||
@then("the transform result should have passed true")
|
||||
def step_check_transform_passed_true(context: Context) -> None:
|
||||
assert context.transform_result["passed"] is True
|
||||
|
||||
|
||||
@then("the transform result should have passed false")
|
||||
def step_check_transform_passed_false(context: Context) -> None:
|
||||
assert context.transform_result["passed"] is False
|
||||
|
||||
|
||||
@then('the transform result should have message "{expected_msg}"')
|
||||
def step_check_transform_message(context: Context, expected_msg: str) -> None:
|
||||
assert context.transform_result["message"] == expected_msg, (
|
||||
f"Expected '{expected_msg}', got '{context.transform_result['message']}'"
|
||||
)
|
||||
|
||||
|
||||
@given("a transform executor with code that does not define a transform function")
|
||||
def step_transform_no_func(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor("x = 1 + 2\n", "test/no-func")
|
||||
|
||||
|
||||
@when("I try to execute the transform with any output")
|
||||
def step_try_execute_transform(context: Context) -> None:
|
||||
context.transform_error = None
|
||||
try:
|
||||
context.transform_executor.execute({"data": "test"})
|
||||
except (TransformExecutionError, ValueError) as exc:
|
||||
context.transform_error = exc
|
||||
|
||||
|
||||
@then('a TransformExecutionError should be raised with message containing "{fragment}"')
|
||||
def step_check_transform_error(context: Context, fragment: str) -> None:
|
||||
assert context.transform_error is not None, "Expected TransformExecutionError"
|
||||
assert isinstance(context.transform_error, TransformExecutionError)
|
||||
assert fragment in str(context.transform_error), (
|
||||
f"Expected '{fragment}' in '{context.transform_error}'"
|
||||
)
|
||||
|
||||
|
||||
@given("a transform executor with code that returns a non-dict value")
|
||||
def step_transform_non_dict_return(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
"def transform(x):\n return 'not a dict'\n",
|
||||
"test/non-dict",
|
||||
)
|
||||
|
||||
|
||||
@given("a transform executor with code that returns a dict without passed key")
|
||||
def step_transform_no_passed(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
'def transform(x):\n return {"message": "no passed"}\n',
|
||||
"test/no-passed",
|
||||
)
|
||||
|
||||
|
||||
@when("I try to create a transform executor with empty code")
|
||||
def step_create_transform_empty(context: Context) -> None:
|
||||
context.transform_construction_error = None
|
||||
try:
|
||||
TransformExecutor(" ", "test/empty")
|
||||
except ValueError as exc:
|
||||
context.transform_construction_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised from transform construction")
|
||||
def step_check_transform_construction_error(context: Context) -> None:
|
||||
assert context.transform_construction_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.transform_construction_error, ValueError)
|
||||
|
||||
|
||||
@given("a transform executor with code that attempts to import os")
|
||||
def step_transform_sandbox_import(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
"def transform(x):\n import os\n return {'passed': True}\n",
|
||||
"test/sandbox",
|
||||
)
|
||||
|
||||
|
||||
@then("a TransformExecutionError should be raised from sandbox restriction")
|
||||
def step_check_sandbox_error(context: Context) -> None:
|
||||
assert context.transform_error is not None, (
|
||||
"Expected TransformExecutionError from sandbox"
|
||||
)
|
||||
assert isinstance(context.transform_error, TransformExecutionError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# WrappedToolExecutor steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a tool registry with a tool "{tool_name}" that returns {output_json}')
|
||||
def step_tool_registry_with_tool(
|
||||
context: Context,
|
||||
tool_name: str,
|
||||
output_json: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_tools"):
|
||||
context.test_tools = {}
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
if not hasattr(context, "tool_call_log"):
|
||||
context.tool_call_log = {}
|
||||
|
||||
output = json.loads(output_json)
|
||||
context.test_tools[tool_name] = (_make_tool(tool_name), output)
|
||||
|
||||
|
||||
@given('a validation "{val_name}" that wraps "{wraps_name}" with a simple transform')
|
||||
def step_validation_simple_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _SIMPLE_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given(
|
||||
'a validation "{val_name}" that wraps "{wraps_name}" '
|
||||
"with argument mapping {mapping_json}"
|
||||
)
|
||||
def step_validation_with_mapping(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
mapping_json: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
mapping = json.loads(mapping_json)
|
||||
v = _make_validation(
|
||||
val_name,
|
||||
wraps_name,
|
||||
_SIMPLE_TRANSFORM,
|
||||
argument_mapping=mapping,
|
||||
)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given(
|
||||
'a validation "{val_name}" that wraps "{wraps_name}" with a passthrough transform'
|
||||
)
|
||||
def step_validation_passthrough_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _PASSTHROUGH_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given('a validation "{val_name}" that wraps "{wraps_name}" with a status transform')
|
||||
def step_validation_status_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _STATUS_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given('a validation "{val_name}" that wraps "{wraps_name}" with a data transform')
|
||||
def step_validation_data_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _DATA_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given("a wrapped tool executor using the test registry")
|
||||
def step_create_executor(context: Context) -> None:
|
||||
tools = getattr(context, "test_tools", {})
|
||||
validations = getattr(context, "test_validations", {})
|
||||
context.tool_call_log = {}
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
if name in validations:
|
||||
return validations[name]
|
||||
if name in tools:
|
||||
return tools[name][0]
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
context.tool_call_log[name] = args
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Tool {name} not found in test registry")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@given("a wrapped tool executor with empty registry")
|
||||
def step_create_empty_executor(context: Context) -> None:
|
||||
context.tool_call_log = {}
|
||||
|
||||
def lookup(name: str) -> None:
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
raise RuntimeError(f"Tool {name} not found")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@given("a wrapped tool executor using the test registry with cycle")
|
||||
def step_create_executor_with_cycle(context: Context) -> None:
|
||||
validations = getattr(context, "test_validations", {})
|
||||
|
||||
def lookup(name: str) -> Validation | None:
|
||||
return validations.get(name)
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
raise RuntimeError("Should not reach leaf executor in cycle")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@when("I execute the wrapping validation with inputs {inputs_json}")
|
||||
def step_execute_wrapping(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
# Find the first validation with wraps
|
||||
val = None
|
||||
for v in context.test_validations.values():
|
||||
if v.wraps is not None:
|
||||
val = v
|
||||
break
|
||||
assert val is not None, "No wrapping validation found"
|
||||
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
|
||||
context.last_executed_validation = val
|
||||
|
||||
|
||||
@when("I execute the outer wrapping validation with inputs {inputs_json}")
|
||||
def step_execute_outer_wrapping(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
val = context.test_validations.get("local/outer-wrapper")
|
||||
assert val is not None, "local/outer-wrapper not found"
|
||||
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
|
||||
|
||||
|
||||
@when("I try to execute the wrapping validation with inputs {inputs_json}")
|
||||
def step_try_execute_wrapping(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
context.wrapped_error = None
|
||||
# Find first validation with wraps
|
||||
val = None
|
||||
for v in getattr(context, "test_validations", {}).values():
|
||||
if v.wraps is not None:
|
||||
val = v
|
||||
break
|
||||
if val is None:
|
||||
return
|
||||
try:
|
||||
context.wrapped_executor.execute(val, inputs)
|
||||
except (WrappedToolNotFoundError, WrappingCycleError) as exc:
|
||||
context.wrapped_error = exc
|
||||
|
||||
|
||||
@when('I try to execute the wrapping validation with cycle from "{val_name}"')
|
||||
def step_try_execute_cycle(context: Context, val_name: str) -> None:
|
||||
context.wrapped_error = None
|
||||
val = context.test_validations[val_name]
|
||||
try:
|
||||
context.wrapped_executor.execute(val, {})
|
||||
except WrappingCycleError as exc:
|
||||
context.wrapped_error = exc
|
||||
|
||||
|
||||
@then("the wrapped execution should succeed with passed true")
|
||||
def step_check_wrapped_passed(context: Context) -> None:
|
||||
assert context.wrapped_result["passed"] is True, (
|
||||
f"Expected passed=True, got {context.wrapped_result}"
|
||||
)
|
||||
|
||||
|
||||
@then('the wrapped tool "{tool_name}" should have been called')
|
||||
def step_check_tool_called(context: Context, tool_name: str) -> None:
|
||||
assert tool_name in context.tool_call_log, (
|
||||
f"Tool '{tool_name}' was not called. "
|
||||
f"Called: {list(context.tool_call_log.keys())}"
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
'the wrapped tool "{tool_name}" should have received argument '
|
||||
'"{arg_name}" with value "{arg_value}"'
|
||||
)
|
||||
def step_check_tool_arg_str(
|
||||
context: Context,
|
||||
tool_name: str,
|
||||
arg_name: str,
|
||||
arg_value: str,
|
||||
) -> None:
|
||||
call_args = context.tool_call_log[tool_name]
|
||||
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
|
||||
assert call_args[arg_name] == arg_value, (
|
||||
f"Expected '{arg_value}', got '{call_args[arg_name]}'"
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
'the wrapped tool "{tool_name}" should have received argument '
|
||||
'"{arg_name}" with value true'
|
||||
)
|
||||
def step_check_tool_arg_true(
|
||||
context: Context,
|
||||
tool_name: str,
|
||||
arg_name: str,
|
||||
) -> None:
|
||||
call_args = context.tool_call_log[tool_name]
|
||||
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
|
||||
assert call_args[arg_name] is True, f"Expected True, got {call_args[arg_name]}"
|
||||
|
||||
|
||||
@then('a WrappedToolNotFoundError should be raised for "{wraps_name}"')
|
||||
def step_check_not_found_error(context: Context, wraps_name: str) -> None:
|
||||
assert context.wrapped_error is not None, "Expected WrappedToolNotFoundError"
|
||||
assert isinstance(context.wrapped_error, WrappedToolNotFoundError)
|
||||
assert wraps_name in str(context.wrapped_error)
|
||||
|
||||
|
||||
@then("a WrappingCycleError should be raised")
|
||||
def step_check_cycle_error(context: Context) -> None:
|
||||
assert context.wrapped_error is not None, "Expected WrappingCycleError"
|
||||
assert isinstance(context.wrapped_error, WrappingCycleError)
|
||||
|
||||
|
||||
@then("the wrapped tool should have received the validation inputs")
|
||||
def step_check_inputs_passed(context: Context) -> None:
|
||||
assert len(context.tool_call_log) > 0, "No tool was called"
|
||||
|
||||
|
||||
@when("I try to execute with a non-Validation object")
|
||||
def step_try_execute_non_validation(context: Context) -> None:
|
||||
context.executor_type_error = None
|
||||
try:
|
||||
context.wrapped_executor.execute("not_a_validation", {}) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.executor_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from the executor")
|
||||
def step_check_executor_type_error(context: Context) -> None:
|
||||
assert context.executor_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.executor_type_error, TypeError)
|
||||
|
||||
|
||||
@given('a validation "{val_name}" without wraps set')
|
||||
def step_validation_no_wraps(context: Context, val_name: str) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_plain_validation(val_name)
|
||||
context.test_validations[val_name] = v
|
||||
context.no_wrap_validation = v
|
||||
|
||||
|
||||
@when("I try to execute the non-wrapping validation")
|
||||
def step_try_execute_no_wraps(context: Context) -> None:
|
||||
context.no_wrap_error = None
|
||||
try:
|
||||
context.wrapped_executor.execute(context.no_wrap_validation, {})
|
||||
except ValueError as exc:
|
||||
context.no_wrap_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised indicating wraps is not set")
|
||||
def step_check_no_wrap_error(context: Context) -> None:
|
||||
assert context.no_wrap_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.no_wrap_error, ValueError)
|
||||
assert "wraps" in str(context.no_wrap_error).lower()
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: WrappingDepthExceededError
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a deep wrapping chain of {count:d} validations")
|
||||
def step_deep_chain(context: Context, count: int) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
if not hasattr(context, "test_tools"):
|
||||
context.test_tools = {}
|
||||
# Create a chain: v0 wraps v1, v1 wraps v2, ..., vN wraps leaf-tool
|
||||
for i in range(count):
|
||||
wraps_name = f"local/deep-v{i + 1}" if i < count - 1 else "local/deep-leaf"
|
||||
v = _make_validation(
|
||||
f"local/deep-v{i}",
|
||||
wraps_name,
|
||||
_PASSTHROUGH_TRANSFORM,
|
||||
)
|
||||
context.test_validations[v.name] = v
|
||||
# Create the leaf tool
|
||||
context.test_tools["local/deep-leaf"] = (
|
||||
_make_tool("local/deep-leaf"),
|
||||
{"status": "ok"},
|
||||
)
|
||||
|
||||
|
||||
@given("a wrapped tool executor using the deep chain registry")
|
||||
def step_create_deep_chain_executor(context: Context) -> None:
|
||||
tools = getattr(context, "test_tools", {})
|
||||
validations = getattr(context, "test_validations", {})
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
if name in validations:
|
||||
return validations[name]
|
||||
if name in tools:
|
||||
return tools[name][0]
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Tool {name} not found")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@when("I try to execute the deep chain wrapping validation")
|
||||
def step_try_execute_deep_chain(context: Context) -> None:
|
||||
context.wrapped_error = None
|
||||
val = context.test_validations["local/deep-v0"]
|
||||
try:
|
||||
context.wrapped_executor.execute(val, {})
|
||||
except WrappingDepthExceededError as exc:
|
||||
context.wrapped_error = exc
|
||||
|
||||
|
||||
@then("a WrappingDepthExceededError should be raised with depth {depth:d}")
|
||||
def step_check_depth_exceeded(context: Context, depth: int) -> None:
|
||||
assert context.wrapped_error is not None, "Expected WrappingDepthExceededError"
|
||||
assert isinstance(context.wrapped_error, WrappingDepthExceededError)
|
||||
assert context.wrapped_error.depth == depth, (
|
||||
f"Expected depth {depth}, got {context.wrapped_error.depth}"
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: ArgumentMapper.mapping property
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the mapper mapping property should return {expected_json}")
|
||||
def step_check_mapper_property(context: Context, expected_json: str) -> None:
|
||||
expected = json.loads(expected_json)
|
||||
assert context.mapper.mapping == expected, (
|
||||
f"Expected {expected}, got {context.mapper.mapping}"
|
||||
)
|
||||
|
||||
|
||||
@then("the mapper mapping property should be None")
|
||||
def step_check_mapper_property_none(context: Context) -> None:
|
||||
assert context.mapper.mapping is None, (
|
||||
f"Expected None, got {context.mapper.mapping}"
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: TransformExecutor type checks
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I try to create a transform executor with non-string code")
|
||||
def step_create_transform_non_string_code(context: Context) -> None:
|
||||
context.transform_type_error = None
|
||||
try:
|
||||
TransformExecutor(12345, "test/non-string-code") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.transform_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from transform code type check")
|
||||
def step_check_transform_code_type_error(context: Context) -> None:
|
||||
assert context.transform_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.transform_type_error, TypeError)
|
||||
|
||||
|
||||
@when("I try to create a transform executor with non-string tool name")
|
||||
def step_create_transform_non_string_name(context: Context) -> None:
|
||||
context.transform_name_type_error = None
|
||||
try:
|
||||
TransformExecutor("def transform(x): return {'passed': True}", 42) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.transform_name_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from transform tool name check")
|
||||
def step_check_transform_name_type_error(context: Context) -> None:
|
||||
assert context.transform_name_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.transform_name_type_error, TypeError)
|
||||
|
||||
|
||||
@given("a transform executor with code that raises an error during exec")
|
||||
def step_transform_exec_error(context: Context) -> None:
|
||||
# This code has a top-level expression that raises during exec
|
||||
context.transform_executor = TransformExecutor(
|
||||
"raise RuntimeError('bad code')\ndef transform(x):\n return {'passed': True}\n",
|
||||
"test/exec-error",
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: WrappedToolExecutor init checks
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with None tool_lookup")
|
||||
def step_create_executor_none_lookup(context: Context) -> None:
|
||||
context.executor_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor(None, lambda n, a: {}) # type: ignore[arg-type]
|
||||
except ValueError as exc:
|
||||
context.executor_init_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised from executor construction for tool_lookup")
|
||||
def step_check_executor_none_lookup_error(context: Context) -> None:
|
||||
assert context.executor_init_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.executor_init_error, ValueError)
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with None tool_executor")
|
||||
def step_create_executor_none_executor(context: Context) -> None:
|
||||
context.executor_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor(lambda n: None, None) # type: ignore[arg-type]
|
||||
except ValueError as exc:
|
||||
context.executor_init_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised from executor construction for tool_executor")
|
||||
def step_check_executor_none_executor_error(context: Context) -> None:
|
||||
assert context.executor_init_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.executor_init_error, ValueError)
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with non-callable tool_lookup")
|
||||
def step_create_executor_non_callable_lookup(context: Context) -> None:
|
||||
context.executor_type_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor("not_callable", lambda n, a: {}) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.executor_type_init_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from executor construction for tool_lookup")
|
||||
def step_check_executor_non_callable_lookup_error(context: Context) -> None:
|
||||
assert context.executor_type_init_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.executor_type_init_error, TypeError)
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with non-callable tool_executor")
|
||||
def step_create_executor_non_callable_executor(context: Context) -> None:
|
||||
context.executor_type_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor(lambda n: None, "not_callable") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.executor_type_init_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from executor construction for tool_executor")
|
||||
def step_check_executor_non_callable_executor_error(context: Context) -> None:
|
||||
assert context.executor_type_init_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.executor_type_init_error, TypeError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: execute with non-dict inputs
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I try to execute wrapping validation with non-dict inputs")
|
||||
def step_try_execute_non_dict_inputs(context: Context) -> None:
|
||||
context.inputs_type_error = None
|
||||
val = None
|
||||
for v in context.test_validations.values():
|
||||
if v.wraps is not None:
|
||||
val = v
|
||||
break
|
||||
assert val is not None
|
||||
try:
|
||||
context.wrapped_executor.execute(val, "not_a_dict") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.inputs_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised for non-dict inputs")
|
||||
def step_check_non_dict_inputs_error(context: Context) -> None:
|
||||
assert context.inputs_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.inputs_type_error, TypeError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: innermost wrapper with no wraps target
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ToolRunner coverage steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a ToolRunner with a mock registry")
|
||||
def step_tool_runner_mock_registry(context: Context) -> None:
|
||||
from cleveragents.tool.registry import ToolRegistry
|
||||
from cleveragents.tool.runner import ToolRunner
|
||||
from cleveragents.tool.runtime import ToolSpec
|
||||
|
||||
registry = ToolRegistry()
|
||||
registry.register(
|
||||
ToolSpec(
|
||||
name="local/test-tool",
|
||||
description="test",
|
||||
handler=lambda inputs: {"result": "ok"},
|
||||
)
|
||||
)
|
||||
context.tool_runner = ToolRunner(registry)
|
||||
|
||||
|
||||
@when("I call resolve_execution_environment on the runner")
|
||||
def step_call_resolve_env(context: Context) -> None:
|
||||
context.resolved_env = context.tool_runner.resolve_execution_environment()
|
||||
|
||||
|
||||
@then("the resolved environment should be local")
|
||||
def step_check_env_local(context: Context) -> None:
|
||||
from cleveragents.domain.models.core.plan import ExecutionEnvironment
|
||||
|
||||
assert context.resolved_env == ExecutionEnvironment.HOST
|
||||
|
||||
|
||||
@given("a ToolRunner with a value-error-raising env resolver")
|
||||
def step_runner_value_error_resolver(context: Context) -> None:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from cleveragents.tool.registry import ToolRegistry
|
||||
from cleveragents.tool.runner import ToolRunner
|
||||
from cleveragents.tool.runtime import ToolSpec
|
||||
|
||||
registry = ToolRegistry()
|
||||
registry.register(
|
||||
ToolSpec(
|
||||
name="local/test-tool",
|
||||
description="test",
|
||||
handler=lambda inputs: {"result": "ok"},
|
||||
)
|
||||
)
|
||||
runner = ToolRunner(registry)
|
||||
# Mock the resolver to raise ValueError
|
||||
mock_resolver = MagicMock()
|
||||
mock_resolver.resolve_and_validate.side_effect = ValueError("bad env config")
|
||||
runner._env_resolver = mock_resolver
|
||||
context.tool_runner = runner
|
||||
|
||||
|
||||
@when("I execute a tool through the runner with env error")
|
||||
def step_execute_runner_env_error(context: Context) -> None:
|
||||
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
|
||||
|
||||
|
||||
@then("the tool result should have success false")
|
||||
def step_check_runner_result_fail(context: Context) -> None:
|
||||
assert context.runner_result.success is False
|
||||
|
||||
|
||||
@then('the tool result error should contain "{fragment}"')
|
||||
def step_check_runner_result_error(context: Context, fragment: str) -> None:
|
||||
assert fragment in (context.runner_result.error or ""), (
|
||||
f"Expected '{fragment}' in '{context.runner_result.error}'"
|
||||
)
|
||||
|
||||
|
||||
@given("a ToolRunner with a container-returning env resolver")
|
||||
def step_runner_container_resolver(context: Context) -> None:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from cleveragents.domain.models.core.plan import ExecutionEnvironment
|
||||
from cleveragents.tool.registry import ToolRegistry
|
||||
from cleveragents.tool.runner import ToolRunner
|
||||
from cleveragents.tool.runtime import ToolSpec
|
||||
|
||||
registry = ToolRegistry()
|
||||
registry.register(
|
||||
ToolSpec(
|
||||
name="local/test-tool",
|
||||
description="test",
|
||||
handler=lambda inputs: {"result": "ok"},
|
||||
)
|
||||
)
|
||||
runner = ToolRunner(registry)
|
||||
# Mock the resolver to return CONTAINER
|
||||
mock_resolver = MagicMock()
|
||||
mock_resolver.resolve_and_validate.return_value = ExecutionEnvironment.CONTAINER
|
||||
runner._env_resolver = mock_resolver
|
||||
context.tool_runner = runner
|
||||
|
||||
|
||||
@when("I execute a tool through the runner with container env")
|
||||
def step_execute_runner_container(context: Context) -> None:
|
||||
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
|
||||
|
||||
|
||||
@when("I directly call _execute_chain with a no-wraps leaf")
|
||||
def step_direct_execute_chain_no_wraps(context: Context) -> None:
|
||||
context.chain_error = None
|
||||
# Create a plain validation with wraps=None
|
||||
leaf_val = _make_plain_validation("local/no-wraps-leaf")
|
||||
try:
|
||||
# Call the private method directly to exercise lines 437-438
|
||||
context.wrapped_executor._execute_chain([leaf_val], {})
|
||||
except ValueError as exc:
|
||||
context.chain_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised for missing wraps target")
|
||||
def step_check_missing_wraps_target(context: Context) -> None:
|
||||
assert context.chain_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.chain_error, ValueError)
|
||||
assert "wraps" in str(context.chain_error).lower()
|
||||
@@ -0,0 +1,220 @@
|
||||
Feature: Tool wrapping runtime
|
||||
As the validation execution engine
|
||||
I need to delegate execution to wrapped tools via wraps + transform
|
||||
So that validations can reuse existing tool implementations
|
||||
|
||||
# ── ArgumentMapper ────────────────────────────────────────────────
|
||||
|
||||
Scenario: ArgumentMapper with None mapping passes arguments through
|
||||
Given an argument mapper with no mapping
|
||||
When I apply the mapper to inputs {"path": "/src", "verbose": true}
|
||||
Then the mapped arguments should be {"path": "/src", "verbose": true}
|
||||
|
||||
Scenario: ArgumentMapper with mapping translates argument names
|
||||
Given an argument mapper with mapping {"test_directory": "source_dir", "coverage_enabled": true}
|
||||
When I apply the mapper to inputs {"source_dir": "/tests"}
|
||||
Then the mapped arguments should be {"test_directory": "/tests", "coverage_enabled": true}
|
||||
|
||||
Scenario: ArgumentMapper with literal values injects fixed values
|
||||
Given an argument mapper with mapping {"mode": "strict", "count": 42}
|
||||
When I apply the mapper to inputs {"extra": "ignored"}
|
||||
Then the mapped arguments should be {"mode": "strict", "count": 42}
|
||||
|
||||
Scenario: ArgumentMapper rejects non-dict inputs
|
||||
Given an argument mapper with no mapping
|
||||
When I try to apply the mapper to non-dict input "not_a_dict"
|
||||
Then a TypeError should be raised from the argument mapper
|
||||
|
||||
Scenario: ArgumentMapper rejects non-dict mapping at construction
|
||||
When I try to create an argument mapper with a non-dict mapping
|
||||
Then a TypeError should be raised from argument mapper construction
|
||||
|
||||
# ── TransformExecutor ─────────────────────────────────────────────
|
||||
|
||||
Scenario: TransformExecutor runs a valid transform function
|
||||
Given a transform executor with code that checks returncode equals zero
|
||||
When I execute the transform with tool output {"returncode": 0, "tests_run": 10}
|
||||
Then the transform result should have passed true
|
||||
And the transform result should have message "All tests passed"
|
||||
|
||||
Scenario: TransformExecutor handles failing transform output
|
||||
Given a transform executor with code that checks returncode equals zero
|
||||
When I execute the transform with tool output {"returncode": 1, "tests_run": 10}
|
||||
Then the transform result should have passed false
|
||||
|
||||
Scenario: TransformExecutor raises on missing transform function
|
||||
Given a transform executor with code that does not define a transform function
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "callable"
|
||||
|
||||
Scenario: TransformExecutor raises on non-dict return value
|
||||
Given a transform executor with code that returns a non-dict value
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "dict"
|
||||
|
||||
Scenario: TransformExecutor raises on missing passed key in result
|
||||
Given a transform executor with code that returns a dict without passed key
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "passed"
|
||||
|
||||
Scenario: TransformExecutor raises on empty transform code
|
||||
When I try to create a transform executor with empty code
|
||||
Then a ValueError should be raised from transform construction
|
||||
|
||||
Scenario: TransformExecutor sandboxes dangerous operations
|
||||
Given a transform executor with code that attempts to import os
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised from sandbox restriction
|
||||
|
||||
# ── WrappedToolExecutor ───────────────────────────────────────────
|
||||
|
||||
Scenario: Simple wrapping delegates to the wrapped tool
|
||||
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
|
||||
And a validation "local/tests-pass" that wraps "local/run-tests" with a simple transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {"path": "/src"}
|
||||
Then the wrapped execution should succeed with passed true
|
||||
And the wrapped tool "local/run-tests" should have been called
|
||||
|
||||
Scenario: Argument mapping translates arguments to the wrapped tool
|
||||
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
|
||||
And a validation "local/mapped-check" that wraps "local/run-tests" with argument mapping {"test_directory": "source_dir", "coverage_enabled": true}
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {"source_dir": "/tests"}
|
||||
Then the wrapped tool "local/run-tests" should have received argument "test_directory" with value "/tests"
|
||||
And the wrapped tool "local/run-tests" should have received argument "coverage_enabled" with value true
|
||||
|
||||
Scenario: Chained wrapping delegates through the chain
|
||||
Given a tool registry with a tool "local/base-tool" that returns {"status": "ok"}
|
||||
And a validation "local/mid-wrapper" that wraps "local/base-tool" with a passthrough transform
|
||||
And a validation "local/outer-wrapper" that wraps "local/mid-wrapper" with a status transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the outer wrapping validation with inputs {}
|
||||
Then the wrapped execution should succeed with passed true
|
||||
And the wrapped tool "local/base-tool" should have been called
|
||||
|
||||
Scenario: Missing wrapped tool raises WrappedToolNotFoundError
|
||||
Given a validation "local/broken-wrap" that wraps "local/nonexistent" with a simple transform
|
||||
And a wrapped tool executor with empty registry
|
||||
When I try to execute the wrapping validation with inputs {}
|
||||
Then a WrappedToolNotFoundError should be raised for "local/nonexistent"
|
||||
|
||||
Scenario: Circular wrapping chain raises WrappingCycleError
|
||||
Given a validation "local/wrap-a" that wraps "local/wrap-b" with a simple transform
|
||||
And a validation "local/wrap-b" that wraps "local/wrap-a" with a simple transform
|
||||
And a wrapped tool executor using the test registry with cycle
|
||||
When I try to execute the wrapping validation with cycle from "local/wrap-a"
|
||||
Then a WrappingCycleError should be raised
|
||||
|
||||
Scenario: Execution context is inherited from wrapping validation
|
||||
Given a tool registry with a tool "local/context-tool" that returns {"data": "value"}
|
||||
And a validation "local/context-wrap" that wraps "local/context-tool" with a data transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {"key": "value"}
|
||||
Then the wrapped tool should have received the validation inputs
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-Validation argument
|
||||
Given a wrapped tool executor with empty registry
|
||||
When I try to execute with a non-Validation object
|
||||
Then a TypeError should be raised from the executor
|
||||
|
||||
Scenario: WrappedToolExecutor rejects validation without wraps
|
||||
Given a validation "local/no-wrap" without wraps set
|
||||
And a wrapped tool executor with empty registry
|
||||
When I try to execute the non-wrapping validation
|
||||
Then a ValueError should be raised indicating wraps is not set
|
||||
|
||||
# ── Additional coverage: WrappingDepthExceededError ──────────────
|
||||
|
||||
Scenario: WrappingDepthExceededError is raised when chain is too deep
|
||||
Given a deep wrapping chain of 11 validations
|
||||
And a wrapped tool executor using the deep chain registry
|
||||
When I try to execute the deep chain wrapping validation
|
||||
Then a WrappingDepthExceededError should be raised with depth 10
|
||||
|
||||
# ── Additional coverage: ArgumentMapper.mapping property ─────────
|
||||
|
||||
Scenario: ArgumentMapper exposes the raw mapping via property
|
||||
Given an argument mapper with mapping {"x": "y"}
|
||||
Then the mapper mapping property should return {"x": "y"}
|
||||
|
||||
Scenario: ArgumentMapper mapping property returns None for identity mapper
|
||||
Given an argument mapper with no mapping
|
||||
Then the mapper mapping property should be None
|
||||
|
||||
# ── Additional coverage: TransformExecutor type checks ───────────
|
||||
|
||||
Scenario: TransformExecutor rejects non-string transform code
|
||||
When I try to create a transform executor with non-string code
|
||||
Then a TypeError should be raised from transform code type check
|
||||
|
||||
Scenario: TransformExecutor rejects non-string tool name
|
||||
When I try to create a transform executor with non-string tool name
|
||||
Then a TypeError should be raised from transform tool name check
|
||||
|
||||
Scenario: TransformExecutor raises on code that throws during exec
|
||||
Given a transform executor with code that raises an error during exec
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "compile"
|
||||
|
||||
# ── Additional coverage: WrappedToolExecutor init checks ─────────
|
||||
|
||||
Scenario: WrappedToolExecutor rejects None tool_lookup
|
||||
When I try to create a wrapped tool executor with None tool_lookup
|
||||
Then a ValueError should be raised from executor construction for tool_lookup
|
||||
|
||||
Scenario: WrappedToolExecutor rejects None tool_executor
|
||||
When I try to create a wrapped tool executor with None tool_executor
|
||||
Then a ValueError should be raised from executor construction for tool_executor
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-callable tool_lookup
|
||||
When I try to create a wrapped tool executor with non-callable tool_lookup
|
||||
Then a TypeError should be raised from executor construction for tool_lookup
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-callable tool_executor
|
||||
When I try to create a wrapped tool executor with non-callable tool_executor
|
||||
Then a TypeError should be raised from executor construction for tool_executor
|
||||
|
||||
# ── Additional coverage: execute with non-dict inputs ────────────
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-dict inputs
|
||||
Given a tool registry with a tool "local/some-tool" that returns {"ok": true}
|
||||
And a validation "local/input-wrap" that wraps "local/some-tool" with a simple transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I try to execute wrapping validation with non-dict inputs
|
||||
Then a TypeError should be raised for non-dict inputs
|
||||
|
||||
# ── Additional coverage: chain with None wraps at end ────────────
|
||||
|
||||
Scenario: Chain resolution handles validation with wraps set to None in chain
|
||||
Given a tool registry with a tool "local/leaf" that returns {"value": 1}
|
||||
And a validation "local/wrapper-no-inner" that wraps "local/leaf" with a passthrough transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {}
|
||||
Then the wrapped execution should succeed with passed true
|
||||
|
||||
# ── Additional coverage: _execute_chain with None wraps target ────
|
||||
|
||||
Scenario: _execute_chain raises ValueError when leaf wraps is None
|
||||
Given a wrapped tool executor with empty registry
|
||||
When I directly call _execute_chain with a no-wraps leaf
|
||||
Then a ValueError should be raised for missing wraps target
|
||||
|
||||
# ── ToolRunner coverage: execution environment and error paths ───
|
||||
|
||||
Scenario: ToolRunner resolve_execution_environment delegates to resolver
|
||||
Given a ToolRunner with a mock registry
|
||||
When I call resolve_execution_environment on the runner
|
||||
Then the resolved environment should be local
|
||||
|
||||
Scenario: ToolRunner execute returns error when env resolver raises ValueError
|
||||
Given a ToolRunner with a value-error-raising env resolver
|
||||
When I execute a tool through the runner with env error
|
||||
Then the tool result should have success false
|
||||
And the tool result error should contain "Execution environment error"
|
||||
|
||||
Scenario: ToolRunner execute returns error for container environment
|
||||
Given a ToolRunner with a container-returning env resolver
|
||||
When I execute a tool through the runner with container env
|
||||
Then the tool result should have success false
|
||||
And the tool result error should contain "Container execution is not yet implemented"
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Robot helper: verify WrappedToolExecutor delegates to wrapped tool."""
|
||||
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import WrappedToolExecutor
|
||||
|
||||
tool = Tool(
|
||||
name="local/run",
|
||||
description="run",
|
||||
source=ToolSource.BUILTIN,
|
||||
capability=ToolCapability(read_only=True),
|
||||
)
|
||||
val = Validation(
|
||||
name="local/check",
|
||||
description="check",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps="local/run",
|
||||
transform='def transform(x):\n return {"passed": True, "data": x}\n',
|
||||
)
|
||||
lookup = lambda n: tool if n == "local/run" else None # noqa: E731
|
||||
called: list[str] = []
|
||||
|
||||
|
||||
def executor_fn(n: str, a: dict) -> dict: # type: ignore[type-arg]
|
||||
"""Record tool call and return output."""
|
||||
called.append(n)
|
||||
return {"result": "ok"}
|
||||
|
||||
|
||||
ex = WrappedToolExecutor(lookup, executor_fn)
|
||||
r = ex.execute(val, {"a": 1})
|
||||
assert r["passed"] is True
|
||||
assert "local/run" in called
|
||||
print("OK")
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Robot helper: verify ArgumentMapper translates arguments."""
|
||||
|
||||
from cleveragents.tool.wrapping import ArgumentMapper
|
||||
|
||||
m = ArgumentMapper({"dest": "src", "flag": True})
|
||||
r = m.apply({"src": "/path"})
|
||||
assert r == {"dest": "/path", "flag": True}, f"Got {r}"
|
||||
print("OK")
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Robot helper: verify ArgumentMapper with None mapping."""
|
||||
|
||||
from cleveragents.tool.wrapping import ArgumentMapper
|
||||
|
||||
m = ArgumentMapper(None)
|
||||
r = m.apply({"x": 1, "y": "two"})
|
||||
assert r == {"x": 1, "y": "two"}, f"Got {r}"
|
||||
print("OK")
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Robot helper: verify WrappedToolExecutor raises on missing tool."""
|
||||
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import WrappedToolExecutor, WrappedToolNotFoundError
|
||||
|
||||
val = Validation(
|
||||
name="local/check",
|
||||
description="check",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps="local/missing",
|
||||
transform='def transform(x):\n return {"passed": True}\n',
|
||||
)
|
||||
ok = False
|
||||
try:
|
||||
WrappedToolExecutor(
|
||||
lambda n: None,
|
||||
lambda n, a: {},
|
||||
).execute(val, {})
|
||||
except WrappedToolNotFoundError:
|
||||
ok = True
|
||||
print(f"error_raised={ok}")
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Robot helper: verify TransformExecutor runs a transform."""
|
||||
|
||||
from cleveragents.tool.wrapping import TransformExecutor
|
||||
|
||||
code = 'def transform(x):\n return {"passed": x.get("ok"), "message": "done"}\n'
|
||||
t = TransformExecutor(code, "test/t")
|
||||
r = t.execute({"ok": True})
|
||||
assert r["passed"] is True
|
||||
print("OK")
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Robot helper: verify TransformExecutor blocks import in sandbox."""
|
||||
|
||||
from cleveragents.tool.wrapping import TransformExecutionError, TransformExecutor
|
||||
|
||||
code = 'def transform(x):\n import os\n return {"passed": True}\n'
|
||||
t = TransformExecutor(code, "test/s")
|
||||
ok = False
|
||||
try:
|
||||
t.execute({})
|
||||
except TransformExecutionError:
|
||||
ok = True
|
||||
print(f"blocked={ok}")
|
||||
@@ -0,0 +1,58 @@
|
||||
*** Settings ***
|
||||
Documentation Tool wrapping runtime smoke tests
|
||||
Library Process
|
||||
Library OperatingSystem
|
||||
|
||||
*** Variables ***
|
||||
${PYTHON} python
|
||||
|
||||
*** Test Cases ***
|
||||
Wrapping Package Is Importable
|
||||
[Documentation] Verify tool wrapping module can be imported
|
||||
${result}= Run Process ${PYTHON} -c
|
||||
... from cleveragents.tool.wrapping import ArgumentMapper, TransformExecutor, WrappedToolExecutor, WrappedToolNotFoundError, WrappingCycleError, WrappingDepthExceededError, TransformExecutionError; print('OK')
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
Wrapping Exports In Tool Init
|
||||
[Documentation] Verify wrapping types are exported from tool package
|
||||
${result}= Run Process ${PYTHON} -c
|
||||
... from cleveragents.tool import ArgumentMapper, TransformExecutor, WrappedToolExecutor; print('OK')
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
ArgumentMapper Identity Mapping
|
||||
[Documentation] Verify ArgumentMapper with None mapping passes args through
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_identity.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
ArgumentMapper With Mapping
|
||||
[Documentation] Verify ArgumentMapper translates arguments
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_configured.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
TransformExecutor Runs Transform
|
||||
[Documentation] Verify TransformExecutor executes transform code
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_transform_run.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
TransformExecutor Sandbox Blocks Import
|
||||
[Documentation] Verify TransformExecutor blocks import in sandbox
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_transform_sandbox.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} blocked=True
|
||||
|
||||
WrappedToolExecutor Simple Delegation
|
||||
[Documentation] Verify WrappedToolExecutor delegates to wrapped tool
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_delegation.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
WrappedToolExecutor Missing Tool Error
|
||||
[Documentation] Verify WrappedToolExecutor raises on missing wrapped tool
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_missing_tool.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} error_raised=True
|
||||
@@ -69,8 +69,18 @@ from cleveragents.tool.schema_validator import (
|
||||
validate_tool_input,
|
||||
validate_tool_output,
|
||||
)
|
||||
from cleveragents.tool.wrapping import (
|
||||
ArgumentMapper,
|
||||
TransformExecutionError,
|
||||
TransformExecutor,
|
||||
WrappedToolExecutor,
|
||||
WrappedToolNotFoundError,
|
||||
WrappingCycleError,
|
||||
WrappingDepthExceededError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ArgumentMapper",
|
||||
"BoundResource",
|
||||
"CancellationToken",
|
||||
"Change",
|
||||
@@ -116,6 +126,12 @@ __all__ = [
|
||||
"ToolSandboxRequiredError",
|
||||
"ToolSchemaValidationError",
|
||||
"ToolSpec",
|
||||
"TransformExecutionError",
|
||||
"TransformExecutor",
|
||||
"WrappedToolExecutor",
|
||||
"WrappedToolNotFoundError",
|
||||
"WrappingCycleError",
|
||||
"WrappingDepthExceededError",
|
||||
"classify_tool_error",
|
||||
"detect_provider_format",
|
||||
"generate_tool_call_id",
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
"""Tool wrapping runtime for CleverAgents v3.
|
||||
|
||||
Implements the ``wraps`` + ``transform`` + ``argument_mapping`` delegation
|
||||
mechanism described in the specification (§ Core Concepts > Validation >
|
||||
Tool Wrapping).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
WrappedToolExecutor
|
||||
|-- ArgumentMapper (translate wrapper args -> wrapped tool args)
|
||||
|-- TransformExecutor (sandboxed transform function on tool output)
|
||||
+-- ToolRuntime (resolve + invoke wrapped tools)
|
||||
```
|
||||
|
||||
## Delegation Chain
|
||||
|
||||
A validation wrapping another validation forms a delegation chain:
|
||||
|
||||
```
|
||||
validation-A -> validation-B -> tool-C
|
||||
(wraps B) (wraps C) (leaf)
|
||||
```
|
||||
|
||||
``WrappedToolExecutor.execute()`` recursively resolves the chain,
|
||||
detecting cycles and enforcing a maximum depth.
|
||||
|
||||
## Sandbox
|
||||
|
||||
``TransformExecutor`` compiles and executes the user-supplied
|
||||
``transform`` function with a restricted set of built-in names.
|
||||
No filesystem, network, or import access is available inside the
|
||||
transform.
|
||||
|
||||
Based on ``docs/specification.md`` § Tool Wrapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from cleveragents.domain.models.core.tool import Validation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Maximum delegation chain depth to prevent infinite recursion.
|
||||
MAX_WRAPPING_DEPTH = 10
|
||||
|
||||
# Safe built-ins available inside the ``transform`` sandbox.
|
||||
_SAFE_BUILTINS: dict[str, Any] = {
|
||||
"True": True,
|
||||
"False": False,
|
||||
"None": None,
|
||||
"abs": abs,
|
||||
"all": all,
|
||||
"any": any,
|
||||
"bool": bool,
|
||||
"dict": dict,
|
||||
"enumerate": enumerate,
|
||||
"float": float,
|
||||
"frozenset": frozenset,
|
||||
"int": int,
|
||||
"isinstance": isinstance,
|
||||
"len": len,
|
||||
"list": list,
|
||||
"map": map,
|
||||
"max": max,
|
||||
"min": min,
|
||||
"range": range,
|
||||
"repr": repr,
|
||||
"reversed": reversed,
|
||||
"round": round,
|
||||
"set": set,
|
||||
"sorted": sorted,
|
||||
"str": str,
|
||||
"sum": sum,
|
||||
"tuple": tuple,
|
||||
"type": type,
|
||||
"zip": zip,
|
||||
}
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Errors
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class WrappedToolNotFoundError(Exception):
|
||||
"""Raised when the tool referenced by ``wraps`` is not registered."""
|
||||
|
||||
def __init__(self, tool_name: str, wraps_name: str) -> None:
|
||||
self.tool_name = tool_name
|
||||
self.wraps_name = wraps_name
|
||||
super().__init__(
|
||||
f"Validation '{tool_name}' wraps '{wraps_name}' but "
|
||||
f"'{wraps_name}' is not registered in the runtime"
|
||||
)
|
||||
|
||||
|
||||
class WrappingCycleError(Exception):
|
||||
"""Raised when a circular wrapping chain is detected."""
|
||||
|
||||
def __init__(self, chain: list[str]) -> None:
|
||||
self.chain = list(chain)
|
||||
cycle_str = " -> ".join(chain)
|
||||
super().__init__(f"Circular wrapping chain detected: {cycle_str}")
|
||||
|
||||
|
||||
class WrappingDepthExceededError(Exception):
|
||||
"""Raised when the delegation chain exceeds ``MAX_WRAPPING_DEPTH``."""
|
||||
|
||||
def __init__(self, depth: int) -> None:
|
||||
self.depth = depth
|
||||
super().__init__(
|
||||
f"Wrapping delegation chain depth ({depth}) exceeds "
|
||||
f"the maximum ({MAX_WRAPPING_DEPTH})"
|
||||
)
|
||||
|
||||
|
||||
class TransformExecutionError(Exception):
|
||||
"""Raised when the ``transform`` function fails during execution."""
|
||||
|
||||
def __init__(self, tool_name: str, detail: str) -> None:
|
||||
self.tool_name = tool_name
|
||||
self.detail = detail
|
||||
super().__init__(f"Transform function for '{tool_name}' failed: {detail}")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ArgumentMapper
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class ArgumentMapper:
|
||||
"""Applies ``argument_mapping`` to translate arguments.
|
||||
|
||||
When a validation wraps a tool, the validation may accept different
|
||||
input arguments. The ``argument_mapping`` dictionary specifies how
|
||||
the validation's inputs map to the wrapped tool's expected inputs.
|
||||
|
||||
Keys are the wrapped tool's parameter names. Values are either:
|
||||
- A string referencing a validation input parameter (forwarded).
|
||||
- A literal value (string, number, boolean) always sent to the
|
||||
wrapped tool.
|
||||
|
||||
When ``argument_mapping`` is ``None``, input arguments are passed
|
||||
through to the wrapped tool unchanged (identity mapping).
|
||||
"""
|
||||
|
||||
def __init__(self, mapping: dict[str, Any] | None) -> None:
|
||||
if mapping is not None and not isinstance(mapping, dict):
|
||||
raise TypeError(
|
||||
f"argument_mapping must be a dict or None, got {type(mapping).__name__}"
|
||||
)
|
||||
self._mapping = mapping
|
||||
|
||||
@property
|
||||
def mapping(self) -> dict[str, Any] | None:
|
||||
"""Return the raw mapping configuration."""
|
||||
return self._mapping
|
||||
|
||||
def apply(self, validation_inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Translate *validation_inputs* into wrapped-tool arguments.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
validation_inputs:
|
||||
The arguments the validation was invoked with.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]:
|
||||
Arguments to forward to the wrapped tool.
|
||||
"""
|
||||
if not isinstance(validation_inputs, dict):
|
||||
raise TypeError(
|
||||
f"validation_inputs must be a dict, "
|
||||
f"got {type(validation_inputs).__name__}"
|
||||
)
|
||||
if self._mapping is None:
|
||||
return dict(validation_inputs)
|
||||
|
||||
mapped: dict[str, Any] = {}
|
||||
for wrapped_param, source in self._mapping.items():
|
||||
if isinstance(source, str) and source in validation_inputs:
|
||||
mapped[wrapped_param] = validation_inputs[source]
|
||||
else:
|
||||
# Literal value (or string not present as a key -- use
|
||||
# as literal).
|
||||
mapped[wrapped_param] = source
|
||||
return mapped
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# TransformExecutor
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class TransformExecutor:
|
||||
"""Executes the ``transform`` function in a sandboxed context.
|
||||
|
||||
The transform code must define a function called ``transform`` that
|
||||
accepts one positional argument (the wrapped tool's output) and
|
||||
returns a dict with at minimum ``{"passed": bool}``.
|
||||
|
||||
The sandbox disallows imports, file I/O, and network access by
|
||||
restricting available built-ins.
|
||||
"""
|
||||
|
||||
def __init__(self, transform_code: str, tool_name: str) -> None:
|
||||
if not isinstance(transform_code, str):
|
||||
raise TypeError(
|
||||
f"transform_code must be a str, got {type(transform_code).__name__}"
|
||||
)
|
||||
if not transform_code.strip():
|
||||
raise ValueError("transform_code must not be empty")
|
||||
if not isinstance(tool_name, str):
|
||||
raise TypeError(f"tool_name must be a str, got {type(tool_name).__name__}")
|
||||
self._code = transform_code
|
||||
self._tool_name = tool_name
|
||||
self._compiled = compile(self._code, f"<transform:{tool_name}>", "exec")
|
||||
|
||||
def execute(self, tool_output: Any) -> dict[str, Any]:
|
||||
"""Run the transform on *tool_output*.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]:
|
||||
The validation-format result (must contain ``passed``).
|
||||
|
||||
Raises
|
||||
------
|
||||
TransformExecutionError:
|
||||
If the transform function is missing, raises, or returns
|
||||
an invalid result.
|
||||
"""
|
||||
sandbox: dict[str, Any] = {"__builtins__": dict(_SAFE_BUILTINS)}
|
||||
|
||||
try:
|
||||
exec(self._compiled, sandbox) # Controlled sandbox execution
|
||||
except Exception as exc:
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
f"Failed to compile/execute transform code: {exc}",
|
||||
) from exc
|
||||
|
||||
transform_fn = sandbox.get("transform")
|
||||
if transform_fn is None or not callable(transform_fn):
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
"Transform code must define a callable 'transform' function",
|
||||
)
|
||||
|
||||
try:
|
||||
result = transform_fn(tool_output)
|
||||
except Exception as exc:
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
f"Transform function raised: {type(exc).__name__}: {exc}",
|
||||
) from exc
|
||||
|
||||
if not isinstance(result, dict):
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
f"Transform must return a dict, got {type(result).__name__}",
|
||||
)
|
||||
|
||||
if "passed" not in result:
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
"Transform result must contain a 'passed' key",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# WrappedToolExecutor
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class WrappedToolExecutor:
|
||||
"""Resolves ``wraps`` references and delegates execution.
|
||||
|
||||
Orchestrates the full wrapping flow:
|
||||
|
||||
1. Resolve the wrapped tool from the tool lookup.
|
||||
2. If the wrapped tool is itself a wrapping validation, recurse.
|
||||
3. Apply ``argument_mapping`` to translate arguments.
|
||||
4. Execute the leaf tool.
|
||||
5. Walk back up the chain applying ``transform`` at each level.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tool_lookup:
|
||||
A callable that accepts a tool name (str) and returns
|
||||
the ``Validation`` or ``Tool`` domain model, or ``None``
|
||||
if not found.
|
||||
tool_executor:
|
||||
A callable ``(tool_name, mapped_args) -> dict`` that
|
||||
actually runs the leaf tool and returns its raw output.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tool_lookup: Any,
|
||||
tool_executor: Any,
|
||||
) -> None:
|
||||
if tool_lookup is None:
|
||||
raise ValueError("tool_lookup must not be None")
|
||||
if tool_executor is None:
|
||||
raise ValueError("tool_executor must not be None")
|
||||
if not callable(tool_lookup):
|
||||
raise TypeError(
|
||||
f"tool_lookup must be callable, got {type(tool_lookup).__name__}"
|
||||
)
|
||||
if not callable(tool_executor):
|
||||
raise TypeError(
|
||||
f"tool_executor must be callable, got {type(tool_executor).__name__}"
|
||||
)
|
||||
self._tool_lookup = tool_lookup
|
||||
self._tool_executor = tool_executor
|
||||
|
||||
def execute(
|
||||
self,
|
||||
validation: Validation,
|
||||
inputs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Execute a wrapping validation.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
validation:
|
||||
The wrapping ``Validation`` (must have ``wraps`` set).
|
||||
inputs:
|
||||
The arguments the validation was invoked with.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]:
|
||||
The validation-format result with ``passed``, optionally
|
||||
``message`` and ``data``.
|
||||
|
||||
Raises
|
||||
------
|
||||
WrappedToolNotFoundError:
|
||||
If any tool in the chain is not registered.
|
||||
WrappingCycleError:
|
||||
If the chain contains a cycle.
|
||||
WrappingDepthExceededError:
|
||||
If the chain exceeds ``MAX_WRAPPING_DEPTH``.
|
||||
TransformExecutionError:
|
||||
If any transform function in the chain fails.
|
||||
"""
|
||||
if not isinstance(validation, Validation):
|
||||
raise TypeError(
|
||||
f"validation must be a Validation, got {type(validation).__name__}"
|
||||
)
|
||||
if not isinstance(inputs, dict):
|
||||
raise TypeError(f"inputs must be a dict, got {type(inputs).__name__}")
|
||||
if validation.wraps is None:
|
||||
raise ValueError(
|
||||
f"Validation '{validation.name}' does not have "
|
||||
f"'wraps' set -- use direct execution instead"
|
||||
)
|
||||
|
||||
chain = self._resolve_chain(validation)
|
||||
return self._execute_chain(chain, inputs)
|
||||
|
||||
# -- Chain resolution --------------------------------------------------
|
||||
|
||||
def _resolve_chain(
|
||||
self,
|
||||
validation: Validation,
|
||||
) -> list[Validation]:
|
||||
"""Walk the wraps chain and return the ordered list.
|
||||
|
||||
The first element is the outermost wrapper, the last is the
|
||||
innermost wrapper (whose ``wraps`` target is the leaf tool).
|
||||
"""
|
||||
chain: list[Validation] = []
|
||||
visited: set[str] = set()
|
||||
current: Validation = validation
|
||||
|
||||
while True:
|
||||
if len(chain) >= MAX_WRAPPING_DEPTH:
|
||||
raise WrappingDepthExceededError(len(chain))
|
||||
|
||||
if current.name in visited:
|
||||
cycle = [v.name for v in chain] + [current.name]
|
||||
raise WrappingCycleError(cycle)
|
||||
|
||||
visited.add(current.name)
|
||||
chain.append(current)
|
||||
|
||||
wraps_name = current.wraps
|
||||
if wraps_name is None:
|
||||
break
|
||||
|
||||
wrapped = self._tool_lookup(wraps_name)
|
||||
if wrapped is None:
|
||||
raise WrappedToolNotFoundError(current.name, wraps_name)
|
||||
|
||||
if isinstance(wrapped, Validation) and wrapped.wraps is not None:
|
||||
current = wrapped
|
||||
else:
|
||||
# Leaf tool found -- stop the chain
|
||||
break
|
||||
|
||||
return chain
|
||||
|
||||
# -- Chain execution ---------------------------------------------------
|
||||
|
||||
def _execute_chain(
|
||||
self,
|
||||
chain: list[Validation],
|
||||
inputs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Execute the delegation chain.
|
||||
|
||||
1. Walk from outermost to innermost, applying argument mappings.
|
||||
2. Execute the leaf tool.
|
||||
3. Walk back from innermost to outermost, applying transforms.
|
||||
"""
|
||||
# Build the fully-mapped arguments by applying each wrapper's
|
||||
# argument_mapping in sequence.
|
||||
mapped_args = dict(inputs)
|
||||
for wrapper in chain:
|
||||
mapper = ArgumentMapper(wrapper.argument_mapping)
|
||||
mapped_args = mapper.apply(mapped_args)
|
||||
|
||||
# Execute the leaf tool (the target of the innermost wraps)
|
||||
leaf_wrapper = chain[-1]
|
||||
leaf_tool_name = leaf_wrapper.wraps
|
||||
if leaf_tool_name is None:
|
||||
raise ValueError(
|
||||
f"Innermost wrapper '{leaf_wrapper.name}' has no 'wraps' target"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Executing wrapped tool",
|
||||
extra={
|
||||
"chain": [v.name for v in chain],
|
||||
"leaf_tool": leaf_tool_name,
|
||||
},
|
||||
)
|
||||
|
||||
raw_output = self._tool_executor(leaf_tool_name, mapped_args)
|
||||
|
||||
# Apply transforms in reverse (innermost first, outermost last)
|
||||
result: dict[str, Any] = (
|
||||
raw_output if isinstance(raw_output, dict) else {"raw": raw_output}
|
||||
)
|
||||
for wrapper in reversed(chain):
|
||||
if wrapper.transform is not None:
|
||||
transform_exec = TransformExecutor(
|
||||
wrapper.transform,
|
||||
wrapper.name,
|
||||
)
|
||||
logger.debug(
|
||||
"Applying transform",
|
||||
extra={
|
||||
"wrapper": wrapper.name,
|
||||
"input_type": type(result).__name__,
|
||||
},
|
||||
)
|
||||
result = transform_exec.execute(result)
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user