feat(validation): implement tool wrapping runtime (wraps + transform delegation)
CI / lint (pull_request) Successful in 14s
CI / typecheck (pull_request) Successful in 34s
CI / security (pull_request) Successful in 29s
CI / quality (pull_request) Successful in 15s
CI / benchmark-publish (pull_request) Has been skipped
CI / build (pull_request) Successful in 14s
CI / unit_tests (pull_request) Successful in 2m4s
CI / integration_tests (pull_request) Successful in 2m50s
CI / coverage (pull_request) Successful in 4m15s
CI / docker (pull_request) Successful in 42s
CI / lint (push) Successful in 12s
CI / typecheck (push) Successful in 31s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 31s
CI / build (push) Successful in 16s
CI / unit_tests (push) Successful in 3m23s
CI / benchmark-regression (push) Has been skipped
CI / docker (push) Successful in 42s
CI / integration_tests (push) Successful in 4m11s
CI / coverage (push) Successful in 4m8s
CI / benchmark-publish (push) Successful in 16m5s
CI / benchmark-regression (pull_request) Successful in 29m59s
CI / lint (pull_request) Successful in 14s
CI / typecheck (pull_request) Successful in 34s
CI / security (pull_request) Successful in 29s
CI / quality (pull_request) Successful in 15s
CI / benchmark-publish (pull_request) Has been skipped
CI / build (pull_request) Successful in 14s
CI / unit_tests (pull_request) Successful in 2m4s
CI / integration_tests (pull_request) Successful in 2m50s
CI / coverage (pull_request) Successful in 4m15s
CI / docker (pull_request) Successful in 42s
CI / lint (push) Successful in 12s
CI / typecheck (push) Successful in 31s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 31s
CI / build (push) Successful in 16s
CI / unit_tests (push) Successful in 3m23s
CI / benchmark-regression (push) Has been skipped
CI / docker (push) Successful in 42s
CI / integration_tests (push) Successful in 4m11s
CI / coverage (push) Successful in 4m8s
CI / benchmark-publish (push) Successful in 16m5s
CI / benchmark-regression (pull_request) Successful in 29m59s
Implement the runtime execution engine for validation tool wrapping, as specified in docs/specification.md § Tool Wrapping. WrappedToolExecutor resolves wraps references and delegates execution to wrapped tools, supporting composable wrapping chains with cycle detection and depth limiting (max 10 levels). ArgumentMapper translates arguments between wrapper and wrapped tool schemas using the argument_mapping configuration. Supports both forwarded parameter names and literal fixed values. TransformExecutor runs user-supplied transform functions in a sandboxed Python environment with restricted builtins (no imports, no filesystem, no network access). Validates that transforms return proper validation-format dicts with a passed boolean. Wired into the tool package public API via tool/__init__.py exports. All new error types (WrappedToolNotFoundError, WrappingCycleError, WrappingDepthExceededError, TransformExecutionError) provide clear diagnostic messages. Tests: 20 Behave scenarios covering argument mapping, transform execution, simple/chained delegation, error handling, and sandbox restrictions. 8 Robot Framework integration smoke tests. ASV benchmarks for delegation overhead measurement. ISSUES CLOSED: #543
This commit was merged in pull request #555.
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
"""ASV benchmarks for tool wrapping delegation overhead.
|
||||
|
||||
Measures the performance of:
|
||||
- ArgumentMapper.apply() with identity and configured mappings
|
||||
- TransformExecutor.execute() with simple transform functions
|
||||
- WrappedToolExecutor.execute() for single and chained delegation
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.domain.models.core.tool import ( # noqa: E402
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import ( # noqa: E402
|
||||
ArgumentMapper,
|
||||
TransformExecutor,
|
||||
WrappedToolExecutor,
|
||||
)
|
||||
|
||||
_SIMPLE_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("returncode") == 0
|
||||
return {"passed": passed, "message": "done", "data": tool_output}
|
||||
"""
|
||||
|
||||
_PASSTHROUGH_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
return {"passed": True, "message": "ok", "data": tool_output}
|
||||
"""
|
||||
|
||||
|
||||
def _make_tool(name: str) -> Tool:
|
||||
return Tool(
|
||||
name=name,
|
||||
description=f"Benchmark tool {name}",
|
||||
source=ToolSource.BUILTIN,
|
||||
capability=ToolCapability(read_only=True),
|
||||
)
|
||||
|
||||
|
||||
def _make_validation(
|
||||
name: str,
|
||||
wraps: str,
|
||||
transform_code: str,
|
||||
argument_mapping: dict[str, Any] | None = None,
|
||||
) -> Validation:
|
||||
return Validation(
|
||||
name=name,
|
||||
description=f"Benchmark validation wrapping {wraps}",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps=wraps,
|
||||
transform=transform_code,
|
||||
argument_mapping=argument_mapping,
|
||||
)
|
||||
|
||||
|
||||
class ArgumentMapperSuite:
|
||||
"""Benchmarks for ArgumentMapper."""
|
||||
|
||||
def time_identity_mapping(self) -> None:
|
||||
"""Measure identity (passthrough) mapping overhead."""
|
||||
mapper = ArgumentMapper(None)
|
||||
for _ in range(1000):
|
||||
mapper.apply({"path": "/src", "verbose": True, "count": 42})
|
||||
|
||||
def time_configured_mapping(self) -> None:
|
||||
"""Measure mapping with configured argument translation."""
|
||||
mapper = ArgumentMapper(
|
||||
{
|
||||
"test_directory": "source_dir",
|
||||
"coverage_enabled": True,
|
||||
"verbose": False,
|
||||
}
|
||||
)
|
||||
for _ in range(1000):
|
||||
mapper.apply({"source_dir": "/tests", "extra": "ignored"})
|
||||
|
||||
|
||||
class TransformExecutorSuite:
|
||||
"""Benchmarks for TransformExecutor."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
|
||||
self._output = {"returncode": 0, "tests_run": 100}
|
||||
|
||||
def time_execute_transform(self) -> None:
|
||||
"""Measure transform function execution."""
|
||||
for _ in range(1000):
|
||||
self._executor.execute(self._output)
|
||||
|
||||
def time_create_and_execute(self) -> None:
|
||||
"""Measure creation + execution (cold path)."""
|
||||
for _ in range(100):
|
||||
executor = TransformExecutor(_SIMPLE_TRANSFORM, "bench/tool")
|
||||
executor.execute(self._output)
|
||||
|
||||
|
||||
class WrappedToolExecutorSuite:
|
||||
"""Benchmarks for WrappedToolExecutor delegation."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._tool = _make_tool("local/base-tool")
|
||||
self._output: dict[str, Any] = {"returncode": 0, "data": "ok"}
|
||||
self._validation = _make_validation(
|
||||
"local/bench-wrap",
|
||||
"local/base-tool",
|
||||
_SIMPLE_TRANSFORM,
|
||||
)
|
||||
|
||||
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
|
||||
"local/base-tool": (self._tool, self._output),
|
||||
}
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
if name in tools:
|
||||
return tools[name][0]
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Not found: {name}")
|
||||
|
||||
self._executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
def time_single_delegation(self) -> None:
|
||||
"""Measure single-level wrapping delegation."""
|
||||
for _ in range(1000):
|
||||
self._executor.execute(self._validation, {"path": "/src"})
|
||||
|
||||
def time_chained_delegation(self) -> None:
|
||||
"""Measure two-level wrapping delegation chain."""
|
||||
inner_val = _make_validation(
|
||||
"local/inner-wrap",
|
||||
"local/base-tool",
|
||||
_PASSTHROUGH_TRANSFORM,
|
||||
)
|
||||
outer_val = _make_validation(
|
||||
"local/outer-wrap",
|
||||
"local/inner-wrap",
|
||||
_SIMPLE_TRANSFORM,
|
||||
)
|
||||
|
||||
tools: dict[str, tuple[Tool, dict[str, Any]]] = {
|
||||
"local/base-tool": (self._tool, self._output),
|
||||
}
|
||||
|
||||
all_items: dict[str, Tool | Validation] = {
|
||||
"local/inner-wrap": inner_val,
|
||||
"local/base-tool": self._tool,
|
||||
}
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
return all_items.get(name)
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> dict[str, Any]:
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Not found: {name}")
|
||||
|
||||
chained_executor = WrappedToolExecutor(lookup, executor)
|
||||
for _ in range(500):
|
||||
chained_executor.execute(outer_val, {"path": "/src"})
|
||||
@@ -0,0 +1,913 @@
|
||||
"""Step definitions for tool wrapping runtime feature tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import (
|
||||
ArgumentMapper,
|
||||
TransformExecutionError,
|
||||
TransformExecutor,
|
||||
WrappedToolExecutor,
|
||||
WrappedToolNotFoundError,
|
||||
WrappingCycleError,
|
||||
WrappingDepthExceededError,
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Helpers
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
_SIMPLE_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("returncode") == 0
|
||||
msg = "All tests passed" if passed else "Tests failed"
|
||||
return {"passed": passed, "message": msg, "data": tool_output}
|
||||
"""
|
||||
|
||||
_PASSTHROUGH_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
return {"passed": True, "message": "ok", "data": tool_output}
|
||||
"""
|
||||
|
||||
_STATUS_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("passed", False)
|
||||
return {"passed": passed, "message": "outer", "data": tool_output}
|
||||
"""
|
||||
|
||||
_DATA_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
return {"passed": True, "message": "data ok", "data": tool_output}
|
||||
"""
|
||||
|
||||
_RETURNCODE_TRANSFORM = """\
|
||||
def transform(tool_output):
|
||||
passed = tool_output.get("returncode") == 0
|
||||
msg = "All tests passed" if passed else "Tests failed"
|
||||
return {"passed": passed, "message": msg, "data": tool_output}
|
||||
"""
|
||||
|
||||
|
||||
def _make_validation(
|
||||
name: str,
|
||||
wraps: str,
|
||||
transform_code: str,
|
||||
argument_mapping: dict[str, Any] | None = None,
|
||||
) -> Validation:
|
||||
"""Create a Validation with wraps set."""
|
||||
return Validation(
|
||||
name=name,
|
||||
description=f"Test validation wrapping {wraps}",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps=wraps,
|
||||
transform=transform_code,
|
||||
argument_mapping=argument_mapping,
|
||||
)
|
||||
|
||||
|
||||
def _make_plain_validation(name: str) -> Validation:
|
||||
"""Create a Validation without wraps."""
|
||||
return Validation(
|
||||
name=name,
|
||||
description="Test validation no wraps",
|
||||
source=ToolSource.CUSTOM,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
code="pass",
|
||||
)
|
||||
|
||||
|
||||
def _make_tool(name: str) -> Tool:
|
||||
"""Create a plain Tool."""
|
||||
return Tool(
|
||||
name=name,
|
||||
description=f"Test tool {name}",
|
||||
source=ToolSource.BUILTIN,
|
||||
capability=ToolCapability(read_only=True),
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ArgumentMapper steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("an argument mapper with no mapping")
|
||||
def step_mapper_no_mapping(context: Context) -> None:
|
||||
context.mapper = ArgumentMapper(None)
|
||||
|
||||
|
||||
@given("an argument mapper with mapping {mapping_json}")
|
||||
def step_mapper_with_mapping(context: Context, mapping_json: str) -> None:
|
||||
mapping = json.loads(mapping_json)
|
||||
context.mapper = ArgumentMapper(mapping)
|
||||
|
||||
|
||||
@when("I apply the mapper to inputs {inputs_json}")
|
||||
def step_apply_mapper(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
context.mapped_result = context.mapper.apply(inputs)
|
||||
|
||||
|
||||
@then("the mapped arguments should be {expected_json}")
|
||||
def step_check_mapped(context: Context, expected_json: str) -> None:
|
||||
expected = json.loads(expected_json)
|
||||
assert context.mapped_result == expected, (
|
||||
f"Expected {expected}, got {context.mapped_result}"
|
||||
)
|
||||
|
||||
|
||||
@when('I try to apply the mapper to non-dict input "{value}"')
|
||||
def step_apply_mapper_non_dict(context: Context, value: str) -> None:
|
||||
context.mapper_error = None
|
||||
try:
|
||||
context.mapper.apply(value) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.mapper_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from the argument mapper")
|
||||
def step_check_mapper_type_error(context: Context) -> None:
|
||||
assert context.mapper_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.mapper_error, TypeError)
|
||||
|
||||
|
||||
@when("I try to create an argument mapper with a non-dict mapping")
|
||||
def step_create_mapper_non_dict(context: Context) -> None:
|
||||
context.mapper_construction_error = None
|
||||
try:
|
||||
ArgumentMapper("not_a_dict") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.mapper_construction_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from argument mapper construction")
|
||||
def step_check_mapper_construction_error(context: Context) -> None:
|
||||
assert context.mapper_construction_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.mapper_construction_error, TypeError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# TransformExecutor steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a transform executor with code that checks returncode equals zero")
|
||||
def step_transform_returncode(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
_RETURNCODE_TRANSFORM, "test/transform"
|
||||
)
|
||||
|
||||
|
||||
@when("I execute the transform with tool output {output_json}")
|
||||
def step_execute_transform(context: Context, output_json: str) -> None:
|
||||
output = json.loads(output_json)
|
||||
context.transform_result = context.transform_executor.execute(output)
|
||||
|
||||
|
||||
@then("the transform result should have passed true")
|
||||
def step_check_transform_passed_true(context: Context) -> None:
|
||||
assert context.transform_result["passed"] is True
|
||||
|
||||
|
||||
@then("the transform result should have passed false")
|
||||
def step_check_transform_passed_false(context: Context) -> None:
|
||||
assert context.transform_result["passed"] is False
|
||||
|
||||
|
||||
@then('the transform result should have message "{expected_msg}"')
|
||||
def step_check_transform_message(context: Context, expected_msg: str) -> None:
|
||||
assert context.transform_result["message"] == expected_msg, (
|
||||
f"Expected '{expected_msg}', got '{context.transform_result['message']}'"
|
||||
)
|
||||
|
||||
|
||||
@given("a transform executor with code that does not define a transform function")
|
||||
def step_transform_no_func(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor("x = 1 + 2\n", "test/no-func")
|
||||
|
||||
|
||||
@when("I try to execute the transform with any output")
|
||||
def step_try_execute_transform(context: Context) -> None:
|
||||
context.transform_error = None
|
||||
try:
|
||||
context.transform_executor.execute({"data": "test"})
|
||||
except (TransformExecutionError, ValueError) as exc:
|
||||
context.transform_error = exc
|
||||
|
||||
|
||||
@then('a TransformExecutionError should be raised with message containing "{fragment}"')
|
||||
def step_check_transform_error(context: Context, fragment: str) -> None:
|
||||
assert context.transform_error is not None, "Expected TransformExecutionError"
|
||||
assert isinstance(context.transform_error, TransformExecutionError)
|
||||
assert fragment in str(context.transform_error), (
|
||||
f"Expected '{fragment}' in '{context.transform_error}'"
|
||||
)
|
||||
|
||||
|
||||
@given("a transform executor with code that returns a non-dict value")
|
||||
def step_transform_non_dict_return(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
"def transform(x):\n return 'not a dict'\n",
|
||||
"test/non-dict",
|
||||
)
|
||||
|
||||
|
||||
@given("a transform executor with code that returns a dict without passed key")
|
||||
def step_transform_no_passed(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
'def transform(x):\n return {"message": "no passed"}\n',
|
||||
"test/no-passed",
|
||||
)
|
||||
|
||||
|
||||
@when("I try to create a transform executor with empty code")
|
||||
def step_create_transform_empty(context: Context) -> None:
|
||||
context.transform_construction_error = None
|
||||
try:
|
||||
TransformExecutor(" ", "test/empty")
|
||||
except ValueError as exc:
|
||||
context.transform_construction_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised from transform construction")
|
||||
def step_check_transform_construction_error(context: Context) -> None:
|
||||
assert context.transform_construction_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.transform_construction_error, ValueError)
|
||||
|
||||
|
||||
@given("a transform executor with code that attempts to import os")
|
||||
def step_transform_sandbox_import(context: Context) -> None:
|
||||
context.transform_executor = TransformExecutor(
|
||||
"def transform(x):\n import os\n return {'passed': True}\n",
|
||||
"test/sandbox",
|
||||
)
|
||||
|
||||
|
||||
@then("a TransformExecutionError should be raised from sandbox restriction")
|
||||
def step_check_sandbox_error(context: Context) -> None:
|
||||
assert context.transform_error is not None, (
|
||||
"Expected TransformExecutionError from sandbox"
|
||||
)
|
||||
assert isinstance(context.transform_error, TransformExecutionError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# WrappedToolExecutor steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a tool registry with a tool "{tool_name}" that returns {output_json}')
|
||||
def step_tool_registry_with_tool(
|
||||
context: Context,
|
||||
tool_name: str,
|
||||
output_json: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_tools"):
|
||||
context.test_tools = {}
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
if not hasattr(context, "tool_call_log"):
|
||||
context.tool_call_log = {}
|
||||
|
||||
output = json.loads(output_json)
|
||||
context.test_tools[tool_name] = (_make_tool(tool_name), output)
|
||||
|
||||
|
||||
@given('a validation "{val_name}" that wraps "{wraps_name}" with a simple transform')
|
||||
def step_validation_simple_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _SIMPLE_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given(
|
||||
'a validation "{val_name}" that wraps "{wraps_name}" '
|
||||
"with argument mapping {mapping_json}"
|
||||
)
|
||||
def step_validation_with_mapping(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
mapping_json: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
mapping = json.loads(mapping_json)
|
||||
v = _make_validation(
|
||||
val_name,
|
||||
wraps_name,
|
||||
_SIMPLE_TRANSFORM,
|
||||
argument_mapping=mapping,
|
||||
)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given(
|
||||
'a validation "{val_name}" that wraps "{wraps_name}" with a passthrough transform'
|
||||
)
|
||||
def step_validation_passthrough_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _PASSTHROUGH_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given('a validation "{val_name}" that wraps "{wraps_name}" with a status transform')
|
||||
def step_validation_status_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _STATUS_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given('a validation "{val_name}" that wraps "{wraps_name}" with a data transform')
|
||||
def step_validation_data_wrap(
|
||||
context: Context,
|
||||
val_name: str,
|
||||
wraps_name: str,
|
||||
) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_validation(val_name, wraps_name, _DATA_TRANSFORM)
|
||||
context.test_validations[val_name] = v
|
||||
|
||||
|
||||
@given("a wrapped tool executor using the test registry")
|
||||
def step_create_executor(context: Context) -> None:
|
||||
tools = getattr(context, "test_tools", {})
|
||||
validations = getattr(context, "test_validations", {})
|
||||
context.tool_call_log = {}
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
if name in validations:
|
||||
return validations[name]
|
||||
if name in tools:
|
||||
return tools[name][0]
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
context.tool_call_log[name] = args
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Tool {name} not found in test registry")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@given("a wrapped tool executor with empty registry")
|
||||
def step_create_empty_executor(context: Context) -> None:
|
||||
context.tool_call_log = {}
|
||||
|
||||
def lookup(name: str) -> None:
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
raise RuntimeError(f"Tool {name} not found")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@given("a wrapped tool executor using the test registry with cycle")
|
||||
def step_create_executor_with_cycle(context: Context) -> None:
|
||||
validations = getattr(context, "test_validations", {})
|
||||
|
||||
def lookup(name: str) -> Validation | None:
|
||||
return validations.get(name)
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
raise RuntimeError("Should not reach leaf executor in cycle")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@when("I execute the wrapping validation with inputs {inputs_json}")
|
||||
def step_execute_wrapping(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
# Find the first validation with wraps
|
||||
val = None
|
||||
for v in context.test_validations.values():
|
||||
if v.wraps is not None:
|
||||
val = v
|
||||
break
|
||||
assert val is not None, "No wrapping validation found"
|
||||
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
|
||||
context.last_executed_validation = val
|
||||
|
||||
|
||||
@when("I execute the outer wrapping validation with inputs {inputs_json}")
|
||||
def step_execute_outer_wrapping(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
val = context.test_validations.get("local/outer-wrapper")
|
||||
assert val is not None, "local/outer-wrapper not found"
|
||||
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
|
||||
|
||||
|
||||
@when("I try to execute the wrapping validation with inputs {inputs_json}")
|
||||
def step_try_execute_wrapping(context: Context, inputs_json: str) -> None:
|
||||
inputs = json.loads(inputs_json)
|
||||
context.wrapped_error = None
|
||||
# Find first validation with wraps
|
||||
val = None
|
||||
for v in getattr(context, "test_validations", {}).values():
|
||||
if v.wraps is not None:
|
||||
val = v
|
||||
break
|
||||
if val is None:
|
||||
return
|
||||
try:
|
||||
context.wrapped_executor.execute(val, inputs)
|
||||
except (WrappedToolNotFoundError, WrappingCycleError) as exc:
|
||||
context.wrapped_error = exc
|
||||
|
||||
|
||||
@when('I try to execute the wrapping validation with cycle from "{val_name}"')
|
||||
def step_try_execute_cycle(context: Context, val_name: str) -> None:
|
||||
context.wrapped_error = None
|
||||
val = context.test_validations[val_name]
|
||||
try:
|
||||
context.wrapped_executor.execute(val, {})
|
||||
except WrappingCycleError as exc:
|
||||
context.wrapped_error = exc
|
||||
|
||||
|
||||
@then("the wrapped execution should succeed with passed true")
|
||||
def step_check_wrapped_passed(context: Context) -> None:
|
||||
assert context.wrapped_result["passed"] is True, (
|
||||
f"Expected passed=True, got {context.wrapped_result}"
|
||||
)
|
||||
|
||||
|
||||
@then('the wrapped tool "{tool_name}" should have been called')
|
||||
def step_check_tool_called(context: Context, tool_name: str) -> None:
|
||||
assert tool_name in context.tool_call_log, (
|
||||
f"Tool '{tool_name}' was not called. "
|
||||
f"Called: {list(context.tool_call_log.keys())}"
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
'the wrapped tool "{tool_name}" should have received argument '
|
||||
'"{arg_name}" with value "{arg_value}"'
|
||||
)
|
||||
def step_check_tool_arg_str(
|
||||
context: Context,
|
||||
tool_name: str,
|
||||
arg_name: str,
|
||||
arg_value: str,
|
||||
) -> None:
|
||||
call_args = context.tool_call_log[tool_name]
|
||||
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
|
||||
assert call_args[arg_name] == arg_value, (
|
||||
f"Expected '{arg_value}', got '{call_args[arg_name]}'"
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
'the wrapped tool "{tool_name}" should have received argument '
|
||||
'"{arg_name}" with value true'
|
||||
)
|
||||
def step_check_tool_arg_true(
|
||||
context: Context,
|
||||
tool_name: str,
|
||||
arg_name: str,
|
||||
) -> None:
|
||||
call_args = context.tool_call_log[tool_name]
|
||||
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
|
||||
assert call_args[arg_name] is True, f"Expected True, got {call_args[arg_name]}"
|
||||
|
||||
|
||||
@then('a WrappedToolNotFoundError should be raised for "{wraps_name}"')
|
||||
def step_check_not_found_error(context: Context, wraps_name: str) -> None:
|
||||
assert context.wrapped_error is not None, "Expected WrappedToolNotFoundError"
|
||||
assert isinstance(context.wrapped_error, WrappedToolNotFoundError)
|
||||
assert wraps_name in str(context.wrapped_error)
|
||||
|
||||
|
||||
@then("a WrappingCycleError should be raised")
|
||||
def step_check_cycle_error(context: Context) -> None:
|
||||
assert context.wrapped_error is not None, "Expected WrappingCycleError"
|
||||
assert isinstance(context.wrapped_error, WrappingCycleError)
|
||||
|
||||
|
||||
@then("the wrapped tool should have received the validation inputs")
|
||||
def step_check_inputs_passed(context: Context) -> None:
|
||||
assert len(context.tool_call_log) > 0, "No tool was called"
|
||||
|
||||
|
||||
@when("I try to execute with a non-Validation object")
|
||||
def step_try_execute_non_validation(context: Context) -> None:
|
||||
context.executor_type_error = None
|
||||
try:
|
||||
context.wrapped_executor.execute("not_a_validation", {}) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.executor_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from the executor")
|
||||
def step_check_executor_type_error(context: Context) -> None:
|
||||
assert context.executor_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.executor_type_error, TypeError)
|
||||
|
||||
|
||||
@given('a validation "{val_name}" without wraps set')
|
||||
def step_validation_no_wraps(context: Context, val_name: str) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
v = _make_plain_validation(val_name)
|
||||
context.test_validations[val_name] = v
|
||||
context.no_wrap_validation = v
|
||||
|
||||
|
||||
@when("I try to execute the non-wrapping validation")
|
||||
def step_try_execute_no_wraps(context: Context) -> None:
|
||||
context.no_wrap_error = None
|
||||
try:
|
||||
context.wrapped_executor.execute(context.no_wrap_validation, {})
|
||||
except ValueError as exc:
|
||||
context.no_wrap_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised indicating wraps is not set")
|
||||
def step_check_no_wrap_error(context: Context) -> None:
|
||||
assert context.no_wrap_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.no_wrap_error, ValueError)
|
||||
assert "wraps" in str(context.no_wrap_error).lower()
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: WrappingDepthExceededError
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a deep wrapping chain of {count:d} validations")
|
||||
def step_deep_chain(context: Context, count: int) -> None:
|
||||
if not hasattr(context, "test_validations"):
|
||||
context.test_validations = {}
|
||||
if not hasattr(context, "test_tools"):
|
||||
context.test_tools = {}
|
||||
# Create a chain: v0 wraps v1, v1 wraps v2, ..., vN wraps leaf-tool
|
||||
for i in range(count):
|
||||
wraps_name = f"local/deep-v{i + 1}" if i < count - 1 else "local/deep-leaf"
|
||||
v = _make_validation(
|
||||
f"local/deep-v{i}",
|
||||
wraps_name,
|
||||
_PASSTHROUGH_TRANSFORM,
|
||||
)
|
||||
context.test_validations[v.name] = v
|
||||
# Create the leaf tool
|
||||
context.test_tools["local/deep-leaf"] = (
|
||||
_make_tool("local/deep-leaf"),
|
||||
{"status": "ok"},
|
||||
)
|
||||
|
||||
|
||||
@given("a wrapped tool executor using the deep chain registry")
|
||||
def step_create_deep_chain_executor(context: Context) -> None:
|
||||
tools = getattr(context, "test_tools", {})
|
||||
validations = getattr(context, "test_validations", {})
|
||||
|
||||
def lookup(name: str) -> Tool | Validation | None:
|
||||
if name in validations:
|
||||
return validations[name]
|
||||
if name in tools:
|
||||
return tools[name][0]
|
||||
return None
|
||||
|
||||
def executor(name: str, args: dict[str, Any]) -> Any:
|
||||
if name in tools:
|
||||
return tools[name][1]
|
||||
raise RuntimeError(f"Tool {name} not found")
|
||||
|
||||
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
||||
|
||||
|
||||
@when("I try to execute the deep chain wrapping validation")
|
||||
def step_try_execute_deep_chain(context: Context) -> None:
|
||||
context.wrapped_error = None
|
||||
val = context.test_validations["local/deep-v0"]
|
||||
try:
|
||||
context.wrapped_executor.execute(val, {})
|
||||
except WrappingDepthExceededError as exc:
|
||||
context.wrapped_error = exc
|
||||
|
||||
|
||||
@then("a WrappingDepthExceededError should be raised with depth {depth:d}")
|
||||
def step_check_depth_exceeded(context: Context, depth: int) -> None:
|
||||
assert context.wrapped_error is not None, "Expected WrappingDepthExceededError"
|
||||
assert isinstance(context.wrapped_error, WrappingDepthExceededError)
|
||||
assert context.wrapped_error.depth == depth, (
|
||||
f"Expected depth {depth}, got {context.wrapped_error.depth}"
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: ArgumentMapper.mapping property
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the mapper mapping property should return {expected_json}")
|
||||
def step_check_mapper_property(context: Context, expected_json: str) -> None:
|
||||
expected = json.loads(expected_json)
|
||||
assert context.mapper.mapping == expected, (
|
||||
f"Expected {expected}, got {context.mapper.mapping}"
|
||||
)
|
||||
|
||||
|
||||
@then("the mapper mapping property should be None")
|
||||
def step_check_mapper_property_none(context: Context) -> None:
|
||||
assert context.mapper.mapping is None, (
|
||||
f"Expected None, got {context.mapper.mapping}"
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: TransformExecutor type checks
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I try to create a transform executor with non-string code")
|
||||
def step_create_transform_non_string_code(context: Context) -> None:
|
||||
context.transform_type_error = None
|
||||
try:
|
||||
TransformExecutor(12345, "test/non-string-code") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.transform_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from transform code type check")
|
||||
def step_check_transform_code_type_error(context: Context) -> None:
|
||||
assert context.transform_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.transform_type_error, TypeError)
|
||||
|
||||
|
||||
@when("I try to create a transform executor with non-string tool name")
|
||||
def step_create_transform_non_string_name(context: Context) -> None:
|
||||
context.transform_name_type_error = None
|
||||
try:
|
||||
TransformExecutor("def transform(x): return {'passed': True}", 42) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.transform_name_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from transform tool name check")
|
||||
def step_check_transform_name_type_error(context: Context) -> None:
|
||||
assert context.transform_name_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.transform_name_type_error, TypeError)
|
||||
|
||||
|
||||
@given("a transform executor with code that raises an error during exec")
|
||||
def step_transform_exec_error(context: Context) -> None:
|
||||
# This code has a top-level expression that raises during exec
|
||||
context.transform_executor = TransformExecutor(
|
||||
"raise RuntimeError('bad code')\ndef transform(x):\n return {'passed': True}\n",
|
||||
"test/exec-error",
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: WrappedToolExecutor init checks
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with None tool_lookup")
|
||||
def step_create_executor_none_lookup(context: Context) -> None:
|
||||
context.executor_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor(None, lambda n, a: {}) # type: ignore[arg-type]
|
||||
except ValueError as exc:
|
||||
context.executor_init_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised from executor construction for tool_lookup")
|
||||
def step_check_executor_none_lookup_error(context: Context) -> None:
|
||||
assert context.executor_init_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.executor_init_error, ValueError)
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with None tool_executor")
|
||||
def step_create_executor_none_executor(context: Context) -> None:
|
||||
context.executor_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor(lambda n: None, None) # type: ignore[arg-type]
|
||||
except ValueError as exc:
|
||||
context.executor_init_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised from executor construction for tool_executor")
|
||||
def step_check_executor_none_executor_error(context: Context) -> None:
|
||||
assert context.executor_init_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.executor_init_error, ValueError)
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with non-callable tool_lookup")
|
||||
def step_create_executor_non_callable_lookup(context: Context) -> None:
|
||||
context.executor_type_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor("not_callable", lambda n, a: {}) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.executor_type_init_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from executor construction for tool_lookup")
|
||||
def step_check_executor_non_callable_lookup_error(context: Context) -> None:
|
||||
assert context.executor_type_init_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.executor_type_init_error, TypeError)
|
||||
|
||||
|
||||
@when("I try to create a wrapped tool executor with non-callable tool_executor")
|
||||
def step_create_executor_non_callable_executor(context: Context) -> None:
|
||||
context.executor_type_init_error = None
|
||||
try:
|
||||
WrappedToolExecutor(lambda n: None, "not_callable") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.executor_type_init_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised from executor construction for tool_executor")
|
||||
def step_check_executor_non_callable_executor_error(context: Context) -> None:
|
||||
assert context.executor_type_init_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.executor_type_init_error, TypeError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: execute with non-dict inputs
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I try to execute wrapping validation with non-dict inputs")
|
||||
def step_try_execute_non_dict_inputs(context: Context) -> None:
|
||||
context.inputs_type_error = None
|
||||
val = None
|
||||
for v in context.test_validations.values():
|
||||
if v.wraps is not None:
|
||||
val = v
|
||||
break
|
||||
assert val is not None
|
||||
try:
|
||||
context.wrapped_executor.execute(val, "not_a_dict") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.inputs_type_error = exc
|
||||
|
||||
|
||||
@then("a TypeError should be raised for non-dict inputs")
|
||||
def step_check_non_dict_inputs_error(context: Context) -> None:
|
||||
assert context.inputs_type_error is not None, "Expected TypeError"
|
||||
assert isinstance(context.inputs_type_error, TypeError)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Additional coverage: innermost wrapper with no wraps target
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ToolRunner coverage steps
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a ToolRunner with a mock registry")
|
||||
def step_tool_runner_mock_registry(context: Context) -> None:
|
||||
from cleveragents.tool.registry import ToolRegistry
|
||||
from cleveragents.tool.runner import ToolRunner
|
||||
from cleveragents.tool.runtime import ToolSpec
|
||||
|
||||
registry = ToolRegistry()
|
||||
registry.register(
|
||||
ToolSpec(
|
||||
name="local/test-tool",
|
||||
description="test",
|
||||
handler=lambda inputs: {"result": "ok"},
|
||||
)
|
||||
)
|
||||
context.tool_runner = ToolRunner(registry)
|
||||
|
||||
|
||||
@when("I call resolve_execution_environment on the runner")
|
||||
def step_call_resolve_env(context: Context) -> None:
|
||||
context.resolved_env = context.tool_runner.resolve_execution_environment()
|
||||
|
||||
|
||||
@then("the resolved environment should be local")
|
||||
def step_check_env_local(context: Context) -> None:
|
||||
from cleveragents.domain.models.core.plan import ExecutionEnvironment
|
||||
|
||||
assert context.resolved_env == ExecutionEnvironment.HOST
|
||||
|
||||
|
||||
@given("a ToolRunner with a value-error-raising env resolver")
|
||||
def step_runner_value_error_resolver(context: Context) -> None:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from cleveragents.tool.registry import ToolRegistry
|
||||
from cleveragents.tool.runner import ToolRunner
|
||||
from cleveragents.tool.runtime import ToolSpec
|
||||
|
||||
registry = ToolRegistry()
|
||||
registry.register(
|
||||
ToolSpec(
|
||||
name="local/test-tool",
|
||||
description="test",
|
||||
handler=lambda inputs: {"result": "ok"},
|
||||
)
|
||||
)
|
||||
runner = ToolRunner(registry)
|
||||
# Mock the resolver to raise ValueError
|
||||
mock_resolver = MagicMock()
|
||||
mock_resolver.resolve_and_validate.side_effect = ValueError("bad env config")
|
||||
runner._env_resolver = mock_resolver
|
||||
context.tool_runner = runner
|
||||
|
||||
|
||||
@when("I execute a tool through the runner with env error")
|
||||
def step_execute_runner_env_error(context: Context) -> None:
|
||||
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
|
||||
|
||||
|
||||
@then("the tool result should have success false")
|
||||
def step_check_runner_result_fail(context: Context) -> None:
|
||||
assert context.runner_result.success is False
|
||||
|
||||
|
||||
@then('the tool result error should contain "{fragment}"')
|
||||
def step_check_runner_result_error(context: Context, fragment: str) -> None:
|
||||
assert fragment in (context.runner_result.error or ""), (
|
||||
f"Expected '{fragment}' in '{context.runner_result.error}'"
|
||||
)
|
||||
|
||||
|
||||
@given("a ToolRunner with a container-returning env resolver")
|
||||
def step_runner_container_resolver(context: Context) -> None:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from cleveragents.domain.models.core.plan import ExecutionEnvironment
|
||||
from cleveragents.tool.registry import ToolRegistry
|
||||
from cleveragents.tool.runner import ToolRunner
|
||||
from cleveragents.tool.runtime import ToolSpec
|
||||
|
||||
registry = ToolRegistry()
|
||||
registry.register(
|
||||
ToolSpec(
|
||||
name="local/test-tool",
|
||||
description="test",
|
||||
handler=lambda inputs: {"result": "ok"},
|
||||
)
|
||||
)
|
||||
runner = ToolRunner(registry)
|
||||
# Mock the resolver to return CONTAINER
|
||||
mock_resolver = MagicMock()
|
||||
mock_resolver.resolve_and_validate.return_value = ExecutionEnvironment.CONTAINER
|
||||
runner._env_resolver = mock_resolver
|
||||
context.tool_runner = runner
|
||||
|
||||
|
||||
@when("I execute a tool through the runner with container env")
|
||||
def step_execute_runner_container(context: Context) -> None:
|
||||
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
|
||||
|
||||
|
||||
@when("I directly call _execute_chain with a no-wraps leaf")
|
||||
def step_direct_execute_chain_no_wraps(context: Context) -> None:
|
||||
context.chain_error = None
|
||||
# Create a plain validation with wraps=None
|
||||
leaf_val = _make_plain_validation("local/no-wraps-leaf")
|
||||
try:
|
||||
# Call the private method directly to exercise lines 437-438
|
||||
context.wrapped_executor._execute_chain([leaf_val], {})
|
||||
except ValueError as exc:
|
||||
context.chain_error = exc
|
||||
|
||||
|
||||
@then("a ValueError should be raised for missing wraps target")
|
||||
def step_check_missing_wraps_target(context: Context) -> None:
|
||||
assert context.chain_error is not None, "Expected ValueError"
|
||||
assert isinstance(context.chain_error, ValueError)
|
||||
assert "wraps" in str(context.chain_error).lower()
|
||||
@@ -0,0 +1,220 @@
|
||||
Feature: Tool wrapping runtime
|
||||
As the validation execution engine
|
||||
I need to delegate execution to wrapped tools via wraps + transform
|
||||
So that validations can reuse existing tool implementations
|
||||
|
||||
# ── ArgumentMapper ────────────────────────────────────────────────
|
||||
|
||||
Scenario: ArgumentMapper with None mapping passes arguments through
|
||||
Given an argument mapper with no mapping
|
||||
When I apply the mapper to inputs {"path": "/src", "verbose": true}
|
||||
Then the mapped arguments should be {"path": "/src", "verbose": true}
|
||||
|
||||
Scenario: ArgumentMapper with mapping translates argument names
|
||||
Given an argument mapper with mapping {"test_directory": "source_dir", "coverage_enabled": true}
|
||||
When I apply the mapper to inputs {"source_dir": "/tests"}
|
||||
Then the mapped arguments should be {"test_directory": "/tests", "coverage_enabled": true}
|
||||
|
||||
Scenario: ArgumentMapper with literal values injects fixed values
|
||||
Given an argument mapper with mapping {"mode": "strict", "count": 42}
|
||||
When I apply the mapper to inputs {"extra": "ignored"}
|
||||
Then the mapped arguments should be {"mode": "strict", "count": 42}
|
||||
|
||||
Scenario: ArgumentMapper rejects non-dict inputs
|
||||
Given an argument mapper with no mapping
|
||||
When I try to apply the mapper to non-dict input "not_a_dict"
|
||||
Then a TypeError should be raised from the argument mapper
|
||||
|
||||
Scenario: ArgumentMapper rejects non-dict mapping at construction
|
||||
When I try to create an argument mapper with a non-dict mapping
|
||||
Then a TypeError should be raised from argument mapper construction
|
||||
|
||||
# ── TransformExecutor ─────────────────────────────────────────────
|
||||
|
||||
Scenario: TransformExecutor runs a valid transform function
|
||||
Given a transform executor with code that checks returncode equals zero
|
||||
When I execute the transform with tool output {"returncode": 0, "tests_run": 10}
|
||||
Then the transform result should have passed true
|
||||
And the transform result should have message "All tests passed"
|
||||
|
||||
Scenario: TransformExecutor handles failing transform output
|
||||
Given a transform executor with code that checks returncode equals zero
|
||||
When I execute the transform with tool output {"returncode": 1, "tests_run": 10}
|
||||
Then the transform result should have passed false
|
||||
|
||||
Scenario: TransformExecutor raises on missing transform function
|
||||
Given a transform executor with code that does not define a transform function
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "callable"
|
||||
|
||||
Scenario: TransformExecutor raises on non-dict return value
|
||||
Given a transform executor with code that returns a non-dict value
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "dict"
|
||||
|
||||
Scenario: TransformExecutor raises on missing passed key in result
|
||||
Given a transform executor with code that returns a dict without passed key
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "passed"
|
||||
|
||||
Scenario: TransformExecutor raises on empty transform code
|
||||
When I try to create a transform executor with empty code
|
||||
Then a ValueError should be raised from transform construction
|
||||
|
||||
Scenario: TransformExecutor sandboxes dangerous operations
|
||||
Given a transform executor with code that attempts to import os
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised from sandbox restriction
|
||||
|
||||
# ── WrappedToolExecutor ───────────────────────────────────────────
|
||||
|
||||
Scenario: Simple wrapping delegates to the wrapped tool
|
||||
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
|
||||
And a validation "local/tests-pass" that wraps "local/run-tests" with a simple transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {"path": "/src"}
|
||||
Then the wrapped execution should succeed with passed true
|
||||
And the wrapped tool "local/run-tests" should have been called
|
||||
|
||||
Scenario: Argument mapping translates arguments to the wrapped tool
|
||||
Given a tool registry with a tool "local/run-tests" that returns {"returncode": 0}
|
||||
And a validation "local/mapped-check" that wraps "local/run-tests" with argument mapping {"test_directory": "source_dir", "coverage_enabled": true}
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {"source_dir": "/tests"}
|
||||
Then the wrapped tool "local/run-tests" should have received argument "test_directory" with value "/tests"
|
||||
And the wrapped tool "local/run-tests" should have received argument "coverage_enabled" with value true
|
||||
|
||||
Scenario: Chained wrapping delegates through the chain
|
||||
Given a tool registry with a tool "local/base-tool" that returns {"status": "ok"}
|
||||
And a validation "local/mid-wrapper" that wraps "local/base-tool" with a passthrough transform
|
||||
And a validation "local/outer-wrapper" that wraps "local/mid-wrapper" with a status transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the outer wrapping validation with inputs {}
|
||||
Then the wrapped execution should succeed with passed true
|
||||
And the wrapped tool "local/base-tool" should have been called
|
||||
|
||||
Scenario: Missing wrapped tool raises WrappedToolNotFoundError
|
||||
Given a validation "local/broken-wrap" that wraps "local/nonexistent" with a simple transform
|
||||
And a wrapped tool executor with empty registry
|
||||
When I try to execute the wrapping validation with inputs {}
|
||||
Then a WrappedToolNotFoundError should be raised for "local/nonexistent"
|
||||
|
||||
Scenario: Circular wrapping chain raises WrappingCycleError
|
||||
Given a validation "local/wrap-a" that wraps "local/wrap-b" with a simple transform
|
||||
And a validation "local/wrap-b" that wraps "local/wrap-a" with a simple transform
|
||||
And a wrapped tool executor using the test registry with cycle
|
||||
When I try to execute the wrapping validation with cycle from "local/wrap-a"
|
||||
Then a WrappingCycleError should be raised
|
||||
|
||||
Scenario: Execution context is inherited from wrapping validation
|
||||
Given a tool registry with a tool "local/context-tool" that returns {"data": "value"}
|
||||
And a validation "local/context-wrap" that wraps "local/context-tool" with a data transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {"key": "value"}
|
||||
Then the wrapped tool should have received the validation inputs
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-Validation argument
|
||||
Given a wrapped tool executor with empty registry
|
||||
When I try to execute with a non-Validation object
|
||||
Then a TypeError should be raised from the executor
|
||||
|
||||
Scenario: WrappedToolExecutor rejects validation without wraps
|
||||
Given a validation "local/no-wrap" without wraps set
|
||||
And a wrapped tool executor with empty registry
|
||||
When I try to execute the non-wrapping validation
|
||||
Then a ValueError should be raised indicating wraps is not set
|
||||
|
||||
# ── Additional coverage: WrappingDepthExceededError ──────────────
|
||||
|
||||
Scenario: WrappingDepthExceededError is raised when chain is too deep
|
||||
Given a deep wrapping chain of 11 validations
|
||||
And a wrapped tool executor using the deep chain registry
|
||||
When I try to execute the deep chain wrapping validation
|
||||
Then a WrappingDepthExceededError should be raised with depth 10
|
||||
|
||||
# ── Additional coverage: ArgumentMapper.mapping property ─────────
|
||||
|
||||
Scenario: ArgumentMapper exposes the raw mapping via property
|
||||
Given an argument mapper with mapping {"x": "y"}
|
||||
Then the mapper mapping property should return {"x": "y"}
|
||||
|
||||
Scenario: ArgumentMapper mapping property returns None for identity mapper
|
||||
Given an argument mapper with no mapping
|
||||
Then the mapper mapping property should be None
|
||||
|
||||
# ── Additional coverage: TransformExecutor type checks ───────────
|
||||
|
||||
Scenario: TransformExecutor rejects non-string transform code
|
||||
When I try to create a transform executor with non-string code
|
||||
Then a TypeError should be raised from transform code type check
|
||||
|
||||
Scenario: TransformExecutor rejects non-string tool name
|
||||
When I try to create a transform executor with non-string tool name
|
||||
Then a TypeError should be raised from transform tool name check
|
||||
|
||||
Scenario: TransformExecutor raises on code that throws during exec
|
||||
Given a transform executor with code that raises an error during exec
|
||||
When I try to execute the transform with any output
|
||||
Then a TransformExecutionError should be raised with message containing "compile"
|
||||
|
||||
# ── Additional coverage: WrappedToolExecutor init checks ─────────
|
||||
|
||||
Scenario: WrappedToolExecutor rejects None tool_lookup
|
||||
When I try to create a wrapped tool executor with None tool_lookup
|
||||
Then a ValueError should be raised from executor construction for tool_lookup
|
||||
|
||||
Scenario: WrappedToolExecutor rejects None tool_executor
|
||||
When I try to create a wrapped tool executor with None tool_executor
|
||||
Then a ValueError should be raised from executor construction for tool_executor
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-callable tool_lookup
|
||||
When I try to create a wrapped tool executor with non-callable tool_lookup
|
||||
Then a TypeError should be raised from executor construction for tool_lookup
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-callable tool_executor
|
||||
When I try to create a wrapped tool executor with non-callable tool_executor
|
||||
Then a TypeError should be raised from executor construction for tool_executor
|
||||
|
||||
# ── Additional coverage: execute with non-dict inputs ────────────
|
||||
|
||||
Scenario: WrappedToolExecutor rejects non-dict inputs
|
||||
Given a tool registry with a tool "local/some-tool" that returns {"ok": true}
|
||||
And a validation "local/input-wrap" that wraps "local/some-tool" with a simple transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I try to execute wrapping validation with non-dict inputs
|
||||
Then a TypeError should be raised for non-dict inputs
|
||||
|
||||
# ── Additional coverage: chain with None wraps at end ────────────
|
||||
|
||||
Scenario: Chain resolution handles validation with wraps set to None in chain
|
||||
Given a tool registry with a tool "local/leaf" that returns {"value": 1}
|
||||
And a validation "local/wrapper-no-inner" that wraps "local/leaf" with a passthrough transform
|
||||
And a wrapped tool executor using the test registry
|
||||
When I execute the wrapping validation with inputs {}
|
||||
Then the wrapped execution should succeed with passed true
|
||||
|
||||
# ── Additional coverage: _execute_chain with None wraps target ────
|
||||
|
||||
Scenario: _execute_chain raises ValueError when leaf wraps is None
|
||||
Given a wrapped tool executor with empty registry
|
||||
When I directly call _execute_chain with a no-wraps leaf
|
||||
Then a ValueError should be raised for missing wraps target
|
||||
|
||||
# ── ToolRunner coverage: execution environment and error paths ───
|
||||
|
||||
Scenario: ToolRunner resolve_execution_environment delegates to resolver
|
||||
Given a ToolRunner with a mock registry
|
||||
When I call resolve_execution_environment on the runner
|
||||
Then the resolved environment should be local
|
||||
|
||||
Scenario: ToolRunner execute returns error when env resolver raises ValueError
|
||||
Given a ToolRunner with a value-error-raising env resolver
|
||||
When I execute a tool through the runner with env error
|
||||
Then the tool result should have success false
|
||||
And the tool result error should contain "Execution environment error"
|
||||
|
||||
Scenario: ToolRunner execute returns error for container environment
|
||||
Given a ToolRunner with a container-returning env resolver
|
||||
When I execute a tool through the runner with container env
|
||||
Then the tool result should have success false
|
||||
And the tool result error should contain "Container execution is not yet implemented"
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Robot helper: verify WrappedToolExecutor delegates to wrapped tool."""
|
||||
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import WrappedToolExecutor
|
||||
|
||||
tool = Tool(
|
||||
name="local/run",
|
||||
description="run",
|
||||
source=ToolSource.BUILTIN,
|
||||
capability=ToolCapability(read_only=True),
|
||||
)
|
||||
val = Validation(
|
||||
name="local/check",
|
||||
description="check",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps="local/run",
|
||||
transform='def transform(x):\n return {"passed": True, "data": x}\n',
|
||||
)
|
||||
lookup = lambda n: tool if n == "local/run" else None # noqa: E731
|
||||
called: list[str] = []
|
||||
|
||||
|
||||
def executor_fn(n: str, a: dict) -> dict: # type: ignore[type-arg]
|
||||
"""Record tool call and return output."""
|
||||
called.append(n)
|
||||
return {"result": "ok"}
|
||||
|
||||
|
||||
ex = WrappedToolExecutor(lookup, executor_fn)
|
||||
r = ex.execute(val, {"a": 1})
|
||||
assert r["passed"] is True
|
||||
assert "local/run" in called
|
||||
print("OK")
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Robot helper: verify ArgumentMapper translates arguments."""
|
||||
|
||||
from cleveragents.tool.wrapping import ArgumentMapper
|
||||
|
||||
m = ArgumentMapper({"dest": "src", "flag": True})
|
||||
r = m.apply({"src": "/path"})
|
||||
assert r == {"dest": "/path", "flag": True}, f"Got {r}"
|
||||
print("OK")
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Robot helper: verify ArgumentMapper with None mapping."""
|
||||
|
||||
from cleveragents.tool.wrapping import ArgumentMapper
|
||||
|
||||
m = ArgumentMapper(None)
|
||||
r = m.apply({"x": 1, "y": "two"})
|
||||
assert r == {"x": 1, "y": "two"}, f"Got {r}"
|
||||
print("OK")
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Robot helper: verify WrappedToolExecutor raises on missing tool."""
|
||||
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
ToolSource,
|
||||
ToolType,
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.wrapping import WrappedToolExecutor, WrappedToolNotFoundError
|
||||
|
||||
val = Validation(
|
||||
name="local/check",
|
||||
description="check",
|
||||
source=ToolSource.WRAPPED,
|
||||
tool_type=ToolType.VALIDATION,
|
||||
mode=ValidationMode.REQUIRED,
|
||||
wraps="local/missing",
|
||||
transform='def transform(x):\n return {"passed": True}\n',
|
||||
)
|
||||
ok = False
|
||||
try:
|
||||
WrappedToolExecutor(
|
||||
lambda n: None,
|
||||
lambda n, a: {},
|
||||
).execute(val, {})
|
||||
except WrappedToolNotFoundError:
|
||||
ok = True
|
||||
print(f"error_raised={ok}")
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Robot helper: verify TransformExecutor runs a transform."""
|
||||
|
||||
from cleveragents.tool.wrapping import TransformExecutor
|
||||
|
||||
code = 'def transform(x):\n return {"passed": x.get("ok"), "message": "done"}\n'
|
||||
t = TransformExecutor(code, "test/t")
|
||||
r = t.execute({"ok": True})
|
||||
assert r["passed"] is True
|
||||
print("OK")
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Robot helper: verify TransformExecutor blocks import in sandbox."""
|
||||
|
||||
from cleveragents.tool.wrapping import TransformExecutionError, TransformExecutor
|
||||
|
||||
code = 'def transform(x):\n import os\n return {"passed": True}\n'
|
||||
t = TransformExecutor(code, "test/s")
|
||||
ok = False
|
||||
try:
|
||||
t.execute({})
|
||||
except TransformExecutionError:
|
||||
ok = True
|
||||
print(f"blocked={ok}")
|
||||
@@ -0,0 +1,58 @@
|
||||
*** Settings ***
|
||||
Documentation Tool wrapping runtime smoke tests
|
||||
Library Process
|
||||
Library OperatingSystem
|
||||
|
||||
*** Variables ***
|
||||
${PYTHON} python
|
||||
|
||||
*** Test Cases ***
|
||||
Wrapping Package Is Importable
|
||||
[Documentation] Verify tool wrapping module can be imported
|
||||
${result}= Run Process ${PYTHON} -c
|
||||
... from cleveragents.tool.wrapping import ArgumentMapper, TransformExecutor, WrappedToolExecutor, WrappedToolNotFoundError, WrappingCycleError, WrappingDepthExceededError, TransformExecutionError; print('OK')
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
Wrapping Exports In Tool Init
|
||||
[Documentation] Verify wrapping types are exported from tool package
|
||||
${result}= Run Process ${PYTHON} -c
|
||||
... from cleveragents.tool import ArgumentMapper, TransformExecutor, WrappedToolExecutor; print('OK')
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
ArgumentMapper Identity Mapping
|
||||
[Documentation] Verify ArgumentMapper with None mapping passes args through
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_identity.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
ArgumentMapper With Mapping
|
||||
[Documentation] Verify ArgumentMapper translates arguments
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_mapper_configured.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
TransformExecutor Runs Transform
|
||||
[Documentation] Verify TransformExecutor executes transform code
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_transform_run.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
TransformExecutor Sandbox Blocks Import
|
||||
[Documentation] Verify TransformExecutor blocks import in sandbox
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_transform_sandbox.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} blocked=True
|
||||
|
||||
WrappedToolExecutor Simple Delegation
|
||||
[Documentation] Verify WrappedToolExecutor delegates to wrapped tool
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_delegation.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} OK
|
||||
|
||||
WrappedToolExecutor Missing Tool Error
|
||||
[Documentation] Verify WrappedToolExecutor raises on missing wrapped tool
|
||||
${result}= Run Process ${PYTHON} robot/scripts/test_missing_tool.py
|
||||
Should Be Equal As Integers ${result.rc} 0 ${result.stderr}
|
||||
Should Contain ${result.stdout} error_raised=True
|
||||
@@ -69,8 +69,18 @@ from cleveragents.tool.schema_validator import (
|
||||
validate_tool_input,
|
||||
validate_tool_output,
|
||||
)
|
||||
from cleveragents.tool.wrapping import (
|
||||
ArgumentMapper,
|
||||
TransformExecutionError,
|
||||
TransformExecutor,
|
||||
WrappedToolExecutor,
|
||||
WrappedToolNotFoundError,
|
||||
WrappingCycleError,
|
||||
WrappingDepthExceededError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ArgumentMapper",
|
||||
"BoundResource",
|
||||
"CancellationToken",
|
||||
"Change",
|
||||
@@ -116,6 +126,12 @@ __all__ = [
|
||||
"ToolSandboxRequiredError",
|
||||
"ToolSchemaValidationError",
|
||||
"ToolSpec",
|
||||
"TransformExecutionError",
|
||||
"TransformExecutor",
|
||||
"WrappedToolExecutor",
|
||||
"WrappedToolNotFoundError",
|
||||
"WrappingCycleError",
|
||||
"WrappingDepthExceededError",
|
||||
"classify_tool_error",
|
||||
"detect_provider_format",
|
||||
"generate_tool_call_id",
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
"""Tool wrapping runtime for CleverAgents v3.
|
||||
|
||||
Implements the ``wraps`` + ``transform`` + ``argument_mapping`` delegation
|
||||
mechanism described in the specification (§ Core Concepts > Validation >
|
||||
Tool Wrapping).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
WrappedToolExecutor
|
||||
|-- ArgumentMapper (translate wrapper args -> wrapped tool args)
|
||||
|-- TransformExecutor (sandboxed transform function on tool output)
|
||||
+-- ToolRuntime (resolve + invoke wrapped tools)
|
||||
```
|
||||
|
||||
## Delegation Chain
|
||||
|
||||
A validation wrapping another validation forms a delegation chain:
|
||||
|
||||
```
|
||||
validation-A -> validation-B -> tool-C
|
||||
(wraps B) (wraps C) (leaf)
|
||||
```
|
||||
|
||||
``WrappedToolExecutor.execute()`` recursively resolves the chain,
|
||||
detecting cycles and enforcing a maximum depth.
|
||||
|
||||
## Sandbox
|
||||
|
||||
``TransformExecutor`` compiles and executes the user-supplied
|
||||
``transform`` function with a restricted set of built-in names.
|
||||
No filesystem, network, or import access is available inside the
|
||||
transform.
|
||||
|
||||
Based on ``docs/specification.md`` § Tool Wrapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from cleveragents.domain.models.core.tool import Validation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Maximum delegation chain depth to prevent infinite recursion.
|
||||
MAX_WRAPPING_DEPTH = 10
|
||||
|
||||
# Safe built-ins available inside the ``transform`` sandbox.
|
||||
_SAFE_BUILTINS: dict[str, Any] = {
|
||||
"True": True,
|
||||
"False": False,
|
||||
"None": None,
|
||||
"abs": abs,
|
||||
"all": all,
|
||||
"any": any,
|
||||
"bool": bool,
|
||||
"dict": dict,
|
||||
"enumerate": enumerate,
|
||||
"float": float,
|
||||
"frozenset": frozenset,
|
||||
"int": int,
|
||||
"isinstance": isinstance,
|
||||
"len": len,
|
||||
"list": list,
|
||||
"map": map,
|
||||
"max": max,
|
||||
"min": min,
|
||||
"range": range,
|
||||
"repr": repr,
|
||||
"reversed": reversed,
|
||||
"round": round,
|
||||
"set": set,
|
||||
"sorted": sorted,
|
||||
"str": str,
|
||||
"sum": sum,
|
||||
"tuple": tuple,
|
||||
"type": type,
|
||||
"zip": zip,
|
||||
}
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Errors
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class WrappedToolNotFoundError(Exception):
|
||||
"""Raised when the tool referenced by ``wraps`` is not registered."""
|
||||
|
||||
def __init__(self, tool_name: str, wraps_name: str) -> None:
|
||||
self.tool_name = tool_name
|
||||
self.wraps_name = wraps_name
|
||||
super().__init__(
|
||||
f"Validation '{tool_name}' wraps '{wraps_name}' but "
|
||||
f"'{wraps_name}' is not registered in the runtime"
|
||||
)
|
||||
|
||||
|
||||
class WrappingCycleError(Exception):
|
||||
"""Raised when a circular wrapping chain is detected."""
|
||||
|
||||
def __init__(self, chain: list[str]) -> None:
|
||||
self.chain = list(chain)
|
||||
cycle_str = " -> ".join(chain)
|
||||
super().__init__(f"Circular wrapping chain detected: {cycle_str}")
|
||||
|
||||
|
||||
class WrappingDepthExceededError(Exception):
|
||||
"""Raised when the delegation chain exceeds ``MAX_WRAPPING_DEPTH``."""
|
||||
|
||||
def __init__(self, depth: int) -> None:
|
||||
self.depth = depth
|
||||
super().__init__(
|
||||
f"Wrapping delegation chain depth ({depth}) exceeds "
|
||||
f"the maximum ({MAX_WRAPPING_DEPTH})"
|
||||
)
|
||||
|
||||
|
||||
class TransformExecutionError(Exception):
|
||||
"""Raised when the ``transform`` function fails during execution."""
|
||||
|
||||
def __init__(self, tool_name: str, detail: str) -> None:
|
||||
self.tool_name = tool_name
|
||||
self.detail = detail
|
||||
super().__init__(f"Transform function for '{tool_name}' failed: {detail}")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# ArgumentMapper
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class ArgumentMapper:
|
||||
"""Applies ``argument_mapping`` to translate arguments.
|
||||
|
||||
When a validation wraps a tool, the validation may accept different
|
||||
input arguments. The ``argument_mapping`` dictionary specifies how
|
||||
the validation's inputs map to the wrapped tool's expected inputs.
|
||||
|
||||
Keys are the wrapped tool's parameter names. Values are either:
|
||||
- A string referencing a validation input parameter (forwarded).
|
||||
- A literal value (string, number, boolean) always sent to the
|
||||
wrapped tool.
|
||||
|
||||
When ``argument_mapping`` is ``None``, input arguments are passed
|
||||
through to the wrapped tool unchanged (identity mapping).
|
||||
"""
|
||||
|
||||
def __init__(self, mapping: dict[str, Any] | None) -> None:
|
||||
if mapping is not None and not isinstance(mapping, dict):
|
||||
raise TypeError(
|
||||
f"argument_mapping must be a dict or None, got {type(mapping).__name__}"
|
||||
)
|
||||
self._mapping = mapping
|
||||
|
||||
@property
|
||||
def mapping(self) -> dict[str, Any] | None:
|
||||
"""Return the raw mapping configuration."""
|
||||
return self._mapping
|
||||
|
||||
def apply(self, validation_inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Translate *validation_inputs* into wrapped-tool arguments.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
validation_inputs:
|
||||
The arguments the validation was invoked with.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]:
|
||||
Arguments to forward to the wrapped tool.
|
||||
"""
|
||||
if not isinstance(validation_inputs, dict):
|
||||
raise TypeError(
|
||||
f"validation_inputs must be a dict, "
|
||||
f"got {type(validation_inputs).__name__}"
|
||||
)
|
||||
if self._mapping is None:
|
||||
return dict(validation_inputs)
|
||||
|
||||
mapped: dict[str, Any] = {}
|
||||
for wrapped_param, source in self._mapping.items():
|
||||
if isinstance(source, str) and source in validation_inputs:
|
||||
mapped[wrapped_param] = validation_inputs[source]
|
||||
else:
|
||||
# Literal value (or string not present as a key -- use
|
||||
# as literal).
|
||||
mapped[wrapped_param] = source
|
||||
return mapped
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# TransformExecutor
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class TransformExecutor:
|
||||
"""Executes the ``transform`` function in a sandboxed context.
|
||||
|
||||
The transform code must define a function called ``transform`` that
|
||||
accepts one positional argument (the wrapped tool's output) and
|
||||
returns a dict with at minimum ``{"passed": bool}``.
|
||||
|
||||
The sandbox disallows imports, file I/O, and network access by
|
||||
restricting available built-ins.
|
||||
"""
|
||||
|
||||
def __init__(self, transform_code: str, tool_name: str) -> None:
|
||||
if not isinstance(transform_code, str):
|
||||
raise TypeError(
|
||||
f"transform_code must be a str, got {type(transform_code).__name__}"
|
||||
)
|
||||
if not transform_code.strip():
|
||||
raise ValueError("transform_code must not be empty")
|
||||
if not isinstance(tool_name, str):
|
||||
raise TypeError(f"tool_name must be a str, got {type(tool_name).__name__}")
|
||||
self._code = transform_code
|
||||
self._tool_name = tool_name
|
||||
self._compiled = compile(self._code, f"<transform:{tool_name}>", "exec")
|
||||
|
||||
def execute(self, tool_output: Any) -> dict[str, Any]:
|
||||
"""Run the transform on *tool_output*.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]:
|
||||
The validation-format result (must contain ``passed``).
|
||||
|
||||
Raises
|
||||
------
|
||||
TransformExecutionError:
|
||||
If the transform function is missing, raises, or returns
|
||||
an invalid result.
|
||||
"""
|
||||
sandbox: dict[str, Any] = {"__builtins__": dict(_SAFE_BUILTINS)}
|
||||
|
||||
try:
|
||||
exec(self._compiled, sandbox) # Controlled sandbox execution
|
||||
except Exception as exc:
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
f"Failed to compile/execute transform code: {exc}",
|
||||
) from exc
|
||||
|
||||
transform_fn = sandbox.get("transform")
|
||||
if transform_fn is None or not callable(transform_fn):
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
"Transform code must define a callable 'transform' function",
|
||||
)
|
||||
|
||||
try:
|
||||
result = transform_fn(tool_output)
|
||||
except Exception as exc:
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
f"Transform function raised: {type(exc).__name__}: {exc}",
|
||||
) from exc
|
||||
|
||||
if not isinstance(result, dict):
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
f"Transform must return a dict, got {type(result).__name__}",
|
||||
)
|
||||
|
||||
if "passed" not in result:
|
||||
raise TransformExecutionError(
|
||||
self._tool_name,
|
||||
"Transform result must contain a 'passed' key",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# WrappedToolExecutor
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
class WrappedToolExecutor:
|
||||
"""Resolves ``wraps`` references and delegates execution.
|
||||
|
||||
Orchestrates the full wrapping flow:
|
||||
|
||||
1. Resolve the wrapped tool from the tool lookup.
|
||||
2. If the wrapped tool is itself a wrapping validation, recurse.
|
||||
3. Apply ``argument_mapping`` to translate arguments.
|
||||
4. Execute the leaf tool.
|
||||
5. Walk back up the chain applying ``transform`` at each level.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tool_lookup:
|
||||
A callable that accepts a tool name (str) and returns
|
||||
the ``Validation`` or ``Tool`` domain model, or ``None``
|
||||
if not found.
|
||||
tool_executor:
|
||||
A callable ``(tool_name, mapped_args) -> dict`` that
|
||||
actually runs the leaf tool and returns its raw output.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tool_lookup: Any,
|
||||
tool_executor: Any,
|
||||
) -> None:
|
||||
if tool_lookup is None:
|
||||
raise ValueError("tool_lookup must not be None")
|
||||
if tool_executor is None:
|
||||
raise ValueError("tool_executor must not be None")
|
||||
if not callable(tool_lookup):
|
||||
raise TypeError(
|
||||
f"tool_lookup must be callable, got {type(tool_lookup).__name__}"
|
||||
)
|
||||
if not callable(tool_executor):
|
||||
raise TypeError(
|
||||
f"tool_executor must be callable, got {type(tool_executor).__name__}"
|
||||
)
|
||||
self._tool_lookup = tool_lookup
|
||||
self._tool_executor = tool_executor
|
||||
|
||||
def execute(
|
||||
self,
|
||||
validation: Validation,
|
||||
inputs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Execute a wrapping validation.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
validation:
|
||||
The wrapping ``Validation`` (must have ``wraps`` set).
|
||||
inputs:
|
||||
The arguments the validation was invoked with.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, Any]:
|
||||
The validation-format result with ``passed``, optionally
|
||||
``message`` and ``data``.
|
||||
|
||||
Raises
|
||||
------
|
||||
WrappedToolNotFoundError:
|
||||
If any tool in the chain is not registered.
|
||||
WrappingCycleError:
|
||||
If the chain contains a cycle.
|
||||
WrappingDepthExceededError:
|
||||
If the chain exceeds ``MAX_WRAPPING_DEPTH``.
|
||||
TransformExecutionError:
|
||||
If any transform function in the chain fails.
|
||||
"""
|
||||
if not isinstance(validation, Validation):
|
||||
raise TypeError(
|
||||
f"validation must be a Validation, got {type(validation).__name__}"
|
||||
)
|
||||
if not isinstance(inputs, dict):
|
||||
raise TypeError(f"inputs must be a dict, got {type(inputs).__name__}")
|
||||
if validation.wraps is None:
|
||||
raise ValueError(
|
||||
f"Validation '{validation.name}' does not have "
|
||||
f"'wraps' set -- use direct execution instead"
|
||||
)
|
||||
|
||||
chain = self._resolve_chain(validation)
|
||||
return self._execute_chain(chain, inputs)
|
||||
|
||||
# -- Chain resolution --------------------------------------------------
|
||||
|
||||
def _resolve_chain(
|
||||
self,
|
||||
validation: Validation,
|
||||
) -> list[Validation]:
|
||||
"""Walk the wraps chain and return the ordered list.
|
||||
|
||||
The first element is the outermost wrapper, the last is the
|
||||
innermost wrapper (whose ``wraps`` target is the leaf tool).
|
||||
"""
|
||||
chain: list[Validation] = []
|
||||
visited: set[str] = set()
|
||||
current: Validation = validation
|
||||
|
||||
while True:
|
||||
if len(chain) >= MAX_WRAPPING_DEPTH:
|
||||
raise WrappingDepthExceededError(len(chain))
|
||||
|
||||
if current.name in visited:
|
||||
cycle = [v.name for v in chain] + [current.name]
|
||||
raise WrappingCycleError(cycle)
|
||||
|
||||
visited.add(current.name)
|
||||
chain.append(current)
|
||||
|
||||
wraps_name = current.wraps
|
||||
if wraps_name is None:
|
||||
break
|
||||
|
||||
wrapped = self._tool_lookup(wraps_name)
|
||||
if wrapped is None:
|
||||
raise WrappedToolNotFoundError(current.name, wraps_name)
|
||||
|
||||
if isinstance(wrapped, Validation) and wrapped.wraps is not None:
|
||||
current = wrapped
|
||||
else:
|
||||
# Leaf tool found -- stop the chain
|
||||
break
|
||||
|
||||
return chain
|
||||
|
||||
# -- Chain execution ---------------------------------------------------
|
||||
|
||||
def _execute_chain(
|
||||
self,
|
||||
chain: list[Validation],
|
||||
inputs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Execute the delegation chain.
|
||||
|
||||
1. Walk from outermost to innermost, applying argument mappings.
|
||||
2. Execute the leaf tool.
|
||||
3. Walk back from innermost to outermost, applying transforms.
|
||||
"""
|
||||
# Build the fully-mapped arguments by applying each wrapper's
|
||||
# argument_mapping in sequence.
|
||||
mapped_args = dict(inputs)
|
||||
for wrapper in chain:
|
||||
mapper = ArgumentMapper(wrapper.argument_mapping)
|
||||
mapped_args = mapper.apply(mapped_args)
|
||||
|
||||
# Execute the leaf tool (the target of the innermost wraps)
|
||||
leaf_wrapper = chain[-1]
|
||||
leaf_tool_name = leaf_wrapper.wraps
|
||||
if leaf_tool_name is None:
|
||||
raise ValueError(
|
||||
f"Innermost wrapper '{leaf_wrapper.name}' has no 'wraps' target"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Executing wrapped tool",
|
||||
extra={
|
||||
"chain": [v.name for v in chain],
|
||||
"leaf_tool": leaf_tool_name,
|
||||
},
|
||||
)
|
||||
|
||||
raw_output = self._tool_executor(leaf_tool_name, mapped_args)
|
||||
|
||||
# Apply transforms in reverse (innermost first, outermost last)
|
||||
result: dict[str, Any] = (
|
||||
raw_output if isinstance(raw_output, dict) else {"raw": raw_output}
|
||||
)
|
||||
for wrapper in reversed(chain):
|
||||
if wrapper.transform is not None:
|
||||
transform_exec = TransformExecutor(
|
||||
wrapper.transform,
|
||||
wrapper.name,
|
||||
)
|
||||
logger.debug(
|
||||
"Applying transform",
|
||||
extra={
|
||||
"wrapper": wrapper.name,
|
||||
"input_type": type(result).__name__,
|
||||
},
|
||||
)
|
||||
result = transform_exec.execute(result)
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user