forked from cleveragents/cleveragents-core
914 lines
32 KiB
Python
914 lines
32 KiB
Python
"""Step definitions for tool wrapping runtime feature tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.domain.models.core.tool import (
|
|
Tool,
|
|
ToolCapability,
|
|
ToolSource,
|
|
ToolType,
|
|
Validation,
|
|
ValidationMode,
|
|
)
|
|
from cleveragents.tool.wrapping import (
|
|
ArgumentMapper,
|
|
TransformExecutionError,
|
|
TransformExecutor,
|
|
WrappedToolExecutor,
|
|
WrappedToolNotFoundError,
|
|
WrappingCycleError,
|
|
WrappingDepthExceededError,
|
|
)
|
|
|
|
# -------------------------------------------------------------------
|
|
# Helpers
|
|
# -------------------------------------------------------------------
|
|
|
|
_SIMPLE_TRANSFORM = """\
|
|
def transform(tool_output):
|
|
passed = tool_output.get("returncode") == 0
|
|
msg = "All tests passed" if passed else "Tests failed"
|
|
return {"passed": passed, "message": msg, "data": tool_output}
|
|
"""
|
|
|
|
_PASSTHROUGH_TRANSFORM = """\
|
|
def transform(tool_output):
|
|
return {"passed": True, "message": "ok", "data": tool_output}
|
|
"""
|
|
|
|
_STATUS_TRANSFORM = """\
|
|
def transform(tool_output):
|
|
passed = tool_output.get("passed", False)
|
|
return {"passed": passed, "message": "outer", "data": tool_output}
|
|
"""
|
|
|
|
_DATA_TRANSFORM = """\
|
|
def transform(tool_output):
|
|
return {"passed": True, "message": "data ok", "data": tool_output}
|
|
"""
|
|
|
|
_RETURNCODE_TRANSFORM = """\
|
|
def transform(tool_output):
|
|
passed = tool_output.get("returncode") == 0
|
|
msg = "All tests passed" if passed else "Tests failed"
|
|
return {"passed": passed, "message": msg, "data": tool_output}
|
|
"""
|
|
|
|
|
|
def _make_validation(
|
|
name: str,
|
|
wraps: str,
|
|
transform_code: str,
|
|
argument_mapping: dict[str, Any] | None = None,
|
|
) -> Validation:
|
|
"""Create a Validation with wraps set."""
|
|
return Validation(
|
|
name=name,
|
|
description=f"Test validation wrapping {wraps}",
|
|
source=ToolSource.WRAPPED,
|
|
tool_type=ToolType.VALIDATION,
|
|
mode=ValidationMode.REQUIRED,
|
|
wraps=wraps,
|
|
transform=transform_code,
|
|
argument_mapping=argument_mapping,
|
|
)
|
|
|
|
|
|
def _make_plain_validation(name: str) -> Validation:
|
|
"""Create a Validation without wraps."""
|
|
return Validation(
|
|
name=name,
|
|
description="Test validation no wraps",
|
|
source=ToolSource.CUSTOM,
|
|
tool_type=ToolType.VALIDATION,
|
|
mode=ValidationMode.REQUIRED,
|
|
code="pass",
|
|
)
|
|
|
|
|
|
def _make_tool(name: str) -> Tool:
|
|
"""Create a plain Tool."""
|
|
return Tool(
|
|
name=name,
|
|
description=f"Test tool {name}",
|
|
source=ToolSource.BUILTIN,
|
|
capability=ToolCapability(read_only=True),
|
|
)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# ArgumentMapper steps
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given("an argument mapper with no mapping")
|
|
def step_mapper_no_mapping(context: Context) -> None:
|
|
context.mapper = ArgumentMapper(None)
|
|
|
|
|
|
@given("an argument mapper with mapping {mapping_json}")
|
|
def step_mapper_with_mapping(context: Context, mapping_json: str) -> None:
|
|
mapping = json.loads(mapping_json)
|
|
context.mapper = ArgumentMapper(mapping)
|
|
|
|
|
|
@when("I apply the mapper to inputs {inputs_json}")
|
|
def step_apply_mapper(context: Context, inputs_json: str) -> None:
|
|
inputs = json.loads(inputs_json)
|
|
context.mapped_result = context.mapper.apply(inputs)
|
|
|
|
|
|
@then("the mapped arguments should be {expected_json}")
|
|
def step_check_mapped(context: Context, expected_json: str) -> None:
|
|
expected = json.loads(expected_json)
|
|
assert context.mapped_result == expected, (
|
|
f"Expected {expected}, got {context.mapped_result}"
|
|
)
|
|
|
|
|
|
@when('I try to apply the mapper to non-dict input "{value}"')
|
|
def step_apply_mapper_non_dict(context: Context, value: str) -> None:
|
|
context.mapper_error = None
|
|
try:
|
|
context.mapper.apply(value) # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.mapper_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised from the argument mapper")
|
|
def step_check_mapper_type_error(context: Context) -> None:
|
|
assert context.mapper_error is not None, "Expected TypeError"
|
|
assert isinstance(context.mapper_error, TypeError)
|
|
|
|
|
|
@when("I try to create an argument mapper with a non-dict mapping")
|
|
def step_create_mapper_non_dict(context: Context) -> None:
|
|
context.mapper_construction_error = None
|
|
try:
|
|
ArgumentMapper("not_a_dict") # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.mapper_construction_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised from argument mapper construction")
|
|
def step_check_mapper_construction_error(context: Context) -> None:
|
|
assert context.mapper_construction_error is not None, "Expected TypeError"
|
|
assert isinstance(context.mapper_construction_error, TypeError)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# TransformExecutor steps
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given("a transform executor with code that checks returncode equals zero")
|
|
def step_transform_returncode(context: Context) -> None:
|
|
context.transform_executor = TransformExecutor(
|
|
_RETURNCODE_TRANSFORM, "test/transform"
|
|
)
|
|
|
|
|
|
@when("I execute the transform with tool output {output_json}")
|
|
def step_execute_transform(context: Context, output_json: str) -> None:
|
|
output = json.loads(output_json)
|
|
context.transform_result = context.transform_executor.execute(output)
|
|
|
|
|
|
@then("the transform result should have passed true")
|
|
def step_check_transform_passed_true(context: Context) -> None:
|
|
assert context.transform_result["passed"] is True
|
|
|
|
|
|
@then("the transform result should have passed false")
|
|
def step_check_transform_passed_false(context: Context) -> None:
|
|
assert context.transform_result["passed"] is False
|
|
|
|
|
|
@then('the transform result should have message "{expected_msg}"')
|
|
def step_check_transform_message(context: Context, expected_msg: str) -> None:
|
|
assert context.transform_result["message"] == expected_msg, (
|
|
f"Expected '{expected_msg}', got '{context.transform_result['message']}'"
|
|
)
|
|
|
|
|
|
@given("a transform executor with code that does not define a transform function")
|
|
def step_transform_no_func(context: Context) -> None:
|
|
context.transform_executor = TransformExecutor("x = 1 + 2\n", "test/no-func")
|
|
|
|
|
|
@when("I try to execute the transform with any output")
|
|
def step_try_execute_transform(context: Context) -> None:
|
|
context.transform_error = None
|
|
try:
|
|
context.transform_executor.execute({"data": "test"})
|
|
except (TransformExecutionError, ValueError) as exc:
|
|
context.transform_error = exc
|
|
|
|
|
|
@then('a TransformExecutionError should be raised with message containing "{fragment}"')
|
|
def step_check_transform_error(context: Context, fragment: str) -> None:
|
|
assert context.transform_error is not None, "Expected TransformExecutionError"
|
|
assert isinstance(context.transform_error, TransformExecutionError)
|
|
assert fragment in str(context.transform_error), (
|
|
f"Expected '{fragment}' in '{context.transform_error}'"
|
|
)
|
|
|
|
|
|
@given("a transform executor with code that returns a non-dict value")
|
|
def step_transform_non_dict_return(context: Context) -> None:
|
|
context.transform_executor = TransformExecutor(
|
|
"def transform(x):\n return 'not a dict'\n",
|
|
"test/non-dict",
|
|
)
|
|
|
|
|
|
@given("a transform executor with code that returns a dict without passed key")
|
|
def step_transform_no_passed(context: Context) -> None:
|
|
context.transform_executor = TransformExecutor(
|
|
'def transform(x):\n return {"message": "no passed"}\n',
|
|
"test/no-passed",
|
|
)
|
|
|
|
|
|
@when("I try to create a transform executor with empty code")
|
|
def step_create_transform_empty(context: Context) -> None:
|
|
context.transform_construction_error = None
|
|
try:
|
|
TransformExecutor(" ", "test/empty")
|
|
except ValueError as exc:
|
|
context.transform_construction_error = exc
|
|
|
|
|
|
@then("a ValueError should be raised from transform construction")
|
|
def step_check_transform_construction_error(context: Context) -> None:
|
|
assert context.transform_construction_error is not None, "Expected ValueError"
|
|
assert isinstance(context.transform_construction_error, ValueError)
|
|
|
|
|
|
@given("a transform executor with code that attempts to import os")
|
|
def step_transform_sandbox_import(context: Context) -> None:
|
|
context.transform_executor = TransformExecutor(
|
|
"def transform(x):\n import os\n return {'passed': True}\n",
|
|
"test/sandbox",
|
|
)
|
|
|
|
|
|
@then("a TransformExecutionError should be raised from sandbox restriction")
|
|
def step_check_sandbox_error(context: Context) -> None:
|
|
assert context.transform_error is not None, (
|
|
"Expected TransformExecutionError from sandbox"
|
|
)
|
|
assert isinstance(context.transform_error, TransformExecutionError)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# WrappedToolExecutor steps
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given('a tool registry with a tool "{tool_name}" that returns {output_json}')
|
|
def step_tool_registry_with_tool(
|
|
context: Context,
|
|
tool_name: str,
|
|
output_json: str,
|
|
) -> None:
|
|
if not hasattr(context, "test_tools"):
|
|
context.test_tools = {}
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
if not hasattr(context, "tool_call_log"):
|
|
context.tool_call_log = {}
|
|
|
|
output = json.loads(output_json)
|
|
context.test_tools[tool_name] = (_make_tool(tool_name), output)
|
|
|
|
|
|
@given('a validation "{val_name}" that wraps "{wraps_name}" with a simple transform')
|
|
def step_validation_simple_wrap(
|
|
context: Context,
|
|
val_name: str,
|
|
wraps_name: str,
|
|
) -> None:
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
v = _make_validation(val_name, wraps_name, _SIMPLE_TRANSFORM)
|
|
context.test_validations[val_name] = v
|
|
|
|
|
|
@given(
|
|
'a validation "{val_name}" that wraps "{wraps_name}" '
|
|
"with argument mapping {mapping_json}"
|
|
)
|
|
def step_validation_with_mapping(
|
|
context: Context,
|
|
val_name: str,
|
|
wraps_name: str,
|
|
mapping_json: str,
|
|
) -> None:
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
mapping = json.loads(mapping_json)
|
|
v = _make_validation(
|
|
val_name,
|
|
wraps_name,
|
|
_SIMPLE_TRANSFORM,
|
|
argument_mapping=mapping,
|
|
)
|
|
context.test_validations[val_name] = v
|
|
|
|
|
|
@given(
|
|
'a validation "{val_name}" that wraps "{wraps_name}" with a passthrough transform'
|
|
)
|
|
def step_validation_passthrough_wrap(
|
|
context: Context,
|
|
val_name: str,
|
|
wraps_name: str,
|
|
) -> None:
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
v = _make_validation(val_name, wraps_name, _PASSTHROUGH_TRANSFORM)
|
|
context.test_validations[val_name] = v
|
|
|
|
|
|
@given('a validation "{val_name}" that wraps "{wraps_name}" with a status transform')
|
|
def step_validation_status_wrap(
|
|
context: Context,
|
|
val_name: str,
|
|
wraps_name: str,
|
|
) -> None:
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
v = _make_validation(val_name, wraps_name, _STATUS_TRANSFORM)
|
|
context.test_validations[val_name] = v
|
|
|
|
|
|
@given('a validation "{val_name}" that wraps "{wraps_name}" with a data transform')
|
|
def step_validation_data_wrap(
|
|
context: Context,
|
|
val_name: str,
|
|
wraps_name: str,
|
|
) -> None:
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
v = _make_validation(val_name, wraps_name, _DATA_TRANSFORM)
|
|
context.test_validations[val_name] = v
|
|
|
|
|
|
@given("a wrapped tool executor using the test registry")
|
|
def step_create_executor(context: Context) -> None:
|
|
tools = getattr(context, "test_tools", {})
|
|
validations = getattr(context, "test_validations", {})
|
|
context.tool_call_log = {}
|
|
|
|
def lookup(name: str) -> Tool | Validation | None:
|
|
if name in validations:
|
|
return validations[name]
|
|
if name in tools:
|
|
return tools[name][0]
|
|
return None
|
|
|
|
def executor(name: str, args: dict[str, Any]) -> Any:
|
|
context.tool_call_log[name] = args
|
|
if name in tools:
|
|
return tools[name][1]
|
|
raise RuntimeError(f"Tool {name} not found in test registry")
|
|
|
|
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
|
|
|
|
|
@given("a wrapped tool executor with empty registry")
|
|
def step_create_empty_executor(context: Context) -> None:
|
|
context.tool_call_log = {}
|
|
|
|
def lookup(name: str) -> None:
|
|
return None
|
|
|
|
def executor(name: str, args: dict[str, Any]) -> Any:
|
|
raise RuntimeError(f"Tool {name} not found")
|
|
|
|
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
|
|
|
|
|
@given("a wrapped tool executor using the test registry with cycle")
|
|
def step_create_executor_with_cycle(context: Context) -> None:
|
|
validations = getattr(context, "test_validations", {})
|
|
|
|
def lookup(name: str) -> Validation | None:
|
|
return validations.get(name)
|
|
|
|
def executor(name: str, args: dict[str, Any]) -> Any:
|
|
raise RuntimeError("Should not reach leaf executor in cycle")
|
|
|
|
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
|
|
|
|
|
@when("I execute the wrapping validation with inputs {inputs_json}")
|
|
def step_execute_wrapping(context: Context, inputs_json: str) -> None:
|
|
inputs = json.loads(inputs_json)
|
|
# Find the first validation with wraps
|
|
val = None
|
|
for v in context.test_validations.values():
|
|
if v.wraps is not None:
|
|
val = v
|
|
break
|
|
assert val is not None, "No wrapping validation found"
|
|
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
|
|
context.last_executed_validation = val
|
|
|
|
|
|
@when("I execute the outer wrapping validation with inputs {inputs_json}")
|
|
def step_execute_outer_wrapping(context: Context, inputs_json: str) -> None:
|
|
inputs = json.loads(inputs_json)
|
|
val = context.test_validations.get("local/outer-wrapper")
|
|
assert val is not None, "local/outer-wrapper not found"
|
|
context.wrapped_result = context.wrapped_executor.execute(val, inputs)
|
|
|
|
|
|
@when("I try to execute the wrapping validation with inputs {inputs_json}")
|
|
def step_try_execute_wrapping(context: Context, inputs_json: str) -> None:
|
|
inputs = json.loads(inputs_json)
|
|
context.wrapped_error = None
|
|
# Find first validation with wraps
|
|
val = None
|
|
for v in getattr(context, "test_validations", {}).values():
|
|
if v.wraps is not None:
|
|
val = v
|
|
break
|
|
if val is None:
|
|
return
|
|
try:
|
|
context.wrapped_executor.execute(val, inputs)
|
|
except (WrappedToolNotFoundError, WrappingCycleError) as exc:
|
|
context.wrapped_error = exc
|
|
|
|
|
|
@when('I try to execute the wrapping validation with cycle from "{val_name}"')
|
|
def step_try_execute_cycle(context: Context, val_name: str) -> None:
|
|
context.wrapped_error = None
|
|
val = context.test_validations[val_name]
|
|
try:
|
|
context.wrapped_executor.execute(val, {})
|
|
except WrappingCycleError as exc:
|
|
context.wrapped_error = exc
|
|
|
|
|
|
@then("the wrapped execution should succeed with passed true")
|
|
def step_check_wrapped_passed(context: Context) -> None:
|
|
assert context.wrapped_result["passed"] is True, (
|
|
f"Expected passed=True, got {context.wrapped_result}"
|
|
)
|
|
|
|
|
|
@then('the wrapped tool "{tool_name}" should have been called')
|
|
def step_check_tool_called(context: Context, tool_name: str) -> None:
|
|
assert tool_name in context.tool_call_log, (
|
|
f"Tool '{tool_name}' was not called. "
|
|
f"Called: {list(context.tool_call_log.keys())}"
|
|
)
|
|
|
|
|
|
@then(
|
|
'the wrapped tool "{tool_name}" should have received argument '
|
|
'"{arg_name}" with value "{arg_value}"'
|
|
)
|
|
def step_check_tool_arg_str(
|
|
context: Context,
|
|
tool_name: str,
|
|
arg_name: str,
|
|
arg_value: str,
|
|
) -> None:
|
|
call_args = context.tool_call_log[tool_name]
|
|
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
|
|
assert call_args[arg_name] == arg_value, (
|
|
f"Expected '{arg_value}', got '{call_args[arg_name]}'"
|
|
)
|
|
|
|
|
|
@then(
|
|
'the wrapped tool "{tool_name}" should have received argument '
|
|
'"{arg_name}" with value true'
|
|
)
|
|
def step_check_tool_arg_true(
|
|
context: Context,
|
|
tool_name: str,
|
|
arg_name: str,
|
|
) -> None:
|
|
call_args = context.tool_call_log[tool_name]
|
|
assert arg_name in call_args, f"Arg '{arg_name}' not in call args: {call_args}"
|
|
assert call_args[arg_name] is True, f"Expected True, got {call_args[arg_name]}"
|
|
|
|
|
|
@then('a WrappedToolNotFoundError should be raised for "{wraps_name}"')
|
|
def step_check_not_found_error(context: Context, wraps_name: str) -> None:
|
|
assert context.wrapped_error is not None, "Expected WrappedToolNotFoundError"
|
|
assert isinstance(context.wrapped_error, WrappedToolNotFoundError)
|
|
assert wraps_name in str(context.wrapped_error)
|
|
|
|
|
|
@then("a WrappingCycleError should be raised")
|
|
def step_check_cycle_error(context: Context) -> None:
|
|
assert context.wrapped_error is not None, "Expected WrappingCycleError"
|
|
assert isinstance(context.wrapped_error, WrappingCycleError)
|
|
|
|
|
|
@then("the wrapped tool should have received the validation inputs")
|
|
def step_check_inputs_passed(context: Context) -> None:
|
|
assert len(context.tool_call_log) > 0, "No tool was called"
|
|
|
|
|
|
@when("I try to execute with a non-Validation object")
|
|
def step_try_execute_non_validation(context: Context) -> None:
|
|
context.executor_type_error = None
|
|
try:
|
|
context.wrapped_executor.execute("not_a_validation", {}) # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.executor_type_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised from the executor")
|
|
def step_check_executor_type_error(context: Context) -> None:
|
|
assert context.executor_type_error is not None, "Expected TypeError"
|
|
assert isinstance(context.executor_type_error, TypeError)
|
|
|
|
|
|
@given('a validation "{val_name}" without wraps set')
|
|
def step_validation_no_wraps(context: Context, val_name: str) -> None:
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
v = _make_plain_validation(val_name)
|
|
context.test_validations[val_name] = v
|
|
context.no_wrap_validation = v
|
|
|
|
|
|
@when("I try to execute the non-wrapping validation")
|
|
def step_try_execute_no_wraps(context: Context) -> None:
|
|
context.no_wrap_error = None
|
|
try:
|
|
context.wrapped_executor.execute(context.no_wrap_validation, {})
|
|
except ValueError as exc:
|
|
context.no_wrap_error = exc
|
|
|
|
|
|
@then("a ValueError should be raised indicating wraps is not set")
|
|
def step_check_no_wrap_error(context: Context) -> None:
|
|
assert context.no_wrap_error is not None, "Expected ValueError"
|
|
assert isinstance(context.no_wrap_error, ValueError)
|
|
assert "wraps" in str(context.no_wrap_error).lower()
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Additional coverage: WrappingDepthExceededError
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given("a deep wrapping chain of {count:d} validations")
|
|
def step_deep_chain(context: Context, count: int) -> None:
|
|
if not hasattr(context, "test_validations"):
|
|
context.test_validations = {}
|
|
if not hasattr(context, "test_tools"):
|
|
context.test_tools = {}
|
|
# Create a chain: v0 wraps v1, v1 wraps v2, ..., vN wraps leaf-tool
|
|
for i in range(count):
|
|
wraps_name = f"local/deep-v{i + 1}" if i < count - 1 else "local/deep-leaf"
|
|
v = _make_validation(
|
|
f"local/deep-v{i}",
|
|
wraps_name,
|
|
_PASSTHROUGH_TRANSFORM,
|
|
)
|
|
context.test_validations[v.name] = v
|
|
# Create the leaf tool
|
|
context.test_tools["local/deep-leaf"] = (
|
|
_make_tool("local/deep-leaf"),
|
|
{"status": "ok"},
|
|
)
|
|
|
|
|
|
@given("a wrapped tool executor using the deep chain registry")
|
|
def step_create_deep_chain_executor(context: Context) -> None:
|
|
tools = getattr(context, "test_tools", {})
|
|
validations = getattr(context, "test_validations", {})
|
|
|
|
def lookup(name: str) -> Tool | Validation | None:
|
|
if name in validations:
|
|
return validations[name]
|
|
if name in tools:
|
|
return tools[name][0]
|
|
return None
|
|
|
|
def executor(name: str, args: dict[str, Any]) -> Any:
|
|
if name in tools:
|
|
return tools[name][1]
|
|
raise RuntimeError(f"Tool {name} not found")
|
|
|
|
context.wrapped_executor = WrappedToolExecutor(lookup, executor)
|
|
|
|
|
|
@when("I try to execute the deep chain wrapping validation")
|
|
def step_try_execute_deep_chain(context: Context) -> None:
|
|
context.wrapped_error = None
|
|
val = context.test_validations["local/deep-v0"]
|
|
try:
|
|
context.wrapped_executor.execute(val, {})
|
|
except WrappingDepthExceededError as exc:
|
|
context.wrapped_error = exc
|
|
|
|
|
|
@then("a WrappingDepthExceededError should be raised with depth {depth:d}")
|
|
def step_check_depth_exceeded(context: Context, depth: int) -> None:
|
|
assert context.wrapped_error is not None, "Expected WrappingDepthExceededError"
|
|
assert isinstance(context.wrapped_error, WrappingDepthExceededError)
|
|
assert context.wrapped_error.depth == depth, (
|
|
f"Expected depth {depth}, got {context.wrapped_error.depth}"
|
|
)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Additional coverage: ArgumentMapper.mapping property
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@then("the mapper mapping property should return {expected_json}")
|
|
def step_check_mapper_property(context: Context, expected_json: str) -> None:
|
|
expected = json.loads(expected_json)
|
|
assert context.mapper.mapping == expected, (
|
|
f"Expected {expected}, got {context.mapper.mapping}"
|
|
)
|
|
|
|
|
|
@then("the mapper mapping property should be None")
|
|
def step_check_mapper_property_none(context: Context) -> None:
|
|
assert context.mapper.mapping is None, (
|
|
f"Expected None, got {context.mapper.mapping}"
|
|
)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Additional coverage: TransformExecutor type checks
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("I try to create a transform executor with non-string code")
|
|
def step_create_transform_non_string_code(context: Context) -> None:
|
|
context.transform_type_error = None
|
|
try:
|
|
TransformExecutor(12345, "test/non-string-code") # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.transform_type_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised from transform code type check")
|
|
def step_check_transform_code_type_error(context: Context) -> None:
|
|
assert context.transform_type_error is not None, "Expected TypeError"
|
|
assert isinstance(context.transform_type_error, TypeError)
|
|
|
|
|
|
@when("I try to create a transform executor with non-string tool name")
|
|
def step_create_transform_non_string_name(context: Context) -> None:
|
|
context.transform_name_type_error = None
|
|
try:
|
|
TransformExecutor("def transform(x): return {'passed': True}", 42) # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.transform_name_type_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised from transform tool name check")
|
|
def step_check_transform_name_type_error(context: Context) -> None:
|
|
assert context.transform_name_type_error is not None, "Expected TypeError"
|
|
assert isinstance(context.transform_name_type_error, TypeError)
|
|
|
|
|
|
@given("a transform executor with code that raises an error during exec")
|
|
def step_transform_exec_error(context: Context) -> None:
|
|
# This code has a top-level expression that raises during exec
|
|
context.transform_executor = TransformExecutor(
|
|
"raise RuntimeError('bad code')\ndef transform(x):\n return {'passed': True}\n",
|
|
"test/exec-error",
|
|
)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Additional coverage: WrappedToolExecutor init checks
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("I try to create a wrapped tool executor with None tool_lookup")
|
|
def step_create_executor_none_lookup(context: Context) -> None:
|
|
context.executor_init_error = None
|
|
try:
|
|
WrappedToolExecutor(None, lambda n, a: {}) # type: ignore[arg-type]
|
|
except ValueError as exc:
|
|
context.executor_init_error = exc
|
|
|
|
|
|
@then("a ValueError should be raised from executor construction for tool_lookup")
|
|
def step_check_executor_none_lookup_error(context: Context) -> None:
|
|
assert context.executor_init_error is not None, "Expected ValueError"
|
|
assert isinstance(context.executor_init_error, ValueError)
|
|
|
|
|
|
@when("I try to create a wrapped tool executor with None tool_executor")
|
|
def step_create_executor_none_executor(context: Context) -> None:
|
|
context.executor_init_error = None
|
|
try:
|
|
WrappedToolExecutor(lambda n: None, None) # type: ignore[arg-type]
|
|
except ValueError as exc:
|
|
context.executor_init_error = exc
|
|
|
|
|
|
@then("a ValueError should be raised from executor construction for tool_executor")
|
|
def step_check_executor_none_executor_error(context: Context) -> None:
|
|
assert context.executor_init_error is not None, "Expected ValueError"
|
|
assert isinstance(context.executor_init_error, ValueError)
|
|
|
|
|
|
@when("I try to create a wrapped tool executor with non-callable tool_lookup")
|
|
def step_create_executor_non_callable_lookup(context: Context) -> None:
|
|
context.executor_type_init_error = None
|
|
try:
|
|
WrappedToolExecutor("not_callable", lambda n, a: {}) # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.executor_type_init_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised from executor construction for tool_lookup")
|
|
def step_check_executor_non_callable_lookup_error(context: Context) -> None:
|
|
assert context.executor_type_init_error is not None, "Expected TypeError"
|
|
assert isinstance(context.executor_type_init_error, TypeError)
|
|
|
|
|
|
@when("I try to create a wrapped tool executor with non-callable tool_executor")
|
|
def step_create_executor_non_callable_executor(context: Context) -> None:
|
|
context.executor_type_init_error = None
|
|
try:
|
|
WrappedToolExecutor(lambda n: None, "not_callable") # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.executor_type_init_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised from executor construction for tool_executor")
|
|
def step_check_executor_non_callable_executor_error(context: Context) -> None:
|
|
assert context.executor_type_init_error is not None, "Expected TypeError"
|
|
assert isinstance(context.executor_type_init_error, TypeError)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Additional coverage: execute with non-dict inputs
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("I try to execute wrapping validation with non-dict inputs")
|
|
def step_try_execute_non_dict_inputs(context: Context) -> None:
|
|
context.inputs_type_error = None
|
|
val = None
|
|
for v in context.test_validations.values():
|
|
if v.wraps is not None:
|
|
val = v
|
|
break
|
|
assert val is not None
|
|
try:
|
|
context.wrapped_executor.execute(val, "not_a_dict") # type: ignore[arg-type]
|
|
except TypeError as exc:
|
|
context.inputs_type_error = exc
|
|
|
|
|
|
@then("a TypeError should be raised for non-dict inputs")
|
|
def step_check_non_dict_inputs_error(context: Context) -> None:
|
|
assert context.inputs_type_error is not None, "Expected TypeError"
|
|
assert isinstance(context.inputs_type_error, TypeError)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Additional coverage: innermost wrapper with no wraps target
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# ToolRunner coverage steps
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given("a ToolRunner with a mock registry")
|
|
def step_tool_runner_mock_registry(context: Context) -> None:
|
|
from cleveragents.tool.registry import ToolRegistry
|
|
from cleveragents.tool.runner import ToolRunner
|
|
from cleveragents.tool.runtime import ToolSpec
|
|
|
|
registry = ToolRegistry()
|
|
registry.register(
|
|
ToolSpec(
|
|
name="local/test-tool",
|
|
description="test",
|
|
handler=lambda inputs: {"result": "ok"},
|
|
)
|
|
)
|
|
context.tool_runner = ToolRunner(registry)
|
|
|
|
|
|
@when("I call resolve_execution_environment on the runner")
|
|
def step_call_resolve_env(context: Context) -> None:
|
|
context.resolved_env = context.tool_runner.resolve_execution_environment()
|
|
|
|
|
|
@then("the resolved environment should be local")
|
|
def step_check_env_local(context: Context) -> None:
|
|
from cleveragents.domain.models.core.plan import ExecutionEnvironment
|
|
|
|
assert context.resolved_env == ExecutionEnvironment.HOST
|
|
|
|
|
|
@given("a ToolRunner with a value-error-raising env resolver")
|
|
def step_runner_value_error_resolver(context: Context) -> None:
|
|
from unittest.mock import MagicMock
|
|
|
|
from cleveragents.tool.registry import ToolRegistry
|
|
from cleveragents.tool.runner import ToolRunner
|
|
from cleveragents.tool.runtime import ToolSpec
|
|
|
|
registry = ToolRegistry()
|
|
registry.register(
|
|
ToolSpec(
|
|
name="local/test-tool",
|
|
description="test",
|
|
handler=lambda inputs: {"result": "ok"},
|
|
)
|
|
)
|
|
runner = ToolRunner(registry)
|
|
# Mock the resolver to raise ValueError
|
|
mock_resolver = MagicMock()
|
|
mock_resolver.resolve_and_validate.side_effect = ValueError("bad env config")
|
|
runner._env_resolver = mock_resolver
|
|
context.tool_runner = runner
|
|
|
|
|
|
@when("I execute a tool through the runner with env error")
|
|
def step_execute_runner_env_error(context: Context) -> None:
|
|
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
|
|
|
|
|
|
@then("the tool result should have success false")
|
|
def step_check_runner_result_fail(context: Context) -> None:
|
|
assert context.runner_result.success is False
|
|
|
|
|
|
@then('the tool result error should contain "{fragment}"')
|
|
def step_check_runner_result_error(context: Context, fragment: str) -> None:
|
|
assert fragment in (context.runner_result.error or ""), (
|
|
f"Expected '{fragment}' in '{context.runner_result.error}'"
|
|
)
|
|
|
|
|
|
@given("a ToolRunner with a container-returning env resolver")
|
|
def step_runner_container_resolver(context: Context) -> None:
|
|
from unittest.mock import MagicMock
|
|
|
|
from cleveragents.domain.models.core.plan import ExecutionEnvironment
|
|
from cleveragents.tool.registry import ToolRegistry
|
|
from cleveragents.tool.runner import ToolRunner
|
|
from cleveragents.tool.runtime import ToolSpec
|
|
|
|
registry = ToolRegistry()
|
|
registry.register(
|
|
ToolSpec(
|
|
name="local/test-tool",
|
|
description="test",
|
|
handler=lambda inputs: {"result": "ok"},
|
|
)
|
|
)
|
|
runner = ToolRunner(registry)
|
|
# Mock the resolver to return CONTAINER
|
|
mock_resolver = MagicMock()
|
|
mock_resolver.resolve_and_validate.return_value = ExecutionEnvironment.CONTAINER
|
|
runner._env_resolver = mock_resolver
|
|
context.tool_runner = runner
|
|
|
|
|
|
@when("I execute a tool through the runner with container env")
|
|
def step_execute_runner_container(context: Context) -> None:
|
|
context.runner_result = context.tool_runner.execute("local/test-tool", {"x": 1})
|
|
|
|
|
|
@when("I directly call _execute_chain with a no-wraps leaf")
|
|
def step_direct_execute_chain_no_wraps(context: Context) -> None:
|
|
context.chain_error = None
|
|
# Create a plain validation with wraps=None
|
|
leaf_val = _make_plain_validation("local/no-wraps-leaf")
|
|
try:
|
|
# Call the private method directly to exercise lines 437-438
|
|
context.wrapped_executor._execute_chain([leaf_val], {})
|
|
except ValueError as exc:
|
|
context.chain_error = exc
|
|
|
|
|
|
@then("a ValueError should be raised for missing wraps target")
|
|
def step_check_missing_wraps_target(context: Context) -> None:
|
|
assert context.chain_error is not None, "Expected ValueError"
|
|
assert isinstance(context.chain_error, ValueError)
|
|
assert "wraps" in str(context.chain_error).lower()
|