freemo
225eab25b1
ci.yml / fix(cli): change `agents validation attach` extra args to use `--key value` named option format (#3837) (push) Failing after 0s
Co-authored-by: Jeffrey Phillips Freeman <the@jeffreyfreeman.me> Co-committed-by: Jeffrey Phillips Freeman <the@jeffreyfreeman.me>
501 lines
15 KiB
Python
501 lines
15 KiB
Python
"""Step definitions for the Tool Runtime Core feature."""
|
|
|
|
import json
|
|
import threading
|
|
from typing import Any
|
|
|
|
from behave import given, then, use_step_matcher, when
|
|
|
|
from cleveragents.domain.models.core.tool import ToolCapability
|
|
from cleveragents.tool.registry import ToolRegistry
|
|
from cleveragents.tool.runner import ToolRunner
|
|
from cleveragents.tool.runtime import ToolError, ToolResult, ToolSpec
|
|
|
|
__all__: list[str] = []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _echo_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return dict(inputs)
|
|
|
|
|
|
def _adder_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"sum": inputs.get("a", 0) + inputs.get("b", 0)}
|
|
|
|
|
|
def _failing_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
raise RuntimeError("handler exploded")
|
|
|
|
|
|
def _non_serialisable_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"bad": object()}
|
|
|
|
|
|
def _scalar_handler(inputs: dict[str, Any]) -> int:
|
|
return 42
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Givens
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a tool registry")
|
|
def step_given_tool_registry(context: Any) -> None:
|
|
context.registry = ToolRegistry()
|
|
|
|
|
|
@given('a tool spec named "{name}" with a handler')
|
|
def step_given_tool_spec_with_handler(context: Any, name: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description=f"Test tool {name}",
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
use_step_matcher("re")
|
|
|
|
|
|
@given(r'a registered tool spec named "(?P<name>[^"]+)"')
|
|
def step_given_registered_tool_spec(context: Any, name: str) -> None:
|
|
spec = ToolSpec(
|
|
name=name,
|
|
description=f"Test tool {name}",
|
|
handler=_echo_handler,
|
|
)
|
|
context.registry.register(spec)
|
|
|
|
|
|
@given(r'a registered tool spec named "(?P<name>[^"]+)" typed as "(?P<tt>[^"]+)"')
|
|
def step_given_registered_tool_spec_with_type(context: Any, name: str, tt: str) -> None:
|
|
spec = ToolSpec(
|
|
name=name,
|
|
description=f"Test tool {name}",
|
|
handler=_echo_handler,
|
|
tool_type=tt,
|
|
)
|
|
context.registry.register(spec)
|
|
|
|
|
|
use_step_matcher("parse")
|
|
|
|
|
|
@given('a registered tool spec named "{name}" with an adder handler')
|
|
def step_given_registered_adder(context: Any, name: str) -> None:
|
|
spec = ToolSpec(
|
|
name=name,
|
|
description=f"Adder tool {name}",
|
|
handler=_adder_handler,
|
|
)
|
|
context.registry.register(spec)
|
|
|
|
|
|
@given('a registered tool spec named "{name}" with a failing handler')
|
|
def step_given_registered_failing(context: Any, name: str) -> None:
|
|
spec = ToolSpec(
|
|
name=name,
|
|
description=f"Failing tool {name}",
|
|
handler=_failing_handler,
|
|
)
|
|
context.registry.register(spec)
|
|
|
|
|
|
@given('a registered tool spec named "{name}" with a non-serialisable handler')
|
|
def step_given_registered_non_serialisable(context: Any, name: str) -> None:
|
|
spec = ToolSpec(
|
|
name=name,
|
|
description=f"Bad output tool {name}",
|
|
handler=_non_serialisable_handler,
|
|
)
|
|
context.registry.register(spec)
|
|
|
|
|
|
@given('a registered tool spec named "{name}" with a scalar handler')
|
|
def step_given_registered_scalar(context: Any, name: str) -> None:
|
|
spec = ToolSpec(
|
|
name=name,
|
|
description=f"Scalar tool {name}",
|
|
handler=_scalar_handler,
|
|
)
|
|
context.registry.register(spec)
|
|
|
|
|
|
@given('a tool spec named "{name}" with read_only capability')
|
|
def step_given_spec_read_only(context: Any, name: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description=f"Read-only tool {name}",
|
|
capabilities=ToolCapability(read_only=True),
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
@given('a tool spec named "{name}" with writes capability')
|
|
def step_given_spec_writes(context: Any, name: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description=f"Writer tool {name}",
|
|
capabilities=ToolCapability(writes=True),
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
@given("a tool runner for the registry")
|
|
def step_given_tool_runner(context: Any) -> None:
|
|
context.runner = ToolRunner(context.registry)
|
|
context.lifecycle_events = []
|
|
|
|
|
|
@given("an alternate registry with {count:d} tools")
|
|
def step_given_alternate_registry(context: Any, count: int) -> None:
|
|
context.alt_registry = ToolRegistry()
|
|
for i in range(count):
|
|
spec = ToolSpec(
|
|
name=f"alt/tool-{i}",
|
|
description=f"Alt tool {i}",
|
|
handler=_echo_handler,
|
|
)
|
|
context.alt_registry.register(spec)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Whens
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I register the tool spec")
|
|
def step_when_register(context: Any) -> None:
|
|
context.registry.register(context.tool_spec)
|
|
|
|
|
|
@when('I get the tool by name "{name}"')
|
|
def step_when_get_tool(context: Any, name: str) -> None:
|
|
context.retrieved_spec = context.registry.get(name)
|
|
|
|
|
|
@when("I list all tools")
|
|
def step_when_list_all(context: Any) -> None:
|
|
context.tool_list = context.registry.list_tools()
|
|
|
|
|
|
@when('I list tools with namespace "{ns}"')
|
|
def step_when_list_namespace(context: Any, ns: str) -> None:
|
|
context.tool_list = context.registry.list_tools(namespace=ns)
|
|
|
|
|
|
@when('I list tools with tool_type "{tt}"')
|
|
def step_when_list_tool_type(context: Any, tt: str) -> None:
|
|
context.tool_list = context.registry.list_tools(tool_type=tt)
|
|
|
|
|
|
@when('I remove the tool "{name}"')
|
|
def step_when_remove(context: Any, name: str) -> None:
|
|
context.removal_result = context.registry.remove(name)
|
|
|
|
|
|
@when('I try to register a duplicate tool "{name}"')
|
|
def step_when_duplicate_register(context: Any, name: str) -> None:
|
|
dup_spec = ToolSpec(
|
|
name=name,
|
|
description=f"Duplicate {name}",
|
|
handler=_echo_handler,
|
|
)
|
|
try:
|
|
context.registry.register(dup_spec)
|
|
context.tool_error = None
|
|
except ToolError as exc:
|
|
context.tool_error = exc
|
|
|
|
|
|
@when("I discover tools from the runner")
|
|
def step_when_discover(context: Any) -> None:
|
|
context.discovered = context.runner.discover()
|
|
context.lifecycle_events.append("discover")
|
|
|
|
|
|
@when('I activate tool "{name}"')
|
|
def step_when_activate(context: Any, name: str) -> None:
|
|
context.activated_spec = context.runner.activate(name)
|
|
if hasattr(context, "lifecycle_events"):
|
|
context.lifecycle_events.append("activate")
|
|
|
|
|
|
@when('I execute tool "{name}" with inputs {inputs_json}')
|
|
def step_when_execute(context: Any, name: str, inputs_json: str) -> None:
|
|
inputs = json.loads(inputs_json)
|
|
context.tool_result = context.runner.execute(name, inputs)
|
|
if hasattr(context, "lifecycle_events"):
|
|
context.lifecycle_events.append("execute")
|
|
|
|
|
|
@when('I deactivate tool "{name}"')
|
|
def step_when_deactivate(context: Any, name: str) -> None:
|
|
context.deactivation_result = context.runner.deactivate(name)
|
|
if hasattr(context, "lifecycle_events"):
|
|
context.lifecycle_events.append("deactivate")
|
|
|
|
|
|
@when("I concurrently register {count:d} tools")
|
|
def step_when_concurrent_register(context: Any, count: int) -> None:
|
|
errors: list[Exception] = []
|
|
barrier = threading.Barrier(count)
|
|
|
|
def _register(idx: int) -> None:
|
|
try:
|
|
barrier.wait(timeout=5)
|
|
spec = ToolSpec(
|
|
name=f"conc/tool-{idx}",
|
|
description=f"Concurrent tool {idx}",
|
|
handler=_echo_handler,
|
|
)
|
|
context.registry.register(spec)
|
|
except Exception as exc:
|
|
errors.append(exc)
|
|
|
|
threads = [threading.Thread(target=_register, args=(i,)) for i in range(count)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join(timeout=10)
|
|
|
|
context.concurrent_errors = errors
|
|
|
|
|
|
@when('I execute tool "{name}" with non-serialisable inputs')
|
|
def step_when_execute_bad_inputs(context: Any, name: str) -> None:
|
|
bad_inputs: dict[str, Any] = {"bad": object()}
|
|
context.tool_result = context.runner.execute(name, bad_inputs)
|
|
|
|
|
|
@when(
|
|
'I create a ToolError with name "{name}" type "{err_type}" and details "{details}"'
|
|
)
|
|
def step_when_create_tool_error(
|
|
context: Any, name: str, err_type: str, details: str
|
|
) -> None:
|
|
context.tool_error_obj = ToolError(
|
|
tool_name=name,
|
|
error_type=err_type,
|
|
details=details,
|
|
)
|
|
|
|
|
|
@when("I create a ToolResult with success True and metadata")
|
|
def step_when_create_result_success(context: Any) -> None:
|
|
context.tool_result_obj = ToolResult(
|
|
success=True,
|
|
output={"value": 42},
|
|
metadata={"trace_id": "abc123"},
|
|
duration_ms=1.5,
|
|
)
|
|
|
|
|
|
@when('I create a ToolResult with success False and error "{error}"')
|
|
def step_when_create_result_failure(context: Any, error: str) -> None:
|
|
context.tool_result_obj = ToolResult(
|
|
success=False,
|
|
output={},
|
|
error=error,
|
|
duration_ms=0.0,
|
|
)
|
|
|
|
|
|
@when('I try to activate a missing tool "{name}"')
|
|
def step_when_activate_missing(context: Any, name: str) -> None:
|
|
try:
|
|
context.runner.activate(name)
|
|
context.tool_error = None
|
|
except ToolError as exc:
|
|
context.tool_error = exc
|
|
|
|
|
|
@when('I try to execute a missing tool "{name}" with inputs {inputs_json}')
|
|
def step_when_execute_missing(context: Any, name: str, inputs_json: str) -> None:
|
|
try:
|
|
inputs = json.loads(inputs_json)
|
|
context.runner.execute(name, inputs)
|
|
context.tool_error = None
|
|
except ToolError as exc:
|
|
context.tool_error = exc
|
|
|
|
|
|
@when("I discover tools from the alternate registry")
|
|
def step_when_discover_alt(context: Any) -> None:
|
|
context.discovered = context.runner.discover(registry=context.alt_registry)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Thens
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the tool should be in the registry")
|
|
def step_then_tool_in_registry(context: Any) -> None:
|
|
found = context.registry.get(context.tool_spec.name)
|
|
assert found is not None, "Tool not found in registry"
|
|
assert found.name == context.tool_spec.name
|
|
|
|
|
|
@then('the retrieved tool spec name should be "{name}"')
|
|
def step_then_retrieved_name(context: Any, name: str) -> None:
|
|
assert context.retrieved_spec is not None
|
|
assert context.retrieved_spec.name == name
|
|
|
|
|
|
@then("the retrieved tool spec should be None")
|
|
def step_then_retrieved_none(context: Any) -> None:
|
|
assert context.retrieved_spec is None
|
|
|
|
|
|
@then("the tool list should contain {count:d} tools")
|
|
def step_then_list_count(context: Any, count: int) -> None:
|
|
assert len(context.tool_list) == count, (
|
|
f"Expected {count} tools, got {len(context.tool_list)}"
|
|
)
|
|
|
|
|
|
@then("the removal should succeed")
|
|
def step_then_removal_success(context: Any) -> None:
|
|
assert context.removal_result is True
|
|
|
|
|
|
@then("the removal should fail")
|
|
def step_then_removal_fail(context: Any) -> None:
|
|
assert context.removal_result is False
|
|
|
|
|
|
@then('getting "{name}" should return None')
|
|
def step_then_get_returns_none(context: Any, name: str) -> None:
|
|
assert context.registry.get(name) is None
|
|
|
|
|
|
@then('a ToolError should be raised with type "{err_type}"')
|
|
def step_then_tool_error_raised(context: Any, err_type: str) -> None:
|
|
assert context.tool_error is not None, "Expected ToolError but none was raised"
|
|
assert context.tool_error.error_type == err_type
|
|
|
|
|
|
@then("the tool spec capabilities read_only should be True")
|
|
def step_then_cap_read_only(context: Any) -> None:
|
|
assert context.tool_spec.capabilities.read_only is True
|
|
|
|
|
|
@then("the tool spec capabilities writes should be False")
|
|
def step_then_cap_no_writes(context: Any) -> None:
|
|
assert context.tool_spec.capabilities.writes is False
|
|
|
|
|
|
@then("the tool spec capabilities writes should be True")
|
|
def step_then_cap_writes(context: Any) -> None:
|
|
assert context.tool_spec.capabilities.writes is True
|
|
|
|
|
|
@then("the lifecycle should complete in order")
|
|
def step_then_lifecycle_order(context: Any) -> None:
|
|
expected = ["discover", "activate", "execute", "deactivate"]
|
|
assert context.lifecycle_events == expected, (
|
|
f"Expected {expected}, got {context.lifecycle_events}"
|
|
)
|
|
|
|
|
|
@then("the tool result should be successful")
|
|
def step_then_result_success(context: Any) -> None:
|
|
assert context.tool_result.success is True
|
|
|
|
|
|
@then("the tool result should not be successful")
|
|
def step_then_result_failure(context: Any) -> None:
|
|
assert context.tool_result.success is False
|
|
|
|
|
|
@then('the tool result output should contain key "{key}"')
|
|
def step_then_result_has_key(context: Any, key: str) -> None:
|
|
assert key in context.tool_result.output, (
|
|
f"Key '{key}' not in output: {context.tool_result.output}"
|
|
)
|
|
|
|
|
|
@then("the tool result duration_ms should be non-negative")
|
|
def step_then_duration_non_negative(context: Any) -> None:
|
|
assert context.tool_result.duration_ms >= 0.0
|
|
|
|
|
|
@then("the tool result error should not be empty")
|
|
def step_then_result_error_present(context: Any) -> None:
|
|
assert context.tool_result.error is not None
|
|
assert len(context.tool_result.error) > 0
|
|
|
|
|
|
@then('the tool result error should mention "{text}"')
|
|
def step_then_result_error_mentions(context: Any, text: str) -> None:
|
|
assert context.tool_result.error is not None
|
|
assert text in context.tool_result.error, (
|
|
f"Expected '{text}' in error: {context.tool_result.error}"
|
|
)
|
|
|
|
|
|
@then("all {count:d} tools should be in the registry")
|
|
def step_then_all_concurrent_registered(context: Any, count: int) -> None:
|
|
assert len(context.concurrent_errors) == 0, (
|
|
f"Errors during concurrent registration: {context.concurrent_errors}"
|
|
)
|
|
tools = context.registry.list_tools()
|
|
assert len(tools) == count, f"Expected {count} tools, got {len(tools)}"
|
|
|
|
|
|
@then('the ToolError tool_name should be "{name}"')
|
|
def step_then_error_tool_name(context: Any, name: str) -> None:
|
|
assert context.tool_error_obj.tool_name == name
|
|
|
|
|
|
@then('the ToolError error_type should be "{err_type}"')
|
|
def step_then_error_type(context: Any, err_type: str) -> None:
|
|
assert context.tool_error_obj.error_type == err_type
|
|
|
|
|
|
@then('the ToolError details should be "{details}"')
|
|
def step_then_error_details(context: Any, details: str) -> None:
|
|
assert context.tool_error_obj.details == details
|
|
|
|
|
|
@then('the ToolError message should contain "{text}"')
|
|
def step_then_error_message_contains(context: Any, text: str) -> None:
|
|
assert text in str(context.tool_error_obj)
|
|
|
|
|
|
@then("the ToolResult success should be True")
|
|
def step_then_result_obj_success_true(context: Any) -> None:
|
|
assert context.tool_result_obj.success is True
|
|
|
|
|
|
@then("the ToolResult success should be False")
|
|
def step_then_result_obj_success_false(context: Any) -> None:
|
|
assert context.tool_result_obj.success is False
|
|
|
|
|
|
@then("the ToolResult metadata should not be empty")
|
|
def step_then_result_metadata_not_empty(context: Any) -> None:
|
|
assert len(context.tool_result_obj.metadata) > 0
|
|
|
|
|
|
@then('the ToolResult error should be "{error}"')
|
|
def step_then_result_obj_error(context: Any, error: str) -> None:
|
|
assert context.tool_result_obj.error == error
|
|
|
|
|
|
@then("the deactivation should return False")
|
|
def step_then_deactivation_false(context: Any) -> None:
|
|
assert context.deactivation_result is False
|
|
|
|
|
|
@then("the discovered list should contain {count:d} tools")
|
|
def step_then_discovered_count(context: Any, count: int) -> None:
|
|
assert len(context.discovered) == count, (
|
|
f"Expected {count} discovered, got {len(context.discovered)}"
|
|
)
|