forked from cleveragents/cleveragents-core
623 lines
21 KiB
Python
623 lines
21 KiB
Python
"""Step implementations for tool_router.feature."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.domain.models.core.tool import ToolCapability
|
|
from cleveragents.tool.registry import ToolRegistry
|
|
from cleveragents.tool.router import (
|
|
NormalizedToolCallResult,
|
|
ProviderFormat,
|
|
StreamingStatus,
|
|
StreamingToolUpdate,
|
|
ToolCallRouter,
|
|
classify_tool_error,
|
|
detect_provider_format,
|
|
generate_tool_call_id,
|
|
normalize_tool_call,
|
|
normalize_tool_schema_for_provider,
|
|
)
|
|
from cleveragents.tool.runner import ToolRunner
|
|
from cleveragents.tool.runtime import ToolSpec
|
|
|
|
# -- Helpers ----------------------------------------------------------------
|
|
|
|
|
|
def _echo_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return dict(inputs)
|
|
|
|
|
|
def _fail_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
raise RuntimeError("handler exploded")
|
|
|
|
|
|
def _validation_pass_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": True, "message": "All checks passed"}
|
|
|
|
|
|
def _validation_fail_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": False, "message": "Check failed"}
|
|
|
|
|
|
def _make_echo_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Echo tool for testing",
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
def _make_fail_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Failing tool for testing",
|
|
handler=_fail_handler,
|
|
)
|
|
|
|
|
|
def _make_validation_pass_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Passing validation tool",
|
|
handler=_validation_pass_handler,
|
|
capabilities=ToolCapability(read_only=True, writes=False),
|
|
output_schema={"validation_mode": "required"},
|
|
)
|
|
|
|
|
|
def _make_validation_fail_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Failing validation tool",
|
|
handler=_validation_fail_handler,
|
|
capabilities=ToolCapability(read_only=True, writes=False),
|
|
output_schema={"validation_mode": "required"},
|
|
)
|
|
|
|
|
|
# -- Provider Format Detection -----------------------------------------------
|
|
|
|
|
|
@given("a tool call payload with name \"{name}\" and arguments '{args}'")
|
|
def step_payload_openai(context: Context, name: str, args: str) -> None:
|
|
context.payload = {"name": name, "arguments": args}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and input {input_dict}')
|
|
def step_payload_anthropic(context: Context, name: str, input_dict: str) -> None:
|
|
context.payload = {"name": name, "input": json.loads(input_dict)}
|
|
|
|
|
|
@given(
|
|
'a tool call payload with name "{name}" and type "{type_val}" and args {args_dict}'
|
|
)
|
|
def step_payload_langchain(
|
|
context: Context, name: str, type_val: str, args_dict: str
|
|
) -> None:
|
|
context.payload = {"name": name, "type": type_val, "args": json.loads(args_dict)}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and args key only')
|
|
def step_payload_langchain_args_only(context: Context, name: str) -> None:
|
|
context.payload = {"name": name, "args": {"key": "value"}}
|
|
|
|
|
|
@given("an empty tool call payload")
|
|
def step_empty_payload(context: Context) -> None:
|
|
context.payload = {}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and dict arguments {args_dict}')
|
|
def step_payload_openai_dict_args(context: Context, name: str, args_dict: str) -> None:
|
|
context.payload = {"name": name, "arguments": json.loads(args_dict)}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and parameters key {params_dict}')
|
|
def step_payload_unknown_with_params(
|
|
context: Context, name: str, params_dict: str
|
|
) -> None:
|
|
context.payload = {"name": name, "parameters": json.loads(params_dict)}
|
|
|
|
|
|
@given("a tool call payload without a name")
|
|
def step_payload_no_name(context: Context) -> None:
|
|
context.payload = {"arguments": "{}"}
|
|
|
|
|
|
@when("I detect the provider format")
|
|
def step_detect_format(context: Context) -> None:
|
|
context.detected_format = detect_provider_format(context.payload)
|
|
|
|
|
|
@then('the detected format should be "{expected}"')
|
|
def step_check_format(context: Context, expected: str) -> None:
|
|
assert context.detected_format.value == expected, (
|
|
f"Expected {expected}, got {context.detected_format.value}"
|
|
)
|
|
|
|
|
|
# -- Payload Normalization ---------------------------------------------------
|
|
|
|
|
|
@when("I normalize the tool call")
|
|
def step_normalize(context: Context) -> None:
|
|
context.normalized = normalize_tool_call(context.payload)
|
|
|
|
|
|
@when("I try to normalize the tool call")
|
|
def step_try_normalize(context: Context) -> None:
|
|
try:
|
|
context.normalized = normalize_tool_call(context.payload)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to normalize a non-dict payload")
|
|
def step_try_normalize_non_dict(context: Context) -> None:
|
|
try:
|
|
normalize_tool_call("not a dict") # type: ignore[arg-type]
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@then('the normalized request tool_name should be "{expected}"')
|
|
def step_check_normalized_name(context: Context, expected: str) -> None:
|
|
assert context.normalized.tool_name == expected
|
|
|
|
|
|
@then('the normalized request arguments should have key "{key}"')
|
|
def step_check_normalized_args_key(context: Context, key: str) -> None:
|
|
assert key in context.normalized.arguments
|
|
|
|
|
|
@then('the normalized request provider_format should be "{expected}"')
|
|
def step_check_normalized_format(context: Context, expected: str) -> None:
|
|
assert context.normalized.provider_format.value == expected
|
|
|
|
|
|
@then('a router ValueError should be raised containing "{text}"')
|
|
def step_check_router_value_error(context: Context, text: str) -> None:
|
|
assert context.raised_error is not None, "Expected ValueError but none was raised"
|
|
assert text.lower() in str(context.raised_error).lower(), (
|
|
f"Expected '{text}' in error message, got: {context.raised_error}"
|
|
)
|
|
|
|
|
|
# -- Stable ID Generation ---------------------------------------------------
|
|
|
|
|
|
@when('I generate a tool call ID for plan "{plan}" sequence {seq:d}')
|
|
def step_generate_id(context: Context, plan: str, seq: int) -> None:
|
|
context.tool_call_id = generate_tool_call_id(plan, seq)
|
|
|
|
|
|
@when('I generate another tool call ID for plan "{plan}" sequence {seq:d}')
|
|
def step_generate_another_id(context: Context, plan: str, seq: int) -> None:
|
|
context.tool_call_id_2 = generate_tool_call_id(plan, seq)
|
|
|
|
|
|
@then('the tool call ID should start with "{prefix}"')
|
|
def step_check_id_prefix(context: Context, prefix: str) -> None:
|
|
assert context.tool_call_id.startswith(prefix), (
|
|
f"Expected prefix '{prefix}', got: {context.tool_call_id}"
|
|
)
|
|
|
|
|
|
@then("the tool call ID should have length {length:d}")
|
|
def step_check_id_length(context: Context, length: int) -> None:
|
|
assert len(context.tool_call_id) == length, (
|
|
f"Expected length {length}, got {len(context.tool_call_id)}"
|
|
)
|
|
|
|
|
|
@then("both tool call IDs should be identical")
|
|
def step_check_ids_same(context: Context) -> None:
|
|
assert context.tool_call_id == context.tool_call_id_2
|
|
|
|
|
|
@then("the tool call IDs should differ")
|
|
def step_check_ids_differ(context: Context) -> None:
|
|
assert context.tool_call_id != context.tool_call_id_2
|
|
|
|
|
|
@when("I try to generate a tool call ID with empty plan_id")
|
|
def step_try_gen_empty_plan(context: Context) -> None:
|
|
try:
|
|
generate_tool_call_id("", 0)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to generate a tool call ID with negative sequence")
|
|
def step_try_gen_neg_seq(context: Context) -> None:
|
|
try:
|
|
generate_tool_call_id("plan-1", -1)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
# -- Router Execution --------------------------------------------------------
|
|
|
|
|
|
@given('a tool registry with an echo tool "{name}"')
|
|
def step_registry_with_echo(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_echo_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool registry with a failing tool "{name}"')
|
|
def step_registry_with_fail(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_fail_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool registry with a passing validation tool "{name}"')
|
|
def step_registry_with_passing_val(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_validation_pass_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool registry with a failing validation tool "{name}"')
|
|
def step_registry_with_failing_val(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_validation_fail_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool call router for plan "{plan_id}"')
|
|
def step_create_router(context: Context, plan_id: str) -> None:
|
|
context.router = ToolCallRouter(
|
|
registry=context.registry,
|
|
runner=context.runner,
|
|
plan_id=plan_id,
|
|
)
|
|
|
|
|
|
@given("an OpenAI-format tool call for \"{name}\" with arguments '{args}'")
|
|
def step_openai_call(context: Context, name: str, args: str) -> None:
|
|
context.payload = {"name": name, "arguments": args}
|
|
|
|
|
|
@given('an Anthropic-format tool call for "{name}" with input {input_dict}')
|
|
def step_anthropic_call(context: Context, name: str, input_dict: str) -> None:
|
|
context.payload = {"name": name, "input": json.loads(input_dict)}
|
|
|
|
|
|
@given('a LangChain-format tool call for "{name}" with args {args_dict}')
|
|
def step_langchain_call(context: Context, name: str, args_dict: str) -> None:
|
|
context.payload = {"name": name, "type": "tool_call", "args": json.loads(args_dict)}
|
|
|
|
|
|
@given('provider metadata with model "{model}" and provider_id "{pid}"')
|
|
def step_provider_metadata(context: Context, model: str, pid: str) -> None:
|
|
context.provider_metadata = {"model": model, "provider_id": pid}
|
|
|
|
|
|
@given('a batch of {count:d} OpenAI-format tool calls for "{name}"')
|
|
def step_batch_calls(context: Context, count: int, name: str) -> None:
|
|
context.batch_payloads = [
|
|
{"name": name, "arguments": json.dumps({"index": i})} for i in range(count)
|
|
]
|
|
|
|
|
|
@when("I route the tool call")
|
|
def step_route(context: Context) -> None:
|
|
context.route_result = context.router.route(context.payload)
|
|
|
|
|
|
@when("I route the tool call with provider metadata")
|
|
def step_route_with_metadata(context: Context) -> None:
|
|
context.route_result = context.router.route(
|
|
context.payload, provider_metadata=context.provider_metadata
|
|
)
|
|
|
|
|
|
@when("I route the batch")
|
|
def step_route_batch(context: Context) -> None:
|
|
context.batch_results = context.router.route_batch(context.batch_payloads)
|
|
|
|
|
|
@when("I route an empty batch")
|
|
def step_route_empty_batch(context: Context) -> None:
|
|
context.batch_results = context.router.route_batch([])
|
|
|
|
|
|
@when("I route the tool call with streaming")
|
|
def step_route_streaming(context: Context) -> None:
|
|
context.stream_items = list(context.router.route_streaming(context.payload))
|
|
|
|
|
|
@then("the normalized result should be successful")
|
|
def step_check_result_success(context: Context) -> None:
|
|
assert context.route_result.result.success, (
|
|
f"Expected success, got error: {context.route_result.result.error}"
|
|
)
|
|
|
|
|
|
@then("the normalized result should not be successful")
|
|
def step_check_result_failure(context: Context) -> None:
|
|
assert not context.route_result.result.success
|
|
|
|
|
|
@then('the normalized result tool_name should be "{expected}"')
|
|
def step_check_result_name(context: Context, expected: str) -> None:
|
|
assert context.route_result.tool_name == expected
|
|
|
|
|
|
@then('the normalized result provider_format should be "{expected}"')
|
|
def step_check_result_provider(context: Context, expected: str) -> None:
|
|
assert context.route_result.provider_format.value == expected
|
|
|
|
|
|
@then('the normalized result tool_call_id should start with "{prefix}"')
|
|
def step_check_result_id_prefix(context: Context, prefix: str) -> None:
|
|
assert context.route_result.tool_call_id.startswith(prefix)
|
|
|
|
|
|
@then('the normalized result error_category should be "{expected}"')
|
|
def step_check_result_error_cat(context: Context, expected: str) -> None:
|
|
assert context.route_result.error_category is not None
|
|
assert context.route_result.error_category.value == expected
|
|
|
|
|
|
@then('the normalized result provider_metadata should contain key "{key}"')
|
|
def step_check_result_metadata_key(context: Context, key: str) -> None:
|
|
assert key in context.route_result.provider_metadata
|
|
|
|
|
|
@then("the normalized result is_validation should be True")
|
|
def step_check_is_validation_true(context: Context) -> None:
|
|
assert context.route_result.is_validation is True
|
|
|
|
|
|
@then("the normalized result validation_passed should be True")
|
|
def step_check_validation_passed_true(context: Context) -> None:
|
|
assert context.route_result.validation_passed is True
|
|
|
|
|
|
@then("the normalized result validation_passed should be False")
|
|
def step_check_validation_passed_false(context: Context) -> None:
|
|
assert context.route_result.validation_passed is False
|
|
|
|
|
|
# -- Batch results -----------------------------------------------------------
|
|
|
|
|
|
@then("the batch should return {count:d} results")
|
|
def step_check_batch_count(context: Context, count: int) -> None:
|
|
assert len(context.batch_results) == count
|
|
|
|
|
|
@then("all batch results should be successful")
|
|
def step_check_batch_all_success(context: Context) -> None:
|
|
for r in context.batch_results:
|
|
assert r.result.success, f"Batch item failed: {r.result.error}"
|
|
|
|
|
|
# -- Streaming results -------------------------------------------------------
|
|
|
|
|
|
@then("the stream should emit a pending update")
|
|
def step_check_stream_pending(context: Context) -> None:
|
|
pending = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.PENDING
|
|
]
|
|
assert len(pending) >= 1, "No pending update found"
|
|
|
|
|
|
@then("the stream should emit a running update")
|
|
def step_check_stream_running(context: Context) -> None:
|
|
running = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.RUNNING
|
|
]
|
|
assert len(running) >= 1, "No running update found"
|
|
|
|
|
|
@then("the stream should emit a complete update")
|
|
def step_check_stream_complete(context: Context) -> None:
|
|
complete = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.COMPLETE
|
|
]
|
|
assert len(complete) >= 1, "No complete update found"
|
|
|
|
|
|
@then("the stream should emit a final result")
|
|
def step_check_stream_final(context: Context) -> None:
|
|
results = [
|
|
i for i in context.stream_items if isinstance(i, NormalizedToolCallResult)
|
|
]
|
|
assert len(results) >= 1, "No final result found"
|
|
|
|
|
|
@then("the stream should emit an error update")
|
|
def step_check_stream_error(context: Context) -> None:
|
|
errors = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.ERROR
|
|
]
|
|
assert len(errors) >= 1, "No error update found"
|
|
|
|
|
|
@then("the stream should emit a final result that is not successful")
|
|
def step_check_stream_final_failure(context: Context) -> None:
|
|
results = [
|
|
i for i in context.stream_items if isinstance(i, NormalizedToolCallResult)
|
|
]
|
|
assert len(results) >= 1, "No final result found"
|
|
assert not results[-1].result.success
|
|
|
|
|
|
# -- Error Classification ---------------------------------------------------
|
|
|
|
|
|
@when('I classify the error "{error_msg}"')
|
|
def step_classify_error(context: Context, error_msg: str) -> None:
|
|
context.error_category = classify_tool_error(error_msg)
|
|
|
|
|
|
@then('the error category should be "{expected}"')
|
|
def step_check_error_cat(context: Context, expected: str) -> None:
|
|
assert context.error_category.value == expected
|
|
|
|
|
|
# -- Schema Normalization ---------------------------------------------------
|
|
|
|
|
|
@given('a tool spec named "{name}" with description "{desc}"')
|
|
def step_tool_spec_with_desc(context: Context, name: str, desc: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description=desc,
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
@given('a tool spec named "{name}" with a very long description')
|
|
def step_tool_spec_long_desc(context: Context, name: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description="A" * 2000,
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
@when('I normalize the schema for "{provider}" provider')
|
|
def step_normalize_schema(context: Context, provider: str) -> None:
|
|
context.normalized_schema = normalize_tool_schema_for_provider(
|
|
context.tool_spec, ProviderFormat(provider)
|
|
)
|
|
|
|
|
|
@when('I normalize the schema for "{provider}" provider with max length {length:d}')
|
|
def step_normalize_schema_max(context: Context, provider: str, length: int) -> None:
|
|
context.normalized_schema = normalize_tool_schema_for_provider(
|
|
context.tool_spec, ProviderFormat(provider), max_description_length=length
|
|
)
|
|
|
|
|
|
@when("I try to normalize the schema with max_description_length {length:d}")
|
|
def step_try_normalize_schema_bad_len(context: Context, length: int) -> None:
|
|
try:
|
|
normalize_tool_schema_for_provider(
|
|
context.tool_spec, ProviderFormat.OPENAI, max_description_length=length
|
|
)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@then('the normalized schema should have key "{key}"')
|
|
def step_check_schema_key(context: Context, key: str) -> None:
|
|
assert key in context.normalized_schema
|
|
|
|
|
|
@then('the normalized schema should have name "{name}"')
|
|
def step_check_schema_name(context: Context, name: str) -> None:
|
|
assert context.normalized_schema["name"] == name
|
|
|
|
|
|
@then("the normalized schema description should be at most {length:d} characters")
|
|
def step_check_schema_desc_len(context: Context, length: int) -> None:
|
|
assert len(context.normalized_schema["description"]) <= length
|
|
|
|
|
|
# -- Schema Export -----------------------------------------------------------
|
|
|
|
|
|
@when('I export schemas for "{provider}" provider')
|
|
def step_export_schemas(context: Context, provider: str) -> None:
|
|
context.exported_schemas = context.router.export_schemas(ProviderFormat(provider))
|
|
|
|
|
|
@then("the exported schemas should contain {count:d} schema")
|
|
def step_check_exported_count(context: Context, count: int) -> None:
|
|
assert len(context.exported_schemas) == count
|
|
|
|
|
|
@then('the first exported schema should have tool_type "{expected}"')
|
|
def step_check_exported_type(context: Context, expected: str) -> None:
|
|
assert context.exported_schemas[0]["tool_type"] == expected
|
|
|
|
|
|
# -- Sequence counter -------------------------------------------------------
|
|
|
|
|
|
@then("the router sequence should be {expected:d}")
|
|
def step_check_sequence(context: Context, expected: int) -> None:
|
|
assert context.router.sequence == expected
|
|
|
|
|
|
# -- Router construction validation -----------------------------------------
|
|
|
|
|
|
@when("I try to create a router with empty plan_id")
|
|
def step_try_create_router_empty(context: Context) -> None:
|
|
try:
|
|
ToolCallRouter(
|
|
registry=context.registry,
|
|
runner=context.runner,
|
|
plan_id="",
|
|
)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to route a non-dict payload")
|
|
def step_try_route_non_dict(context: Context) -> None:
|
|
try:
|
|
context.router.route("not a dict") # type: ignore[arg-type]
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to route_batch a non-list payload")
|
|
def step_try_route_batch_non_list(context: Context) -> None:
|
|
try:
|
|
context.router.route_batch("not a list") # type: ignore[arg-type]
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
# -- Detect format non-dict -------------------------------------------------
|
|
|
|
|
|
@when("I detect format for a non-dict value")
|
|
def step_detect_non_dict(context: Context) -> None:
|
|
context.detected_format = detect_provider_format("not-a-dict") # type: ignore[arg-type]
|
|
|
|
|
|
@when("I classify an empty error message")
|
|
def step_classify_empty_error(context: Context) -> None:
|
|
context.error_category = classify_tool_error("")
|
|
|
|
|
|
@given("a tool call with invalid JSON arguments")
|
|
def step_payload_invalid_json(context: Context) -> None:
|
|
context.payload = {"name": "test/echo", "arguments": "not valid json"}
|