hamza.khyari
cd9cb9e889
CI / build (pull_request) Successful in 15s
CI / helm (pull_request) Successful in 22s
CI / lint (pull_request) Successful in 3m19s
CI / quality (pull_request) Successful in 3m54s
CI / typecheck (pull_request) Successful in 3m57s
CI / security (pull_request) Successful in 4m27s
CI / integration_tests (pull_request) Successful in 7m1s
CI / unit_tests (pull_request) Successful in 7m44s
CI / docker (pull_request) Successful in 1m31s
CI / coverage (pull_request) Successful in 12m18s
CI / e2e_tests (pull_request) Successful in 21m52s
CI / status-check (pull_request) Successful in 1s
CI / benchmark-publish (pull_request) Has been skipped
CI / benchmark-regression (pull_request) Successful in 54m34s
Thread plan_env and project_env through the tool execution chain so the ExecutionEnvironmentResolver receives project-level execution environment values stored in ContextConfig.execution_environment. The resolver's precedence logic (tool > plan > project > default) was already correct, but callers never passed project_env — tools always fell through to the global HOST default, ignoring project-level overrides entirely. Changes: - PlanExecutionContext: add plan_env and project_env fields + properties - ToolCallRouter: accept plan_env/project_env in constructor, pass to runner.execute() in route() and route_streaming() - ToolCallingRuntime: accept plan_env/project_env, pass to runner.execute() in the direct-runner fallback path - Update monkey-patched execute stubs in tool_router_steps.py to accept explicit plan_env/project_env keyword arguments - Add 4 BDD scenarios proving plan_env/project_env reach the resolver via ToolCallRouter (project-only, plan-only, both, neither) Dependency: requires PR #1135 to be merged first (adds CLI flag and persistence for project-level execution-env-priority). Closes #1080 ISSUES CLOSED: #1080
1033 lines
34 KiB
Python
1033 lines
34 KiB
Python
"""Step implementations for tool_router.feature."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.domain.models.core.tool import ToolCapability
|
|
from cleveragents.tool.registry import ToolRegistry
|
|
from cleveragents.tool.router import (
|
|
NormalizedToolCallResult,
|
|
ProviderFormat,
|
|
StreamingStatus,
|
|
StreamingToolUpdate,
|
|
ToolCallRouter,
|
|
classify_tool_error,
|
|
detect_provider_format,
|
|
generate_tool_call_id,
|
|
normalize_tool_call,
|
|
normalize_tool_schema_for_provider,
|
|
)
|
|
from cleveragents.tool.runner import ToolRunner
|
|
from cleveragents.tool.runtime import ToolResult, ToolSpec
|
|
|
|
# -- Helpers ----------------------------------------------------------------
|
|
|
|
|
|
def _echo_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return dict(inputs)
|
|
|
|
|
|
def _fail_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
raise RuntimeError("handler exploded")
|
|
|
|
|
|
def _validation_pass_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": True, "message": "All checks passed"}
|
|
|
|
|
|
def _validation_fail_handler(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": False, "message": "Check failed"}
|
|
|
|
|
|
def _make_echo_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Echo tool for testing",
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
def _make_fail_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Failing tool for testing",
|
|
handler=_fail_handler,
|
|
)
|
|
|
|
|
|
def _make_validation_pass_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Passing validation tool",
|
|
handler=_validation_pass_handler,
|
|
capabilities=ToolCapability(read_only=True, writes=False),
|
|
output_schema={"validation_mode": "required"},
|
|
)
|
|
|
|
|
|
def _make_validation_fail_spec(name: str) -> ToolSpec:
|
|
return ToolSpec(
|
|
name=name,
|
|
description="Failing validation tool",
|
|
handler=_validation_fail_handler,
|
|
capabilities=ToolCapability(read_only=True, writes=False),
|
|
output_schema={"validation_mode": "required"},
|
|
)
|
|
|
|
|
|
# -- Provider Format Detection -----------------------------------------------
|
|
|
|
|
|
@given("a tool call payload with name \"{name}\" and arguments '{args}'")
|
|
def step_payload_openai(context: Context, name: str, args: str) -> None:
|
|
context.payload = {"name": name, "arguments": args}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and input {input_dict}')
|
|
def step_payload_anthropic(context: Context, name: str, input_dict: str) -> None:
|
|
context.payload = {"name": name, "input": json.loads(input_dict)}
|
|
|
|
|
|
@given(
|
|
'a tool call payload with name "{name}" and type "{type_val}" and args {args_dict}'
|
|
)
|
|
def step_payload_langchain(
|
|
context: Context, name: str, type_val: str, args_dict: str
|
|
) -> None:
|
|
context.payload = {"name": name, "type": type_val, "args": json.loads(args_dict)}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and args key only')
|
|
def step_payload_langchain_args_only(context: Context, name: str) -> None:
|
|
context.payload = {"name": name, "args": {"key": "value"}}
|
|
|
|
|
|
@given("an empty tool call payload")
|
|
def step_empty_payload(context: Context) -> None:
|
|
context.payload = {}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and dict arguments {args_dict}')
|
|
def step_payload_openai_dict_args(context: Context, name: str, args_dict: str) -> None:
|
|
context.payload = {"name": name, "arguments": json.loads(args_dict)}
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and parameters key {params_dict}')
|
|
def step_payload_unknown_with_params(
|
|
context: Context, name: str, params_dict: str
|
|
) -> None:
|
|
context.payload = {"name": name, "parameters": json.loads(params_dict)}
|
|
|
|
|
|
@given("a tool call payload without a name")
|
|
def step_payload_no_name(context: Context) -> None:
|
|
context.payload = {"arguments": "{}"}
|
|
|
|
|
|
@when("I detect the provider format")
|
|
def step_detect_format(context: Context) -> None:
|
|
context.detected_format = detect_provider_format(context.payload)
|
|
|
|
|
|
@then('the detected format should be "{expected}"')
|
|
def step_check_format(context: Context, expected: str) -> None:
|
|
assert context.detected_format.value == expected, (
|
|
f"Expected {expected}, got {context.detected_format.value}"
|
|
)
|
|
|
|
|
|
# -- Payload Normalization ---------------------------------------------------
|
|
|
|
|
|
@when("I normalize the tool call")
|
|
def step_normalize(context: Context) -> None:
|
|
context.normalized = normalize_tool_call(context.payload)
|
|
|
|
|
|
@when("I try to normalize the tool call")
|
|
def step_try_normalize(context: Context) -> None:
|
|
try:
|
|
context.normalized = normalize_tool_call(context.payload)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to normalize a non-dict payload")
|
|
def step_try_normalize_non_dict(context: Context) -> None:
|
|
try:
|
|
normalize_tool_call("not a dict") # type: ignore[arg-type]
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@then('the normalized request tool_name should be "{expected}"')
|
|
def step_check_normalized_name(context: Context, expected: str) -> None:
|
|
assert context.normalized.tool_name == expected
|
|
|
|
|
|
@then('the normalized request arguments should have key "{key}"')
|
|
def step_check_normalized_args_key(context: Context, key: str) -> None:
|
|
assert key in context.normalized.arguments
|
|
|
|
|
|
@then('the normalized request provider_format should be "{expected}"')
|
|
def step_check_normalized_format(context: Context, expected: str) -> None:
|
|
assert context.normalized.provider_format.value == expected
|
|
|
|
|
|
@then('a router ValueError should be raised containing "{text}"')
|
|
def step_check_router_value_error(context: Context, text: str) -> None:
|
|
assert context.raised_error is not None, "Expected ValueError but none was raised"
|
|
assert text.lower() in str(context.raised_error).lower(), (
|
|
f"Expected '{text}' in error message, got: {context.raised_error}"
|
|
)
|
|
|
|
|
|
# -- Stable ID Generation ---------------------------------------------------
|
|
|
|
|
|
@when('I generate a tool call ID for plan "{plan}" sequence {seq:d}')
|
|
def step_generate_id(context: Context, plan: str, seq: int) -> None:
|
|
context.tool_call_id = generate_tool_call_id(plan, seq)
|
|
|
|
|
|
@when('I generate another tool call ID for plan "{plan}" sequence {seq:d}')
|
|
def step_generate_another_id(context: Context, plan: str, seq: int) -> None:
|
|
context.tool_call_id_2 = generate_tool_call_id(plan, seq)
|
|
|
|
|
|
@then('the tool call ID should start with "{prefix}"')
|
|
def step_check_id_prefix(context: Context, prefix: str) -> None:
|
|
assert context.tool_call_id.startswith(prefix), (
|
|
f"Expected prefix '{prefix}', got: {context.tool_call_id}"
|
|
)
|
|
|
|
|
|
@then("the tool call ID should have length {length:d}")
|
|
def step_check_id_length(context: Context, length: int) -> None:
|
|
assert len(context.tool_call_id) == length, (
|
|
f"Expected length {length}, got {len(context.tool_call_id)}"
|
|
)
|
|
|
|
|
|
@then("both tool call IDs should be identical")
|
|
def step_check_ids_same(context: Context) -> None:
|
|
assert context.tool_call_id == context.tool_call_id_2
|
|
|
|
|
|
@then("the tool call IDs should differ")
|
|
def step_check_ids_differ(context: Context) -> None:
|
|
assert context.tool_call_id != context.tool_call_id_2
|
|
|
|
|
|
@when("I try to generate a tool call ID with empty plan_id")
|
|
def step_try_gen_empty_plan(context: Context) -> None:
|
|
try:
|
|
generate_tool_call_id("", 0)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to generate a tool call ID with negative sequence")
|
|
def step_try_gen_neg_seq(context: Context) -> None:
|
|
try:
|
|
generate_tool_call_id("plan-1", -1)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
# -- Router Execution --------------------------------------------------------
|
|
|
|
|
|
@given('a tool registry with an echo tool "{name}"')
|
|
def step_registry_with_echo(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_echo_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool registry with a failing tool "{name}"')
|
|
def step_registry_with_fail(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_fail_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool registry with a passing validation tool "{name}"')
|
|
def step_registry_with_passing_val(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_validation_pass_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool registry with a failing validation tool "{name}"')
|
|
def step_registry_with_failing_val(context: Context, name: str) -> None:
|
|
context.registry = ToolRegistry()
|
|
spec = _make_validation_fail_spec(name)
|
|
context.registry.register(spec)
|
|
context.runner = ToolRunner(context.registry)
|
|
|
|
|
|
@given('a tool call router for plan "{plan_id}"')
|
|
def step_create_router(context: Context, plan_id: str) -> None:
|
|
registry = getattr(context, "tool_registry", None) or context.registry
|
|
context.router = ToolCallRouter(
|
|
registry=registry,
|
|
runner=context.runner,
|
|
plan_id=plan_id,
|
|
)
|
|
|
|
|
|
@given("an OpenAI-format tool call for \"{name}\" with arguments '{args}'")
|
|
def step_openai_call(context: Context, name: str, args: str) -> None:
|
|
context.payload = {"name": name, "arguments": args}
|
|
|
|
|
|
@given('an Anthropic-format tool call for "{name}" with input {input_dict}')
|
|
def step_anthropic_call(context: Context, name: str, input_dict: str) -> None:
|
|
context.payload = {"name": name, "input": json.loads(input_dict)}
|
|
|
|
|
|
@given('a LangChain-format tool call for "{name}" with args {args_dict}')
|
|
def step_langchain_call(context: Context, name: str, args_dict: str) -> None:
|
|
context.payload = {"name": name, "type": "tool_call", "args": json.loads(args_dict)}
|
|
|
|
|
|
@given('provider metadata with model "{model}" and provider_id "{pid}"')
|
|
def step_provider_metadata(context: Context, model: str, pid: str) -> None:
|
|
context.provider_metadata = {"model": model, "provider_id": pid}
|
|
|
|
|
|
@given('a batch of {count:d} OpenAI-format tool calls for "{name}"')
|
|
def step_batch_calls(context: Context, count: int, name: str) -> None:
|
|
context.batch_payloads = [
|
|
{"name": name, "arguments": json.dumps({"index": i})} for i in range(count)
|
|
]
|
|
|
|
|
|
@when("I route the tool call")
|
|
def step_route(context: Context) -> None:
|
|
context.route_result = context.router.route(context.payload)
|
|
|
|
|
|
@when("I route the tool call with provider metadata")
|
|
def step_route_with_metadata(context: Context) -> None:
|
|
context.route_result = context.router.route(
|
|
context.payload, provider_metadata=context.provider_metadata
|
|
)
|
|
|
|
|
|
@when("I route the batch")
|
|
def step_route_batch(context: Context) -> None:
|
|
context.batch_results = context.router.route_batch(context.batch_payloads)
|
|
|
|
|
|
@when("I route an empty batch")
|
|
def step_route_empty_batch(context: Context) -> None:
|
|
context.batch_results = context.router.route_batch([])
|
|
|
|
|
|
@when("I route the tool call with streaming")
|
|
def step_route_streaming(context: Context) -> None:
|
|
context.stream_items = list(context.router.route_streaming(context.payload))
|
|
|
|
|
|
@then("the normalized result should be successful")
|
|
def step_check_result_success(context: Context) -> None:
|
|
assert context.route_result.result.success, (
|
|
f"Expected success, got error: {context.route_result.result.error}"
|
|
)
|
|
|
|
|
|
@then("the normalized result should not be successful")
|
|
def step_check_result_failure(context: Context) -> None:
|
|
assert not context.route_result.result.success
|
|
|
|
|
|
@then('the normalized result tool_name should be "{expected}"')
|
|
def step_check_result_name(context: Context, expected: str) -> None:
|
|
assert context.route_result.tool_name == expected
|
|
|
|
|
|
@then('the normalized result provider_format should be "{expected}"')
|
|
def step_check_result_provider(context: Context, expected: str) -> None:
|
|
assert context.route_result.provider_format.value == expected
|
|
|
|
|
|
@then('the normalized result tool_call_id should start with "{prefix}"')
|
|
def step_check_result_id_prefix(context: Context, prefix: str) -> None:
|
|
assert context.route_result.tool_call_id.startswith(prefix)
|
|
|
|
|
|
@then('the normalized result error_category should be "{expected}"')
|
|
def step_check_result_error_cat(context: Context, expected: str) -> None:
|
|
assert context.route_result.error_category is not None
|
|
assert context.route_result.error_category.value == expected
|
|
|
|
|
|
@then('the normalized result provider_metadata should contain key "{key}"')
|
|
def step_check_result_metadata_key(context: Context, key: str) -> None:
|
|
assert key in context.route_result.provider_metadata
|
|
|
|
|
|
@then("the normalized result is_validation should be True")
|
|
def step_check_is_validation_true(context: Context) -> None:
|
|
assert context.route_result.is_validation is True
|
|
|
|
|
|
@then("the normalized result validation_passed should be True")
|
|
def step_check_validation_passed_true(context: Context) -> None:
|
|
assert context.route_result.validation_passed is True
|
|
|
|
|
|
@then("the normalized result validation_passed should be False")
|
|
def step_check_validation_passed_false(context: Context) -> None:
|
|
assert context.route_result.validation_passed is False
|
|
|
|
|
|
# -- Batch results -----------------------------------------------------------
|
|
|
|
|
|
@then("the batch should return {count:d} results")
|
|
def step_check_batch_count(context: Context, count: int) -> None:
|
|
assert len(context.batch_results) == count
|
|
|
|
|
|
@then("all batch results should be successful")
|
|
def step_check_batch_all_success(context: Context) -> None:
|
|
for r in context.batch_results:
|
|
assert r.result.success, f"Batch item failed: {r.result.error}"
|
|
|
|
|
|
# -- Streaming results -------------------------------------------------------
|
|
|
|
|
|
@then("the stream should emit a pending update")
|
|
def step_check_stream_pending(context: Context) -> None:
|
|
pending = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.PENDING
|
|
]
|
|
assert len(pending) >= 1, "No pending update found"
|
|
|
|
|
|
@then("the stream should emit a running update")
|
|
def step_check_stream_running(context: Context) -> None:
|
|
running = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.RUNNING
|
|
]
|
|
assert len(running) >= 1, "No running update found"
|
|
|
|
|
|
@then("the stream should emit a complete update")
|
|
def step_check_stream_complete(context: Context) -> None:
|
|
complete = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.COMPLETE
|
|
]
|
|
assert len(complete) >= 1, "No complete update found"
|
|
|
|
|
|
@then("the stream should emit a final result")
|
|
def step_check_stream_final(context: Context) -> None:
|
|
results = [
|
|
i for i in context.stream_items if isinstance(i, NormalizedToolCallResult)
|
|
]
|
|
assert len(results) >= 1, "No final result found"
|
|
|
|
|
|
@then("the stream should emit an error update")
|
|
def step_check_stream_error(context: Context) -> None:
|
|
errors = [
|
|
i
|
|
for i in context.stream_items
|
|
if isinstance(i, StreamingToolUpdate) and i.status == StreamingStatus.ERROR
|
|
]
|
|
assert len(errors) >= 1, "No error update found"
|
|
|
|
|
|
@then("the stream should emit a final result that is not successful")
|
|
def step_check_stream_final_failure(context: Context) -> None:
|
|
results = [
|
|
i for i in context.stream_items if isinstance(i, NormalizedToolCallResult)
|
|
]
|
|
assert len(results) >= 1, "No final result found"
|
|
assert not results[-1].result.success
|
|
|
|
|
|
# -- Error Classification ---------------------------------------------------
|
|
|
|
|
|
@when('I classify the error "{error_msg}"')
|
|
def step_classify_error(context: Context, error_msg: str) -> None:
|
|
context.error_category = classify_tool_error(error_msg)
|
|
|
|
|
|
@then('the error category should be "{expected}"')
|
|
def step_check_error_cat(context: Context, expected: str) -> None:
|
|
assert context.error_category.value == expected
|
|
|
|
|
|
# -- Schema Normalization ---------------------------------------------------
|
|
|
|
|
|
@given('a tool spec named "{name}" with description "{desc}"')
|
|
def step_tool_spec_with_desc(context: Context, name: str, desc: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description=desc,
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
@given('a tool spec named "{name}" with a very long description')
|
|
def step_tool_spec_long_desc(context: Context, name: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description="A" * 2000,
|
|
handler=_echo_handler,
|
|
)
|
|
|
|
|
|
@when('I normalize the schema for "{provider}" provider')
|
|
def step_normalize_schema(context: Context, provider: str) -> None:
|
|
context.normalized_schema = normalize_tool_schema_for_provider(
|
|
context.tool_spec, ProviderFormat(provider)
|
|
)
|
|
|
|
|
|
@when('I normalize the schema for "{provider}" provider with max length {length:d}')
|
|
def step_normalize_schema_max(context: Context, provider: str, length: int) -> None:
|
|
context.normalized_schema = normalize_tool_schema_for_provider(
|
|
context.tool_spec, ProviderFormat(provider), max_description_length=length
|
|
)
|
|
|
|
|
|
@when("I try to normalize the schema with max_description_length {length:d}")
|
|
def step_try_normalize_schema_bad_len(context: Context, length: int) -> None:
|
|
try:
|
|
normalize_tool_schema_for_provider(
|
|
context.tool_spec, ProviderFormat.OPENAI, max_description_length=length
|
|
)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@then('the normalized schema should have key "{key}"')
|
|
def step_check_schema_key(context: Context, key: str) -> None:
|
|
assert key in context.normalized_schema
|
|
|
|
|
|
@then('the normalized schema should have name "{name}"')
|
|
def step_check_schema_name(context: Context, name: str) -> None:
|
|
assert context.normalized_schema["name"] == name
|
|
|
|
|
|
@then("the normalized schema description should be at most {length:d} characters")
|
|
def step_check_schema_desc_len(context: Context, length: int) -> None:
|
|
assert len(context.normalized_schema["description"]) <= length
|
|
|
|
|
|
# -- Schema Export -----------------------------------------------------------
|
|
|
|
|
|
@when('I export schemas for "{provider}" provider')
|
|
def step_export_schemas(context: Context, provider: str) -> None:
|
|
context.exported_schemas = context.router.export_schemas(ProviderFormat(provider))
|
|
|
|
|
|
@then("the exported schemas should contain {count:d} schema")
|
|
def step_check_exported_count(context: Context, count: int) -> None:
|
|
assert len(context.exported_schemas) == count
|
|
|
|
|
|
@then('the first exported schema should have tool_type "{expected}"')
|
|
def step_check_exported_type(context: Context, expected: str) -> None:
|
|
assert context.exported_schemas[0]["tool_type"] == expected
|
|
|
|
|
|
# -- Sequence counter -------------------------------------------------------
|
|
|
|
|
|
@then("the router sequence should be {expected:d}")
|
|
def step_check_sequence(context: Context, expected: int) -> None:
|
|
assert context.router.sequence == expected
|
|
|
|
|
|
# -- Router construction validation -----------------------------------------
|
|
|
|
|
|
@when("I try to create a router with empty plan_id")
|
|
def step_try_create_router_empty(context: Context) -> None:
|
|
try:
|
|
ToolCallRouter(
|
|
registry=context.registry,
|
|
runner=context.runner,
|
|
plan_id="",
|
|
)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to route a non-dict payload")
|
|
def step_try_route_non_dict(context: Context) -> None:
|
|
try:
|
|
context.router.route("not a dict") # type: ignore[arg-type]
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I try to route_batch a non-list payload")
|
|
def step_try_route_batch_non_list(context: Context) -> None:
|
|
try:
|
|
context.router.route_batch("not a list") # type: ignore[arg-type]
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
# -- Detect format non-dict -------------------------------------------------
|
|
|
|
|
|
@when("I detect format for a non-dict value")
|
|
def step_detect_non_dict(context: Context) -> None:
|
|
context.detected_format = detect_provider_format("not-a-dict") # type: ignore[arg-type]
|
|
|
|
|
|
@when("I classify an empty error message")
|
|
def step_classify_empty_error(context: Context) -> None:
|
|
context.error_category = classify_tool_error("")
|
|
|
|
|
|
@given("a tool call with invalid JSON arguments")
|
|
def step_payload_invalid_json(context: Context) -> None:
|
|
context.payload = {"name": "test/echo", "arguments": "not valid json"}
|
|
|
|
|
|
# -- Coverage: provider branches and validation ---------------------------------
|
|
|
|
|
|
@given('a tool call payload with name "{name}" and dict arguments')
|
|
def step_payload_dict_args(context: Context, name: str) -> None:
|
|
context.payload = {"name": name, "arguments": {"msg": "hello"}}
|
|
|
|
|
|
@given("an OpenAI payload with integer arguments")
|
|
def step_payload_int_args(context: Context) -> None:
|
|
context.payload = {"name": "test/echo", "arguments": 42}
|
|
|
|
|
|
@when("I try to normalize the invalid openai arguments")
|
|
def step_normalize_invalid_openai(context: Context) -> None:
|
|
try:
|
|
normalize_tool_call(context.payload)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@given("an Anthropic payload with string input")
|
|
def step_payload_anthropic_str(context: Context) -> None:
|
|
context.payload = {
|
|
"name": "test/echo",
|
|
"id": "call-1",
|
|
"type": "tool_use",
|
|
"input": "bad",
|
|
}
|
|
|
|
|
|
@when("I try to normalize the invalid anthropic input")
|
|
def step_normalize_invalid_anthropic(context: Context) -> None:
|
|
try:
|
|
normalize_tool_call(context.payload)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@given("a LangChain payload with dict args")
|
|
def step_payload_langchain_dict(context: Context) -> None:
|
|
context.payload = {"name": "test/echo", "args": {"key": "val"}, "id": "lc-1"}
|
|
|
|
|
|
@given("a LangChain payload with list args")
|
|
def step_payload_langchain_list(context: Context) -> None:
|
|
context.payload = {"name": "test/echo", "args": [1, 2, 3], "id": "lc-2"}
|
|
|
|
|
|
@when("I try to normalize the invalid langchain args")
|
|
def step_normalize_invalid_langchain(context: Context) -> None:
|
|
try:
|
|
normalize_tool_call(context.payload)
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@given("an unknown format payload with parameters key")
|
|
def step_payload_unknown_params(context: Context) -> None:
|
|
context.payload = {"name": "test/echo", "parameters": {"query": "hello"}}
|
|
|
|
|
|
@given("an unknown format payload with JSON string in args key")
|
|
def step_payload_unknown_json_str(context: Context) -> None:
|
|
# Use "arguments" with a non-parseable string first then "input" as JSON string
|
|
# to trigger the unknown branch with a JSON string in a fallback key.
|
|
# "arguments" with int makes it not str/dict -> unknown; "input" as JSON string.
|
|
context.payload = {"name": "test/echo", "input": '{"data": "value"}'}
|
|
|
|
|
|
@given("an unknown format payload with bad JSON and valid fallback")
|
|
def step_payload_unknown_bad_json(context: Context) -> None:
|
|
# arguments=42 makes detect_provider_format return UNKNOWN.
|
|
# In normalize unknown branch: "arguments" (int) skipped,
|
|
# "input" (bad JSON str) triggers JSONDecodeError→continue,
|
|
# "parameters" (dict) is used successfully.
|
|
context.payload = {
|
|
"name": "test/echo",
|
|
"arguments": 42,
|
|
"input": "not json",
|
|
"parameters": {"ok": True},
|
|
}
|
|
|
|
|
|
@when('I normalize the tool call for "{fmt}" format')
|
|
def step_normalize_for_format(context: Context, fmt: str) -> None:
|
|
context.normalized_request = normalize_tool_call(context.payload)
|
|
|
|
|
|
@then('the normalized request should have tool name "{name}"')
|
|
def step_then_request_tool_name(context: Context, name: str) -> None:
|
|
assert context.normalized_request.tool_name == name, (
|
|
f"Expected '{name}', got '{context.normalized_request.tool_name}'"
|
|
)
|
|
|
|
|
|
@then('the normalized arguments should be a dict with key "{key}"')
|
|
def step_then_args_dict_key(context: Context, key: str) -> None:
|
|
assert key in context.normalized_request.arguments
|
|
|
|
|
|
@then('the normalized arguments should have key "{key}"')
|
|
def step_then_args_key(context: Context, key: str) -> None:
|
|
assert key in context.normalized_request.arguments
|
|
|
|
|
|
@given('a tool spec named "{name}"')
|
|
def step_given_tool_spec(context: Context, name: str) -> None:
|
|
context.tool_spec = ToolSpec(
|
|
name=name,
|
|
description="test tool",
|
|
input_schema={"type": "object", "properties": {"msg": {"type": "string"}}},
|
|
output_schema={},
|
|
capabilities=ToolCapability(read_only=False, writes=True),
|
|
handler=lambda inputs: inputs,
|
|
)
|
|
|
|
|
|
@then('the schema should have key "{key}"')
|
|
def step_then_schema_key(context: Context, key: str) -> None:
|
|
assert key in context.normalized_schema, (
|
|
f"Missing key '{key}' in {context.normalized_schema}"
|
|
)
|
|
|
|
|
|
@then('the router plan_id should be "{expected}"')
|
|
def step_then_router_plan_id(context: Context, expected: str) -> None:
|
|
assert context.router.plan_id == expected
|
|
|
|
|
|
# -- Tools that raise or fail for coverage tests ---------
|
|
|
|
|
|
@given("a tool registry with a tool that raises RuntimeError")
|
|
def step_registry_runtime_error(context: Context) -> None:
|
|
registry = ToolRegistry()
|
|
|
|
def _noop(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {}
|
|
|
|
registry.register(
|
|
ToolSpec(
|
|
name="test/boom",
|
|
description="exploding tool",
|
|
input_schema={},
|
|
output_schema={},
|
|
capabilities=ToolCapability(read_only=True),
|
|
handler=_noop,
|
|
)
|
|
)
|
|
context.tool_registry = registry
|
|
runner = ToolRunner(registry)
|
|
|
|
# Override execute to raise RuntimeError, bypassing ToolRunner's catch
|
|
def _raising_execute(
|
|
name: str,
|
|
arguments: dict[str, Any],
|
|
*,
|
|
plan_env: str | None = None,
|
|
project_env: str | None = None,
|
|
**_kwargs: Any,
|
|
) -> ToolResult:
|
|
raise RuntimeError("boom")
|
|
|
|
runner.execute = _raising_execute # type: ignore[assignment]
|
|
context.runner = runner
|
|
|
|
|
|
@when("I route an OpenAI payload for the error tool")
|
|
def step_route_error_tool(context: Context) -> None:
|
|
context.route_result = context.router.route(
|
|
{"name": "test/boom", "arguments": "{}"}
|
|
)
|
|
|
|
|
|
@then("the route result should have success false")
|
|
def step_then_result_fail(context: Context) -> None:
|
|
assert context.route_result.result.success is False
|
|
|
|
|
|
@then('the route result error should contain "{text}"')
|
|
def step_then_result_error_contains(context: Context, text: str) -> None:
|
|
assert text in (context.route_result.result.error or "")
|
|
|
|
|
|
@given("a tool registry with a tool that returns failed result")
|
|
def step_registry_fail_result(context: Context) -> None:
|
|
registry = ToolRegistry()
|
|
|
|
def _fail(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"error": "tool not found: missing"}
|
|
|
|
spec = ToolSpec(
|
|
name="test/fail",
|
|
description="failing tool",
|
|
input_schema={},
|
|
output_schema={},
|
|
capabilities=ToolCapability(read_only=True),
|
|
handler=_fail,
|
|
)
|
|
registry.register(spec)
|
|
context.tool_registry = registry
|
|
|
|
runner = ToolRunner(registry)
|
|
|
|
# Override execute to return a ToolResult with success=False
|
|
def _execute_fail(
|
|
name: str,
|
|
arguments: dict[str, Any],
|
|
*,
|
|
plan_env: str | None = None,
|
|
project_env: str | None = None,
|
|
**_kwargs: Any,
|
|
) -> ToolResult:
|
|
return ToolResult(
|
|
success=False,
|
|
output={},
|
|
error="tool not found: missing",
|
|
duration_ms=1.0,
|
|
)
|
|
|
|
runner.execute = _execute_fail # type: ignore[assignment]
|
|
context.runner = runner
|
|
|
|
|
|
@when("I route an OpenAI payload for the fail tool")
|
|
def step_route_fail_tool(context: Context) -> None:
|
|
context.route_result = context.router.route(
|
|
{"name": "test/fail", "arguments": "{}"}
|
|
)
|
|
|
|
|
|
@then("the route result should have error_category set")
|
|
def step_then_error_category_set(context: Context) -> None:
|
|
assert context.route_result.error_category is not None
|
|
|
|
|
|
@given("a tool registry with a validation tool")
|
|
def step_registry_validation_tool(context: Context) -> None:
|
|
registry = ToolRegistry()
|
|
|
|
def _validate(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": True}
|
|
|
|
registry.register(
|
|
ToolSpec(
|
|
name="validate/check",
|
|
description="validate something",
|
|
input_schema={},
|
|
output_schema={"validation_mode": "strict"},
|
|
capabilities=ToolCapability(read_only=True, writes=False),
|
|
handler=_validate,
|
|
)
|
|
)
|
|
context.tool_registry = registry
|
|
context.runner = ToolRunner(registry)
|
|
|
|
|
|
@when("I route an OpenAI payload for the validation tool")
|
|
def step_route_validation_tool(context: Context) -> None:
|
|
context.route_result = context.router.route(
|
|
{"name": "validate/check", "arguments": "{}"}
|
|
)
|
|
|
|
|
|
@then("the route result should have is_validation true")
|
|
def step_then_is_validation(context: Context) -> None:
|
|
assert context.route_result.is_validation is True
|
|
|
|
|
|
@then("the route result should have validation_passed true")
|
|
def step_then_validation_passed(context: Context) -> None:
|
|
assert context.route_result.validation_passed is True
|
|
|
|
|
|
# -- Streaming coverage ----
|
|
|
|
|
|
@when("I stream-route an OpenAI payload for the error tool")
|
|
def step_stream_route_error(context: Context) -> None:
|
|
results = list(
|
|
context.router.route_streaming({"name": "test/boom", "arguments": "{}"})
|
|
)
|
|
context.streaming_results = results
|
|
|
|
|
|
@then("the streaming updates should include ERROR status")
|
|
def step_then_streaming_error_status(context: Context) -> None:
|
|
statuses = [
|
|
r.status
|
|
for r in context.streaming_results
|
|
if isinstance(r, StreamingToolUpdate)
|
|
]
|
|
assert StreamingStatus.ERROR in statuses
|
|
|
|
|
|
@then("the streaming final result should have success false")
|
|
def step_then_streaming_fail(context: Context) -> None:
|
|
final = [
|
|
r for r in context.streaming_results if isinstance(r, NormalizedToolCallResult)
|
|
]
|
|
assert len(final) == 1
|
|
assert final[0].result.success is False
|
|
|
|
|
|
@when("I try to stream-route a non-dict payload")
|
|
def step_try_stream_non_dict(context: Context) -> None:
|
|
try:
|
|
list(context.router.route_streaming("not a dict")) # type: ignore[arg-type]
|
|
context.raised_error = None
|
|
except ValueError as exc:
|
|
context.raised_error = exc
|
|
|
|
|
|
@when("I stream-route an OpenAI payload for the validation tool")
|
|
def step_stream_route_validation(context: Context) -> None:
|
|
results = list(
|
|
context.router.route_streaming({"name": "validate/check", "arguments": "{}"})
|
|
)
|
|
context.streaming_results = results
|
|
|
|
|
|
@then("the streaming final result should have is_validation true")
|
|
def step_then_streaming_validation(context: Context) -> None:
|
|
final = [
|
|
r for r in context.streaming_results if isinstance(r, NormalizedToolCallResult)
|
|
]
|
|
assert len(final) == 1
|
|
assert final[0].is_validation is True
|
|
|
|
|
|
# -- Schema export coverage ----
|
|
|
|
|
|
@when('I export tool schemas for "{provider}"')
|
|
def step_export_schemas(context: Context, provider: str) -> None:
|
|
fmt = ProviderFormat(provider)
|
|
context.exported_schemas = context.router.export_schemas(fmt)
|
|
|
|
|
|
@then('the exported schemas should include a tool with type "{tool_type}"')
|
|
def step_then_schema_tool_type(context: Context, tool_type: str) -> None:
|
|
types = [s.get("tool_type") for s in context.exported_schemas]
|
|
assert tool_type in types, f"Expected type '{tool_type}' in {types}"
|
|
|
|
|
|
@given('a tool registry with a validation tool with mode "{mode}"')
|
|
def step_registry_validation_with_mode(context: Context, mode: str) -> None:
|
|
registry = ToolRegistry()
|
|
|
|
def _validate(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": True}
|
|
|
|
registry.register(
|
|
ToolSpec(
|
|
name="validate/strict",
|
|
description="strict validator",
|
|
input_schema={},
|
|
output_schema={"validation_mode": mode},
|
|
capabilities=ToolCapability(read_only=True, writes=False),
|
|
handler=_validate,
|
|
)
|
|
)
|
|
context.tool_registry = registry
|
|
context.runner = ToolRunner(registry)
|
|
|
|
|
|
@then('the exported validation schema should have mode "{mode}"')
|
|
def step_then_schema_mode(context: Context, mode: str) -> None:
|
|
validation_schemas = [
|
|
s for s in context.exported_schemas if s.get("tool_type") == "validation"
|
|
]
|
|
assert len(validation_schemas) >= 1
|
|
assert validation_schemas[0].get("validation_mode") == mode
|
|
|
|
|
|
@given("a tool registry with a validation tool with numeric mode")
|
|
def step_registry_validation_numeric_mode(context: Context) -> None:
|
|
registry = ToolRegistry()
|
|
|
|
def _validate(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": True}
|
|
|
|
registry.register(
|
|
ToolSpec(
|
|
name="validate/numeric",
|
|
description="numeric mode validator",
|
|
input_schema={},
|
|
output_schema={"validation_mode": 123},
|
|
capabilities=ToolCapability(read_only=True, writes=False),
|
|
handler=_validate,
|
|
)
|
|
)
|
|
context.tool_registry = registry
|
|
context.runner = ToolRunner(registry)
|
|
|
|
|
|
@then("the exported validation schema should not have a mode")
|
|
def step_then_schema_no_mode(context: Context) -> None:
|
|
validation_schemas = [
|
|
s for s in context.exported_schemas if s.get("tool_type") == "validation"
|
|
]
|
|
assert len(validation_schemas) >= 1
|
|
assert "validation_mode" not in validation_schemas[0]
|