fix(reactive): forward actor options block in ToolCallingLLMCaller._resolve_llm #11244

Merged
HAL9000 merged 2 commits from bugfix/m7-tool-calling-llm-options into master 2026-05-28 03:08:32 +00:00
3 changed files with 256 additions and 0 deletions
+33
View File
@@ -272,3 +272,36 @@ Feature: Actor run tool-calling via ToolCallingRuntime
Given a ToolCallingLLMCaller with an actor config without system_prompt
When invoke is called and the LLM returns a response with encoded tool call names
Then the LLMResponse contains the decoded tool calls
# ---------- V: ToolCallingLLMCaller options block forwarding (#11243) ----------
@tdd_issue @tdd_issue_11243
Scenario: ToolCallingLLMCaller forwards openai_api_base and openai_api_key from options block
Given a ToolCallingLLMCaller with actor config options containing openai_api_base and openai_api_key
When _resolve_llm is called on the ToolCallingLLMCaller with empty tool_schemas
Then create_llm was called with openai_api_base forwarded from options
And create_llm was called with __api_key_sentinel extracted from options openai_api_key
@tdd_issue @tdd_issue_11243
Scenario: ToolCallingLLMCaller forwards allowed options keys to create_llm
Given a ToolCallingLLMCaller with actor config options containing allowed extra keys
When _resolve_llm is called on the ToolCallingLLMCaller with empty tool_schemas
Then create_llm was called with the allowed options keys forwarded
@tdd_issue @tdd_issue_11243
Scenario: ToolCallingLLMCaller rejects reserved keys in options block with a warning
Given a ToolCallingLLMCaller with actor config options containing reserved key provider_type
When _resolve_llm is called on the ToolCallingLLMCaller with empty tool_schemas
Then create_llm was NOT called with the reserved key provider_type
@tdd_issue @tdd_issue_11243
Scenario: ToolCallingLLMCaller rejects unknown keys in options block with a warning
Given a ToolCallingLLMCaller with actor config options containing unknown key foo_bar
When _resolve_llm is called on the ToolCallingLLMCaller with empty tool_schemas
Then create_llm was NOT called with the unknown key foo_bar
@tdd_issue @tdd_issue_11243
Scenario: ToolCallingLLMCaller top-level config takes precedence over options block
Given a ToolCallingLLMCaller with actor config with top-level temperature and options temperature
When _resolve_llm is called on the ToolCallingLLMCaller with empty tool_schemas
Then create_llm was called with the top-level temperature value
@@ -1089,3 +1089,178 @@ def step_response_contains_decoded_tool_calls(context: Any) -> None:
assert "server:local/my-tool" in names, (
f"Expected 'server:local/my-tool' in tool call names, got {names}"
)
# ---------------------------------------------------------------------------
# Scenarios V: ToolCallingLLMCaller options block forwarding (#11243)
# ---------------------------------------------------------------------------
@given(
"a ToolCallingLLMCaller with actor config options containing openai_api_base and openai_api_key"
)
def step_caller_with_options_api_base_and_key(context: Any) -> None:
context.options_caller = ToolCallingLLMCaller(
actor_config={
"provider": "openai",
"model": "gpt-4",
"options": {
"openai_api_base": "http://localhost:8080/v1",
"openai_api_key": "none",
},
}
)
context.options_mock_llm = MagicMock()
context.options_mock_llm.bind_tools.return_value = context.options_mock_llm
@when("_resolve_llm is called on the ToolCallingLLMCaller with empty tool_schemas")
def step_resolve_llm_options_empty_schemas(context: Any) -> None:
with patch("cleveragents.reactive.tool_caller.get_provider_registry") as mock_reg:
mock_reg.return_value.create_llm.return_value = context.options_mock_llm
context.options_caller._resolve_llm([])
context.options_create_llm_kwargs = (
mock_reg.return_value.create_llm.call_args.kwargs
)
@then("create_llm was called with openai_api_base forwarded from options")
def step_create_llm_called_with_api_base(context: Any) -> None:
kwargs = context.options_create_llm_kwargs
assert "openai_api_base" in kwargs, (
f"Expected 'openai_api_base' in create_llm kwargs, got {kwargs}"
)
assert kwargs["openai_api_base"] == "http://localhost:8080/v1", (
f"Expected openai_api_base='http://localhost:8080/v1', got {kwargs['openai_api_base']}"
)
@then(
"create_llm was called with __api_key_sentinel extracted from options openai_api_key"
)
def step_create_llm_called_with_api_key_sentinel(context: Any) -> None:
kwargs = context.options_create_llm_kwargs
assert kwargs.get("__api_key_sentinel") == "none", (
f"Expected __api_key_sentinel='none' from options.openai_api_key, got {kwargs}"
)
assert "openai_api_key" not in kwargs, (
"openai_api_key must be extracted to __api_key_sentinel, not passed directly"
)
@given("a ToolCallingLLMCaller with actor config options containing allowed extra keys")
def step_caller_with_allowed_options_keys(context: Any) -> None:
context.options_caller = ToolCallingLLMCaller(
actor_config={
"provider": "openai",
"model": "gpt-4",
"options": {
"timeout": 30,
"top_p": 0.9,
"frequency_penalty": 0.1,
"presence_penalty": 0.2,
},
}
)
context.options_mock_llm = MagicMock()
context.options_mock_llm.bind_tools.return_value = context.options_mock_llm
@then("create_llm was called with the allowed options keys forwarded")
def step_create_llm_called_with_allowed_keys(context: Any) -> None:
kwargs = context.options_create_llm_kwargs
assert kwargs.get("timeout") == 30, (
f"Expected timeout=30, got {kwargs.get('timeout')}"
)
assert kwargs.get("top_p") == 0.9, f"Expected top_p=0.9, got {kwargs.get('top_p')}"
assert kwargs.get("frequency_penalty") == 0.1, (
f"Expected frequency_penalty=0.1, got {kwargs.get('frequency_penalty')}"
)
assert kwargs.get("presence_penalty") == 0.2, (
f"Expected presence_penalty=0.2, got {kwargs.get('presence_penalty')}"
)
@given(
"a ToolCallingLLMCaller with actor config options containing reserved key provider_type"
)
def step_caller_with_reserved_key(context: Any) -> None:
context.options_caller = ToolCallingLLMCaller(
actor_config={
"provider": "openai",
"model": "gpt-4",
"options": {
"provider_type": "should-be-rejected",
},
}
)
context.options_mock_llm = MagicMock()
context.options_mock_llm.bind_tools.return_value = context.options_mock_llm
@then("create_llm was NOT called with the reserved key provider_type")
def step_create_llm_not_called_with_reserved(context: Any) -> None:
kwargs = context.options_create_llm_kwargs
# create_llm is always called with provider_type (the actor's real provider).
# What must NOT happen is the options value ("should-be-rejected") overriding it.
# The call must have succeeded without a TypeError from duplicate keyword arguments,
# and the provider_type must be the actor's real value, not the options value.
assert context.options_create_llm_kwargs is not None, (
"_resolve_llm must succeed even when options contains reserved keys"
)
assert kwargs.get("provider_type") != "should-be-rejected", (
f"Reserved key 'provider_type' from options must not override the actor provider; "
f"got provider_type={kwargs.get('provider_type')!r}"
)
@given(
"a ToolCallingLLMCaller with actor config options containing unknown key foo_bar"
)
def step_caller_with_unknown_key(context: Any) -> None:
context.options_caller = ToolCallingLLMCaller(
actor_config={
"provider": "openai",
"model": "gpt-4",
"options": {
"foo_bar": "should-be-rejected",
},
}
)
context.options_mock_llm = MagicMock()
context.options_mock_llm.bind_tools.return_value = context.options_mock_llm
@then("create_llm was NOT called with the unknown key foo_bar")
def step_create_llm_not_called_with_unknown(context: Any) -> None:
kwargs = context.options_create_llm_kwargs
assert "foo_bar" not in kwargs, (
f"Unknown key 'foo_bar' must not be forwarded to create_llm, got {kwargs}"
)
@given(
"a ToolCallingLLMCaller with actor config with top-level temperature and options temperature"
)
def step_caller_top_level_temperature_vs_options(context: Any) -> None:
context.options_caller = ToolCallingLLMCaller(
actor_config={
"provider": "openai",
"model": "gpt-4",
"temperature": 0.5, # top-level value — must take precedence
"options": {
"temperature": 0.99, # options value — must be ignored
},
}
)
context.options_mock_llm = MagicMock()
context.options_mock_llm.bind_tools.return_value = context.options_mock_llm
@then("create_llm was called with the top-level temperature value")
def step_create_llm_called_with_top_level_temp(context: Any) -> None:
kwargs = context.options_create_llm_kwargs
assert kwargs.get("temperature") == 0.5, (
f"Expected top-level temperature=0.5 to take precedence over options, "
f"got temperature={kwargs.get('temperature')}"
)
+48
View File
@@ -138,6 +138,54 @@ class ToolCallingLLMCaller:
if max_retries is not None:
llm_kwargs["max_retries"] = max_retries
# M7 (#11243): merge options block so custom LLM backend kwargs
# (e.g. openai_api_base, openai_api_key) are forwarded to the
# LLM constructor — mirrors the fix applied to SimpleLLMAgent in
# stream_router.py (PR #11225 / commit b3851693).
#
# Options are applied after the fixed keys so that explicit
# top-level keys (temperature, max_tokens, etc.) take precedence
# over any duplicate in the options block.
#
# openai_api_key is handled specially: the registry uses a
# __api_key_sentinel mechanism to distinguish explicitly provided
# keys from environment-sourced ones. If the user supplies
# openai_api_key in options, we extract it and inject it via the
# sentinel so it overrides the registry's default.
_ALLOWED_OPTIONS: frozenset[str] = frozenset(
{
"openai_api_base",
"temperature",
"max_tokens",
"timeout",
"top_p",
"frequency_penalty",
"presence_penalty",
}
)
_RESERVED: frozenset[str] = frozenset({"provider_type", "model_id"})
options = dict(self._actor_config.get("options") or {})
if "openai_api_key" in options:
llm_kwargs["__api_key_sentinel"] = options.pop("openai_api_key")
for key, value in options.items():
if key in _RESERVED:
logger.warning(
"Actor options block contains reserved key '%s' "
"that conflicts with positional arguments; ignoring.",
key,
)
continue
if key not in llm_kwargs:
if key in _ALLOWED_OPTIONS:
llm_kwargs[key] = value
else:
logger.warning(
"Actor options block contains unrecognized key '%s'; "
"ignoring for security. Allowed keys: %s.",
key,
sorted(_ALLOWED_OPTIONS),
)
registry = get_provider_registry()
# Annotated as Any because create_llm() returns BaseLanguageModel but the
# real runtime object is BaseChatModel which has bind_tools().