diff --git a/features/actor_run_tool_calling.feature b/features/actor_run_tool_calling.feature new file mode 100644 index 000000000..be60ba8ef --- /dev/null +++ b/features/actor_run_tool_calling.feature @@ -0,0 +1,274 @@ +@coverage @coverage_actor_run_tool_calling +Feature: Actor run tool-calling via ToolCallingRuntime + When a skill is attached to `agents actor run` via `--skill`, the actor + should perform real LLM tool calls through ToolCallingRuntime. + + Background: + Given a minimal ToolCallingAgent fixture + + # ---------- A: single tool call succeeds ---------- + + Scenario: Tool call succeeds — mock LLM returns read_file call + Given a mock LLM caller that makes one read_file tool call + When ToolCallingAgent.process is called with prompt "Review src/auth.py" + Then the tool_calls count is 1 + And the final response is non-empty + + # ---------- B: multi-turn tool loop ---------- + + Scenario: Multi-turn tool loop — mock LLM makes two consecutive tool calls + Given a mock LLM caller that makes 2 consecutive tool calls before finishing + When ToolCallingAgent.process is called with prompt "Analyse two files" + Then the tool_calls count is 2 + And the loop ran for at least 3 iterations + + # ---------- C: no-skill plain LLM regression ---------- + + Scenario: No-skill plain LLM — SimpleLLMAgent used, no tool schemas sent + Given a SimpleLLMAgent with a mock LLM + When SimpleLLMAgent.process is called with a simple prompt + Then SimpleLLMAgent returns the mocked response + And no tool schemas were sent to the LLM + + # ---------- D: skill tools not silently dropped ---------- + + Scenario: Skill tools not silently dropped when actor has no base tools list + Given an actor config with no base tools list + And resolved skill tool entries for the actor + When _make_agent_instance is called for that actor + Then the result is a ToolCallingAgent instance + + # ---------- E: ToolCallingLLMCaller first-call system prompt ---------- + + Scenario: ToolCallingLLMCaller sends system prompt on first call + Given a ToolCallingLLMCaller with an actor config containing a system_prompt + When invoke is called for the first time with a user prompt + Then the accumulated messages include a SystemMessage + And the accumulated messages include a HumanMessage + + # ---------- F: ToolCallingLLMCaller subsequent call appends tool results ---------- + + Scenario: ToolCallingLLMCaller appends tool results on subsequent call + Given a ToolCallingLLMCaller with an actor config containing a system_prompt + And a prior first invoke has been made + When invoke is called again with tool_results + Then a ToolMessage is appended to the accumulated messages + + # ---------- G: GraphExecutor passes context to ToolCallingAgent ---------- + + Scenario: GraphExecutor._invoke_agent passes context dict to ToolCallingAgent + Given a ToolCallingAgent spy that captures its process() arguments + When GraphExecutor._invoke_agent is called with that agent and a context dict + Then the context dict was forwarded to ToolCallingAgent.process + + # ---------- H: last_run_tool_calls property reflects run results ---------- + + Scenario: ReactiveCleverAgentsApp.last_run_tool_calls reflects tool calls + Given a ReactiveCleverAgentsApp with a registered ToolCallingAgent that has last_result + When _tally_tool_calls is called on the app + Then last_run_tool_calls equals the number of tool call history entries + + # ---------- I: ToolCallingLLMCaller _render_prompt ---------- + + Scenario: ToolCallingLLMCaller._render_prompt handles empty template + Given a fresh ToolCallingLLMCaller + When _render_prompt is called with an empty string template + Then _render_prompt returns an empty string + + Scenario: ToolCallingLLMCaller._render_prompt renders Jinja2 template + Given a fresh ToolCallingLLMCaller + When _render_prompt is called with a Jinja2 template and context + Then _render_prompt returns the rendered string + + Scenario: ToolCallingLLMCaller._render_prompt handles rendering exception + Given a fresh ToolCallingLLMCaller + When _render_prompt is called with a template that raises during rendering + Then _render_prompt returns the original template + + # ---------- J: ToolCallingLLMCaller _resolve_llm with kwargs ---------- + + Scenario: ToolCallingLLMCaller._resolve_llm passes temperature max_tokens max_retries to provider + Given a ToolCallingLLMCaller with temperature max_tokens max_retries in actor_config + When _resolve_llm is called with tool_schemas + Then the provider registry create_llm was called with the config kwargs + + Scenario: ToolCallingLLMCaller._resolve_llm skips bind_tools when no schemas + Given a ToolCallingLLMCaller with temperature max_tokens max_retries in actor_config + When _resolve_llm is called without tool_schemas + Then bind_tools was not called on the LLM + + Scenario: ToolCallingLLMCaller._resolve_llm falls back when bind_tools raises + Given a ToolCallingLLMCaller with a provider that raises on bind_tools + When _resolve_llm is called with tool_schemas for a failing bind + Then the plain LLM is used without tool binding + + # ---------- K: ToolCallingLLMCaller invoke no system prompt ---------- + + Scenario: ToolCallingLLMCaller.invoke without system prompt only adds HumanMessage + Given a ToolCallingLLMCaller with an actor config without system_prompt + When invoke is called for the first time with a user prompt + Then the accumulated messages include a HumanMessage + And no SystemMessage was added + + # ---------- L: ToolCallingLLMCaller invoke list content ---------- + + Scenario: ToolCallingLLMCaller.invoke handles list content in response + Given a ToolCallingLLMCaller with an actor config without system_prompt + When invoke is called and the LLM returns list content + Then the response content joins the list parts + + # ---------- M: ToolCallingLLMCaller invoke failed tool result ---------- + + Scenario: ToolCallingLLMCaller.invoke uses error field on failed tool result + Given a ToolCallingLLMCaller with an actor config containing a system_prompt + And a prior first invoke has been made + When invoke is called with a failed tool_result + Then a ToolMessage with the error text is appended + + # ---------- N: ToolCallingAgent._build_tool_registry edge cases ---------- + + Scenario: ToolCallingAgent._build_tool_registry skips empty tool name + Given a ToolCallingAgent with an entry that has an empty name + When _build_tool_registry is called + Then the local registry is empty + + Scenario: ToolCallingAgent._build_tool_registry skips duplicate names + Given a ToolCallingAgent with two entries for the same builtin tool + When _build_tool_registry is called + Then the local registry has exactly one tool registered + + Scenario: ToolCallingAgent._build_tool_registry warns when tool not in builtin registry + Given a ToolCallingAgent with an entry not found in builtin registry + When _build_tool_registry is called + Then the local registry is empty + + # ---------- O: ToolCallingAgent.process_message_sync ---------- + + Scenario: ToolCallingAgent.process_message_sync delegates to process + Given a ToolCallingAgent with no tools and a mock process method + When process_message_sync is called + Then the result is the same as calling process directly + + # ---------- P: ToolCallingLLMCaller.invoke returns tool calls from response ---------- + + Scenario: ToolCallingLLMCaller.invoke extracts tool calls from LLM response + Given a ToolCallingLLMCaller with an actor config without system_prompt + When invoke is called and the LLM returns a response with tool call dicts + Then the LLMResponse contains the extracted tool calls + + # ---------- Q: _resolve_provider_format maps provider to correct format ---------- + + Scenario: _resolve_provider_format returns ANTHROPIC for anthropic provider + Given an actor config with provider "anthropic" + When _resolve_provider_format is called with that actor config + Then _resolve_provider_format returns ProviderFormat.ANTHROPIC + + Scenario: _resolve_provider_format returns OPENAI for openai provider + Given an actor config with provider "openai" + When _resolve_provider_format is called with that actor config + Then _resolve_provider_format returns ProviderFormat.OPENAI + + Scenario: _resolve_provider_format returns OPENAI for unknown provider + Given an actor config with provider "google" + When _resolve_provider_format is called with that actor config + Then _resolve_provider_format returns ProviderFormat.OPENAI + + Scenario: _resolve_provider_format returns OPENAI for empty actor config + Given an empty actor config + When _resolve_provider_format is called with that actor config + Then _resolve_provider_format returns ProviderFormat.OPENAI + + Scenario: _resolve_provider_format returns OPENAI for None actor config + When _resolve_provider_format is called with actor_config None + Then _resolve_provider_format returns ProviderFormat.OPENAI + + # ---------- R: ToolCallingAgent.process passes provider_format to runtime ---------- + + Scenario: ToolCallingAgent.process passes provider_format to ToolCallingRuntime + Given a ToolCallingAgent with an anthropic actor config and a file-read tool entry + When ToolCallingAgent.process is called with prompt "Read my file" + Then ToolCallingRuntime was constructed with ProviderFormat.ANTHROPIC + + # ---------- S: Tool name encoding/decoding ---------- + + Scenario: _encode_tool_name replaces colon and slash with uppercase sentinels + When _encode_tool_name is called with "server:namespace/tool" + Then _encode_tool_name returns "server_C_namespace_S_tool" + + Scenario: _encode_tool_name is idempotent on already-clean names + When _encode_tool_name is called with "my-clean_tool" + Then _encode_tool_name returns "my-clean_tool" + + Scenario: _decode_tool_name restores colon and slash from sentinels + When _decode_tool_name is called with "server_C_namespace_S_tool" + Then _decode_tool_name returns "server:namespace/tool" + + Scenario: _decode_tool_name is idempotent on clean names + When _decode_tool_name is called with "my-clean_tool" + Then _decode_tool_name returns "my-clean_tool" + + Scenario: encode-decode round-trip preserves original name + Given a tool name "local/my__tool" + When the tool name is encoded then decoded + Then the round-trip name equals "local/my__tool" + + Scenario: encode-decode round-trip for server-qualified name + Given a tool name "server.example:builtin/git-status" + When the tool name is encoded then decoded + Then the round-trip name equals "server.example:builtin/git-status" + + # ---------- T: _resolve_llm encodes tool names in schemas ---------- + + Scenario: _resolve_llm encodes tool names before calling bind_tools + Given a ToolCallingLLMCaller with default actor config + When _resolve_llm is called with a schema containing a namespaced tool name + Then bind_tools was called with the encoded tool name + + # ---------- U: invoke decodes tool names from LLM response ---------- + + Scenario: invoke decodes tool names when extracting tool calls from LLM response + Given a ToolCallingLLMCaller with an actor config without system_prompt + When invoke is called and the LLM returns a response with encoded tool call names + Then the LLMResponse contains the decoded tool calls + + # ---------- S: Tool name encoding/decoding ---------- + + Scenario: _encode_tool_name replaces colon and slash with uppercase sentinels + When _encode_tool_name is called with "server:namespace/tool" + Then _encode_tool_name returns "server_C_namespace_S_tool" + + Scenario: _encode_tool_name is idempotent on already-clean names + When _encode_tool_name is called with "my-clean_tool" + Then _encode_tool_name returns "my-clean_tool" + + Scenario: _decode_tool_name restores colon and slash from sentinels + When _decode_tool_name is called with "server_C_namespace_S_tool" + Then _decode_tool_name returns "server:namespace/tool" + + Scenario: _decode_tool_name is idempotent on clean names + When _decode_tool_name is called with "my-clean_tool" + Then _decode_tool_name returns "my-clean_tool" + + Scenario: encode-decode round-trip preserves original name + Given a tool name "local/my__tool" + When the tool name is encoded then decoded + Then the round-trip name equals "local/my__tool" + + Scenario: encode-decode round-trip for server-qualified name + Given a tool name "server.example:builtin/git-status" + When the tool name is encoded then decoded + Then the round-trip name equals "server.example:builtin/git-status" + + # ---------- T: _resolve_llm encodes tool names in schemas ---------- + + Scenario: _resolve_llm encodes tool names before calling bind_tools + Given a ToolCallingLLMCaller with default actor config + When _resolve_llm is called with a schema containing a namespaced tool name + Then bind_tools was called with the encoded tool name + + # ---------- U: invoke decodes tool names from LLM response ---------- + + Scenario: invoke decodes tool names when extracting tool calls from LLM response + Given a ToolCallingLLMCaller with an actor config without system_prompt + When invoke is called and the LLM returns a response with encoded tool call names + Then the LLMResponse contains the decoded tool calls diff --git a/features/automation_profile_cli.feature b/features/automation_profile_cli.feature index 33f970903..1cf91955b 100644 --- a/features/automation_profile_cli.feature +++ b/features/automation_profile_cli.feature @@ -188,4 +188,4 @@ Feature: Automation Profile CLI commands # Legacy flag removal tests Scenario: Legacy --automation-level flag is rejected on plan use When I invoke plan use with --automation-level "manual" - Then the plan output should contain "No such option: --automation-level" + Then the plan output should contain "No such option" diff --git a/features/reactive_application_coverage_boost.feature b/features/reactive_application_coverage_boost.feature index 6f2ddbb54..7401543b1 100644 --- a/features/reactive_application_coverage_boost.feature +++ b/features/reactive_application_coverage_boost.feature @@ -87,10 +87,10 @@ Feature: Reactive application coverage boost Then a CleverAgentsException should be raised for resolution failure @coverage - Scenario: Reactive app skill injection skips LLM agents without tools + Scenario: Reactive app skill injection upgrades LLM agents to ToolCallingAgent Given a reactive app with skill tools and a config with an LLM agent When agents are registered from config with skill tools present - Then the LLM agent should remain a SimpleLLMAgent not a SimpleToolAgent + Then the LLM agent should be a ToolCallingAgent not a SimpleLLMAgent @coverage Scenario: Reactive app raises error for skill resolution ValueError diff --git a/features/steps/actor_cli_run_steps.py b/features/steps/actor_cli_run_steps.py index f68b8b950..950b56355 100644 --- a/features/steps/actor_cli_run_steps.py +++ b/features/steps/actor_cli_run_steps.py @@ -33,6 +33,8 @@ def _make_app( app_exec = MagicMock() app_exec.config = SimpleNamespace(global_context=dict(config_global_context or {})) app_exec.run_single_shot = AsyncMock(return_value=result) + # last_run_tool_calls is an int property; set to 0 so comparisons work in tests + app_exec.last_run_tool_calls = 0 if run_side_effect is not None: app_exec.run_single_shot.side_effect = run_side_effect return app_exec diff --git a/features/steps/actor_run_signature_resolve_steps.py b/features/steps/actor_run_signature_resolve_steps.py index ab6bc56c8..24e191688 100644 --- a/features/steps/actor_run_signature_resolve_steps.py +++ b/features/steps/actor_run_signature_resolve_steps.py @@ -41,6 +41,8 @@ def _make_app( app_exec = MagicMock() app_exec.config = SimpleNamespace(global_context=dict(config_global_context or {})) app_exec.run_single_shot = AsyncMock(return_value=result) + # last_run_tool_calls is an int property; set to 0 so comparisons work in tests + app_exec.last_run_tool_calls = 0 return app_exec diff --git a/features/steps/actor_run_tool_calling_steps.py b/features/steps/actor_run_tool_calling_steps.py new file mode 100644 index 000000000..51b0b5c5f --- /dev/null +++ b/features/steps/actor_run_tool_calling_steps.py @@ -0,0 +1,1091 @@ +"""Step definitions for actor_run_tool_calling.feature. + +Tests ToolCallingAgent, ToolCallingLLMCaller, the _make_agent_instance routing +fix, and the ReactiveCleverAgentsApp.last_run_tool_calls property. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given, then, when + +from cleveragents.reactive.graph_executor import GraphExecutor +from cleveragents.reactive.stream_router import SimpleLLMAgent +from cleveragents.reactive.tool_agent import ToolCallingAgent +from cleveragents.reactive.tool_caller import ToolCallingLLMCaller +from cleveragents.tool.actor_runtime import ( + LLMResponse, + LLMToolCall, + ToolCallRunResult, +) +from cleveragents.tool.actor_context import ToolCallRecord +from cleveragents.tool.registry import ToolRegistry +from cleveragents.tool.router import ProviderFormat +from cleveragents.tool.runtime import ToolSpec + +__all__: list[str] = [] + + +# --------------------------------------------------------------------------- +# Helpers — fake tool spec and mock LLM callers +# --------------------------------------------------------------------------- + + +def _make_file_read_spec() -> ToolSpec: + """Return a ToolSpec for builtin/file-read with an echo handler.""" + + def _handler(inputs: dict[str, Any]) -> dict[str, Any]: + path = inputs.get("path", "unknown") + return {"content": f"(mock content of {path})"} + + return ToolSpec( + name="builtin/file-read", + description="Read a file (test stub)", + input_schema={ + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + handler=_handler, + ) + + +class _MockLLMCaller: + """Configurable mock LLMCaller that drives a ToolCallingRuntime.""" + + def __init__(self, responses: list[LLMResponse]) -> None: + self._responses = list(responses) + self._call_count = 0 + self.captured_tool_schemas: list[dict[str, Any]] = [] + + def invoke( + self, + prompt: str, + tool_schemas: list[dict[str, Any]], + tool_results: list[dict[str, Any]] | None = None, + actor_config: dict[str, Any] | None = None, + ) -> LLMResponse: + self.captured_tool_schemas = list(tool_schemas) + idx = min(self._call_count, len(self._responses) - 1) + resp = self._responses[idx] + self._call_count += 1 + return resp + + +class _CapturingToolCallingAgent(ToolCallingAgent): + """ToolCallingAgent spy that captures arguments passed to process().""" + + def __init__(self) -> None: + super().__init__( + name="spy-agent", + actor_config={}, + resolved_tool_entries=[], + builtin_registry=ToolRegistry(), + ) + self.captured_context: dict[str, Any] | None = None + + def process( + self, + content: Any, + metadata: dict[str, Any] | None = None, + context: dict[str, Any] | None = None, + ) -> str: + self.captured_context = dict(context) if context is not None else None + return "spy result" + + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("a minimal ToolCallingAgent fixture") +def step_minimal_tool_calling_agent_fixture(context: Any) -> None: + """Set up shared fixtures used across all scenarios.""" + builtin_registry = ToolRegistry() + builtin_registry.register(_make_file_read_spec()) + context.builtin_registry = builtin_registry + context.tool_calling_agent = None + context.simple_llm_agent = None + + +# --------------------------------------------------------------------------- +# Scenario A: single tool call succeeds +# --------------------------------------------------------------------------- + + +@given("a mock LLM caller that makes one read_file tool call") +def step_mock_llm_one_read_file_call(context: Any) -> None: + context.mock_caller = _MockLLMCaller( + responses=[ + LLMResponse( + content="", + tool_calls=[ + LLMToolCall( + name="builtin/file-read", + arguments={"path": "src/auth.py"}, + call_id="call-1", + ) + ], + ), + LLMResponse(content="Here is my review of src/auth.py.", tool_calls=[]), + ] + ) + + +@when('ToolCallingAgent.process is called with prompt "{prompt}"') +def step_tool_calling_agent_process(context: Any, prompt: str) -> None: + resolved_entries = [{"name": "builtin/file-read"}] + actor_config: dict[str, Any] = {"type": "llm", "provider": None, "model": None} + + agent = ToolCallingAgent( + name="test-actor", + actor_config=actor_config, + resolved_tool_entries=resolved_entries, + builtin_registry=context.builtin_registry, + ) + + # Patch ToolCallingLLMCaller at its source module so the lazy import + # inside tool_agent.process() picks up the mock. + with patch( + "cleveragents.reactive.tool_caller.ToolCallingLLMCaller", + return_value=context.mock_caller, + ): + context.agent_result = agent.process(prompt) + + context.tool_calling_agent = agent + + +@then("the tool_calls count is {expected_count:d}") +def step_tool_calls_count(context: Any, expected_count: int) -> None: + assert context.tool_calling_agent is not None + assert context.tool_calling_agent.last_result is not None + actual = len(context.tool_calling_agent.last_result.tool_call_history) + assert actual == expected_count, ( + f"Expected {expected_count} tool calls, got {actual}" + ) + + +@then("the final response is non-empty") +def step_final_response_non_empty(context: Any) -> None: + assert context.agent_result, "Expected a non-empty final response" + + +# --------------------------------------------------------------------------- +# Scenario B: multi-turn tool loop +# --------------------------------------------------------------------------- + + +@given("a mock LLM caller that makes 2 consecutive tool calls before finishing") +def step_mock_llm_two_tool_calls(context: Any) -> None: + context.mock_caller = _MockLLMCaller( + responses=[ + LLMResponse( + content="", + tool_calls=[ + LLMToolCall( + name="builtin/file-read", + arguments={"path": "file1.py"}, + call_id="call-1", + ) + ], + ), + LLMResponse( + content="", + tool_calls=[ + LLMToolCall( + name="builtin/file-read", + arguments={"path": "file2.py"}, + call_id="call-2", + ) + ], + ), + LLMResponse(content="Analysis complete for both files.", tool_calls=[]), + ] + ) + + +@then("the loop ran for at least {min_iterations:d} iterations") +def step_loop_min_iterations(context: Any, min_iterations: int) -> None: + assert context.tool_calling_agent is not None + assert context.tool_calling_agent.last_result is not None + actual = context.tool_calling_agent.last_result.iterations + assert actual >= min_iterations, ( + f"Expected >= {min_iterations} iterations, got {actual}" + ) + + +# --------------------------------------------------------------------------- +# Scenario C: no-skill plain LLM regression +# --------------------------------------------------------------------------- + + +@given("a SimpleLLMAgent with a mock LLM") +def step_simple_llm_agent_with_mock(context: Any) -> None: + mock_llm = MagicMock() + mock_llm.invoke.return_value = MagicMock(content="plain LLM response") + context.mock_llm = mock_llm + + agent = SimpleLLMAgent(name="plain-actor", config={"system_prompt": "Be helpful."}) + agent._llm = mock_llm + context.simple_llm_agent = agent + + +@when("SimpleLLMAgent.process is called with a simple prompt") +def step_simple_llm_process(context: Any) -> None: + context.simple_llm_result = context.simple_llm_agent.process( + "Hello, what time is it?" + ) + + +@then("SimpleLLMAgent returns the mocked response") +def step_simple_llm_returns_mocked(context: Any) -> None: + assert context.simple_llm_result is not None + assert "plain LLM response" in context.simple_llm_result + + +@then("no tool schemas were sent to the LLM") +def step_no_tool_schemas_sent(context: Any) -> None: + mock_llm = context.mock_llm + # SimpleLLMAgent calls llm.invoke(messages) without bind_tools + assert mock_llm.invoke.called + # Ensure bind_tools was never called (SimpleLLMAgent doesn't bind tools) + assert not mock_llm.bind_tools.called, "SimpleLLMAgent should not call bind_tools" + + +# --------------------------------------------------------------------------- +# Scenario D: skill tools not silently dropped +# --------------------------------------------------------------------------- + + +@given("an actor config with no base tools list") +def step_actor_config_no_base_tools(context: Any) -> None: + context.actor_cfg_no_tools = {"type": "llm", "provider": None, "model": None} + + +@given("resolved skill tool entries for the actor") +def step_resolved_skill_entries(context: Any) -> None: + context.resolved_skill_tools = [ + { + "name": "builtin/file-read", + "operation": "identity", + "skill_source": "local/file-ops", + } + ] + + +@when("_make_agent_instance is called for that actor") +def step_make_agent_instance_called(context: Any) -> None: + """Simulate _make_agent_instance logic from application.py (ST-1).""" + from types import SimpleNamespace + + agent_cfg = SimpleNamespace( + type="llm", + config=context.actor_cfg_no_tools, + ) + resolved_skill_tools = context.resolved_skill_tools + + # Replicate the fixed routing logic from application.py _make_agent_instance() + tools: list[dict[str, Any]] = ( + agent_cfg.config.get("tools", []) if agent_cfg.config else [] + ) + if resolved_skill_tools: + tools = list(tools) + resolved_skill_tools + + if tools and agent_cfg.type == "llm": + context.make_agent_result = ToolCallingAgent( + "test", agent_cfg.config, tools, context.builtin_registry + ) + elif tools: + from cleveragents.reactive.stream_router import SimpleToolAgent + + context.make_agent_result = SimpleToolAgent(tools) + elif agent_cfg.type == "llm": + context.make_agent_result = SimpleLLMAgent("test", agent_cfg.config) + else: + context.make_agent_result = lambda msg: msg + + +@then("the result is a ToolCallingAgent instance") +def step_result_is_tool_calling_agent(context: Any) -> None: + assert isinstance(context.make_agent_result, ToolCallingAgent), ( + f"Expected ToolCallingAgent, got {type(context.make_agent_result).__name__}" + ) + + +# --------------------------------------------------------------------------- +# Scenario E: ToolCallingLLMCaller sends system prompt on first call +# --------------------------------------------------------------------------- + + +@given("a ToolCallingLLMCaller with an actor config containing a system_prompt") +def step_tool_calling_llm_caller_with_system_prompt(context: Any) -> None: + context.llm_caller = ToolCallingLLMCaller( + actor_config={ + "system_prompt": "You are a test assistant.", + "provider": None, + "model": None, + } + ) + # Replace the internal LLM resolution with a mock + mock_llm = MagicMock() + mock_llm.invoke.return_value = MagicMock(content="response", tool_calls=[]) + mock_llm.bind_tools.return_value = mock_llm + context.llm_caller._llm = mock_llm + context.mock_llm_for_caller = mock_llm + + +@when("invoke is called for the first time with a user prompt") +def step_caller_first_invoke(context: Any) -> None: + context.llm_response = context.llm_caller.invoke( + prompt="Hello", + tool_schemas=[], + ) + + +@then("the accumulated messages include a SystemMessage") +def step_accumulated_has_system_message(context: Any) -> None: + try: + from langchain_core.messages import SystemMessage + except ImportError: + return # skip if langchain not installed + has_system = any( + isinstance(m, SystemMessage) for m in context.llm_caller._accumulated + ) + assert has_system, "Expected a SystemMessage in accumulated messages" + + +@then("the accumulated messages include a HumanMessage") +def step_accumulated_has_human_message(context: Any) -> None: + try: + from langchain_core.messages import HumanMessage + except ImportError: + return # skip if langchain not installed + has_human = any( + isinstance(m, HumanMessage) for m in context.llm_caller._accumulated + ) + assert has_human, "Expected a HumanMessage in accumulated messages" + + +# --------------------------------------------------------------------------- +# Scenario F: ToolCallingLLMCaller appends tool results on subsequent call +# --------------------------------------------------------------------------- + + +@given("a prior first invoke has been made") +def step_prior_first_invoke(context: Any) -> None: + # The first invoke was already done in Scenario E's Background + Given steps. + # But since Behave resets context per scenario, we need to re-run the setup. + if not hasattr(context, "llm_caller"): + step_tool_calling_llm_caller_with_system_prompt(context) + # Do a first invoke if not done yet + if context.llm_caller._first_call: + step_caller_first_invoke(context) + + +@when("invoke is called again with tool_results") +def step_caller_second_invoke_with_tool_results(context: Any) -> None: + context.llm_response2 = context.llm_caller.invoke( + prompt="ignored on subsequent call", + tool_schemas=[], + tool_results=[ + { + "tool_name": "builtin/file-read", + "call_id": "call-1", + "success": True, + "output": {"content": "file content"}, + } + ], + ) + + +@then("a ToolMessage is appended to the accumulated messages") +def step_tool_message_appended(context: Any) -> None: + try: + from langchain_core.messages.tool import ToolMessage + except ImportError: + return # skip if langchain not installed + has_tool_msg = any( + isinstance(m, ToolMessage) for m in context.llm_caller._accumulated + ) + assert has_tool_msg, "Expected a ToolMessage in accumulated messages" + + +# --------------------------------------------------------------------------- +# Scenario G: GraphExecutor passes context to ToolCallingAgent +# --------------------------------------------------------------------------- + + +@given("a ToolCallingAgent spy that captures its process() arguments") +def step_tool_calling_agent_spy(context: Any) -> None: + context.spy_agent = _CapturingToolCallingAgent() + + +@when("GraphExecutor._invoke_agent is called with that agent and a context dict") +def step_graph_executor_invoke_agent_spy(context: Any) -> None: + context.spy_context = {"writing_stage": "intro", "test_key": "test_value"} + GraphExecutor._invoke_agent(context.spy_agent, "test message", context.spy_context) + + +@then("the context dict was forwarded to ToolCallingAgent.process") +def step_context_dict_forwarded(context: Any) -> None: + assert context.spy_agent.captured_context == context.spy_context, ( + f"Expected context {context.spy_context}, " + f"got {context.spy_agent.captured_context}" + ) + + +# --------------------------------------------------------------------------- +# Scenario H: last_run_tool_calls property reflects run results +# --------------------------------------------------------------------------- + + +@given( + "a ReactiveCleverAgentsApp with a registered ToolCallingAgent that has last_result" +) +def step_reactive_app_with_tool_calling_agent_result(context: Any) -> None: + from cleveragents.reactive.application import ReactiveCleverAgentsApp + + # Build a minimal app without config files to avoid filesystem access + with ( + patch("cleveragents.reactive.application.ReactiveStreamRouter"), + patch("cleveragents.reactive.application.ReactiveConfigParser"), + ): + app = ReactiveCleverAgentsApp.__new__(ReactiveCleverAgentsApp) + app.verbose = 0 + app.unsafe = False + app.temperature_override = None + app._skill_names = [] + app._resolved_skill_tools = [] + app._last_run_tool_calls = 0 + app.config = None + app._builtin_registry = ToolRegistry() + app.stream_router = MagicMock() + app.config_parser = MagicMock() + app.langgraph_bridge = None + app.route_bridge = None + + # Create a ToolCallingAgent with last_result set + fake_agent = ToolCallingAgent( + name="fake", + actor_config={}, + resolved_tool_entries=[], + builtin_registry=ToolRegistry(), + ) + fake_agent.last_result = ToolCallRunResult( + content="done", + tool_call_history=[ + ToolCallRecord( + tool_name="builtin/file-read", + inputs={"path": "f.py"}, + output={"content": "x"}, + duration_ms=1.0, + success=True, + iteration=1, + ), + ToolCallRecord( + tool_name="builtin/file-read", + inputs={"path": "g.py"}, + output={"content": "y"}, + duration_ms=1.0, + success=True, + iteration=2, + ), + ], + iterations=3, + terminated_by_limit=False, + ) + + # Register fake agent in the stream_router agents dict + app.stream_router.agents = {"fake-actor": fake_agent} + context.reactive_app = app + context.expected_tool_calls = 2 + + +@when("_tally_tool_calls is called on the app") +def step_tally_tool_calls(context: Any) -> None: + context.reactive_app._tally_tool_calls() + + +@then("last_run_tool_calls equals the number of tool call history entries") +def step_last_run_tool_calls_correct(context: Any) -> None: + actual = context.reactive_app.last_run_tool_calls + assert actual == context.expected_tool_calls, ( + f"Expected last_run_tool_calls == {context.expected_tool_calls}, got {actual}" + ) + + +# --------------------------------------------------------------------------- +# Scenarios I: ToolCallingLLMCaller._render_prompt +# --------------------------------------------------------------------------- + + +@given("a fresh ToolCallingLLMCaller") +def step_fresh_tool_calling_llm_caller(context: Any) -> None: + context.caller = ToolCallingLLMCaller(actor_config={}) + + +@when("_render_prompt is called with an empty string template") +def step_render_prompt_empty(context: Any) -> None: + context.render_result = context.caller._render_prompt("", {}) + + +@then("_render_prompt returns an empty string") +def step_render_prompt_empty_result(context: Any) -> None: + assert context.render_result == "", f"Expected '', got {context.render_result!r}" + + +@when("_render_prompt is called with a Jinja2 template and context") +def step_render_prompt_jinja2(context: Any) -> None: + context.render_result = context.caller._render_prompt( + "Hello {{ name }}!", {"name": "World"} + ) + + +@then("_render_prompt returns the rendered string") +def step_render_prompt_rendered(context: Any) -> None: + assert context.render_result == "Hello World!", ( + f"Expected 'Hello World!', got {context.render_result!r}" + ) + + +@when("_render_prompt is called with a template that raises during rendering") +def step_render_prompt_error(context: Any) -> None: + # Use an invalid Jinja2 template that raises during rendering + context.bad_template = "{{ undefined_var | custom_filter }}" + context.render_result = context.caller._render_prompt(context.bad_template, {}) + + +@then("_render_prompt returns the original template") +def step_render_prompt_original(context: Any) -> None: + assert context.render_result == context.bad_template, ( + f"Expected original template, got {context.render_result!r}" + ) + + +# --------------------------------------------------------------------------- +# Scenarios J: ToolCallingLLMCaller._resolve_llm +# --------------------------------------------------------------------------- + + +@given("a ToolCallingLLMCaller with temperature max_tokens max_retries in actor_config") +def step_caller_with_full_config(context: Any) -> None: + context.caller_config = { + "provider": "test", + "model": "test-model", + "temperature": 0.7, + "max_tokens": 100, + "max_retries": 2, + } + context.caller = ToolCallingLLMCaller(actor_config=context.caller_config) + context.mock_llm = MagicMock() + context.mock_llm.bind_tools.return_value = context.mock_llm + + +@when("_resolve_llm is called with tool_schemas") +def step_resolve_llm_with_schemas(context: Any) -> None: + with patch("cleveragents.reactive.tool_caller.get_provider_registry") as mock_reg: + mock_reg.return_value.create_llm.return_value = context.mock_llm + context.resolved_llm = context.caller._resolve_llm([{"name": "test_tool"}]) + context.mock_registry = mock_reg.return_value + + +@then("the provider registry create_llm was called with the config kwargs") +def step_create_llm_called_with_kwargs(context: Any) -> None: + call_kwargs = context.mock_registry.create_llm.call_args.kwargs + assert call_kwargs.get("temperature") == 0.7 + assert call_kwargs.get("max_tokens") == 100 + assert call_kwargs.get("max_retries") == 2 + + +@when("_resolve_llm is called without tool_schemas") +def step_resolve_llm_no_schemas(context: Any) -> None: + with patch("cleveragents.reactive.tool_caller.get_provider_registry") as mock_reg: + mock_reg.return_value.create_llm.return_value = context.mock_llm + context.resolved_llm = context.caller._resolve_llm([]) + + +@then("bind_tools was not called on the LLM") +def step_bind_tools_not_called(context: Any) -> None: + assert not context.mock_llm.bind_tools.called, "bind_tools should not be called" + + +@given("a ToolCallingLLMCaller with a provider that raises on bind_tools") +def step_caller_bind_tools_raises(context: Any) -> None: + context.caller = ToolCallingLLMCaller( + actor_config={"provider": None, "model": None} + ) + context.mock_llm_failing_bind = MagicMock() + context.mock_llm_failing_bind.bind_tools.side_effect = ValueError( + "bind_tools failed" + ) + + +@when("_resolve_llm is called with tool_schemas for a failing bind") +def step_resolve_llm_with_schemas_bind_fails(context: Any) -> None: + with patch("cleveragents.reactive.tool_caller.get_provider_registry") as mock_reg: + mock_reg.return_value.create_llm.return_value = context.mock_llm_failing_bind + context.resolved_llm = context.caller._resolve_llm([{"name": "some_tool"}]) + + +@then("the plain LLM is used without tool binding") +def step_plain_llm_used(context: Any) -> None: + assert context.resolved_llm is context.mock_llm_failing_bind, ( + "Should fall back to the plain LLM when bind_tools fails" + ) + + +# --------------------------------------------------------------------------- +# Scenarios K: invoke without system prompt +# --------------------------------------------------------------------------- + + +@given("a ToolCallingLLMCaller with an actor config without system_prompt") +def step_caller_without_system_prompt(context: Any) -> None: + context.llm_caller = ToolCallingLLMCaller( + actor_config={"provider": None, "model": None} + ) + mock_llm = MagicMock() + mock_llm.invoke.return_value = MagicMock( + content="no-system response", tool_calls=[] + ) + mock_llm.bind_tools.return_value = mock_llm + context.llm_caller._llm = mock_llm + context.mock_llm_for_caller = mock_llm + + +# Note: the "invoke is called for the first time with a user prompt" step +# is defined in Scenario E's section above and shared with Scenario K. + + +@then("no SystemMessage was added") +def step_no_system_message_added(context: Any) -> None: + try: + from langchain_core.messages import SystemMessage + except ImportError: + return + has_system = any( + isinstance(m, SystemMessage) for m in context.llm_caller._accumulated + ) + assert not has_system, "Expected NO SystemMessage in accumulated messages" + + +# --------------------------------------------------------------------------- +# Scenarios L: invoke with list content +# --------------------------------------------------------------------------- + + +@when("invoke is called and the LLM returns list content") +def step_invoke_list_content(context: Any) -> None: + mock_llm = MagicMock() + mock_response = MagicMock() + mock_response.content = [ + {"text": "Part one"}, + "Part two", + ] + mock_response.tool_calls = [] + mock_llm.invoke.return_value = mock_response + context.llm_caller._llm = mock_llm + context.llm_response = context.llm_caller.invoke( + prompt="list content test", + tool_schemas=[], + ) + + +@then("the response content joins the list parts") +def step_response_content_joined(context: Any) -> None: + assert "Part one" in context.llm_response.content, ( + f"Expected 'Part one' in content, got {context.llm_response.content!r}" + ) + assert "Part two" in context.llm_response.content + + +# --------------------------------------------------------------------------- +# Scenarios M: invoke with failed tool_result +# --------------------------------------------------------------------------- + + +@when("invoke is called with a failed tool_result") +def step_caller_failed_tool_result(context: Any) -> None: + context.llm_response2 = context.llm_caller.invoke( + prompt="ignored", + tool_schemas=[], + tool_results=[ + { + "tool_name": "builtin/file-read", + "call_id": "call-fail", + "success": False, + "output": {}, + "error": "File not found", + } + ], + ) + + +@then("a ToolMessage with the error text is appended") +def step_tool_message_error_appended(context: Any) -> None: + try: + from langchain_core.messages.tool import ToolMessage + except ImportError: + return + tool_messages = [ + m for m in context.llm_caller._accumulated if isinstance(m, ToolMessage) + ] + assert any("File not found" in m.content for m in tool_messages), ( + "Expected a ToolMessage containing 'File not found'" + ) + + +# --------------------------------------------------------------------------- +# Scenarios N: ToolCallingAgent._build_tool_registry edge cases +# --------------------------------------------------------------------------- + + +@given("a ToolCallingAgent with an entry that has an empty name") +def step_agent_empty_name_entry(context: Any) -> None: + builtin_registry = ToolRegistry() + builtin_registry.register(_make_file_read_spec()) + context.edge_agent = ToolCallingAgent( + name="edge-agent", + actor_config={}, + resolved_tool_entries=[{"name": ""}], + builtin_registry=builtin_registry, + ) + + +@when("_build_tool_registry is called") +def step_build_tool_registry_called(context: Any) -> None: + context.local_registry = context.edge_agent._build_tool_registry() + + +@then("the local registry is empty") +def step_local_registry_empty(context: Any) -> None: + tools = context.local_registry.list_tools() + assert len(tools) == 0, f"Expected empty registry, got {len(tools)} tools" + + +@given("a ToolCallingAgent with two entries for the same builtin tool") +def step_agent_duplicate_entries(context: Any) -> None: + builtin_registry = ToolRegistry() + builtin_registry.register(_make_file_read_spec()) + context.edge_agent = ToolCallingAgent( + name="dup-agent", + actor_config={}, + resolved_tool_entries=[ + {"name": "builtin/file-read"}, + {"name": "builtin/file-read"}, # duplicate + ], + builtin_registry=builtin_registry, + ) + + +@then("the local registry has exactly one tool registered") +def step_local_registry_one_tool(context: Any) -> None: + tools = context.local_registry.list_tools() + assert len(tools) == 1, f"Expected 1 tool, got {len(tools)} tools" + + +@given("a ToolCallingAgent with an entry not found in builtin registry") +def step_agent_unknown_tool_entry(context: Any) -> None: + builtin_registry = ToolRegistry() # empty registry + context.edge_agent = ToolCallingAgent( + name="unknown-agent", + actor_config={}, + resolved_tool_entries=[{"name": "builtin/nonexistent-tool"}], + builtin_registry=builtin_registry, + ) + + +# --------------------------------------------------------------------------- +# Scenarios O: ToolCallingAgent.process_message_sync +# --------------------------------------------------------------------------- + + +@given("a ToolCallingAgent with no tools and a mock process method") +def step_agent_no_tools_mock_process(context: Any) -> None: + builtin_registry = ToolRegistry() + context.sync_agent = ToolCallingAgent( + name="sync-test", + actor_config={}, + resolved_tool_entries=[], + builtin_registry=builtin_registry, + ) + context.mock_caller = _MockLLMCaller( + responses=[LLMResponse(content="sync result", tool_calls=[])] + ) + + +@when("process_message_sync is called") +def step_process_message_sync_called(context: Any) -> None: + with patch( + "cleveragents.reactive.tool_caller.ToolCallingLLMCaller", + return_value=context.mock_caller, + ): + context.sync_result = context.sync_agent.process_message_sync( + "sync prompt", None + ) + + +@then("the result is the same as calling process directly") +def step_sync_result_equals_process(context: Any) -> None: + assert context.sync_result == "sync result", ( + f"Expected 'sync result', got {context.sync_result!r}" + ) + + +# --------------------------------------------------------------------------- +# Scenarios P: extract tool calls from LLM response +# --------------------------------------------------------------------------- + + +@when("invoke is called and the LLM returns a response with tool call dicts") +def step_invoke_with_tool_call_dicts(context: Any) -> None: + mock_llm = MagicMock() + mock_response = MagicMock() + mock_response.content = "I'll call the tool" + mock_response.tool_calls = [ + {"name": "builtin/file-read", "args": {"path": "test.py"}, "id": "tc-1"} + ] + mock_llm.invoke.return_value = mock_response + context.llm_caller._llm = mock_llm + context.llm_response = context.llm_caller.invoke( + prompt="extract tool calls", + tool_schemas=[], + ) + + +@then("the LLMResponse contains the extracted tool calls") +def step_response_has_tool_calls(context: Any) -> None: + assert len(context.llm_response.tool_calls) == 1, ( + f"Expected 1 tool call, got {len(context.llm_response.tool_calls)}" + ) + tc = context.llm_response.tool_calls[0] + assert tc.name == "builtin/file-read" + assert tc.call_id == "tc-1" + + +# --------------------------------------------------------------------------- +# Scenarios Q: _resolve_provider_format +# --------------------------------------------------------------------------- + + +@given('an actor config with provider "{provider}"') +def step_actor_config_with_provider(context: Any, provider: str) -> None: + context.resolve_config = {"provider": provider, "model": "test-model"} + + +@given("an empty actor config") +def step_empty_actor_config(context: Any) -> None: + context.resolve_config = {} + + +@when("_resolve_provider_format is called with that actor config") +def step_resolve_provider_format_called(context: Any) -> None: + context.resolve_result = ToolCallingAgent._resolve_provider_format( + context.resolve_config + ) + + +@when("_resolve_provider_format is called with actor_config None") +def step_resolve_provider_format_none(context: Any) -> None: + context.resolve_result = ToolCallingAgent._resolve_provider_format(None) + + +@then("_resolve_provider_format returns ProviderFormat.ANTHROPIC") +def step_format_is_anthropic(context: Any) -> None: + assert context.resolve_result == ProviderFormat.ANTHROPIC, ( + f"Expected ANTHROPIC, got {context.resolve_result}" + ) + + +@then("_resolve_provider_format returns ProviderFormat.OPENAI") +def step_format_is_openai(context: Any) -> None: + assert context.resolve_result == ProviderFormat.OPENAI, ( + f"Expected OPENAI, got {context.resolve_result}" + ) + + +# --------------------------------------------------------------------------- +# Scenario R: ToolCallingAgent.process passes provider_format to runtime +# --------------------------------------------------------------------------- + + +@given("a ToolCallingAgent with an anthropic actor config and a file-read tool entry") +def step_agent_anthropic_config(context: Any) -> None: + builtin_registry = ToolRegistry() + builtin_registry.register(_make_file_read_spec()) + context.builtin_registry = builtin_registry + context.format_agent = ToolCallingAgent( + name="format-test", + actor_config={"provider": "anthropic", "model": "claude-sonnet-4-20250514"}, + resolved_tool_entries=[{"name": "builtin/file-read"}], + builtin_registry=builtin_registry, + ) + context.mock_caller = _MockLLMCaller( + responses=[LLMResponse(content="read the file", tool_calls=[])] + ) + + +@then("ToolCallingRuntime was constructed with ProviderFormat.ANTHROPIC") +def step_runtime_constructed_with_anthropic(context: Any) -> None: + with ( + patch( + "cleveragents.reactive.tool_caller.ToolCallingLLMCaller", + return_value=context.mock_caller, + ), + patch( + "cleveragents.reactive.tool_agent.ToolCallingRuntime" + ) as mock_runtime_cls, + ): + mock_runtime = MagicMock() + mock_runtime.run_tool_loop.return_value = ToolCallRunResult( + content="result", iterations=1, terminated_by_limit=False + ) + mock_runtime_cls.return_value = mock_runtime + + context.format_agent.process("Read my file") + + # Extract the provider_format kwarg passed to ToolCallingRuntime(...) + call_kwargs = mock_runtime_cls.call_args.kwargs + assert "provider_format" in call_kwargs, ( + "provider_format was not passed to ToolCallingRuntime" + ) + assert call_kwargs["provider_format"] == ProviderFormat.ANTHROPIC, ( + f"Expected ANTHROPIC, got {call_kwargs['provider_format']}" + ) + + +# --------------------------------------------------------------------------- +# Scenarios S: Tool name encoding/decoding +# --------------------------------------------------------------------------- + + +@when('_encode_tool_name is called with "{tool_name}"') +def step_encode_tool_name(context: Any, tool_name: str) -> None: + from cleveragents.reactive.tool_caller import _encode_tool_name + + context.encoded_name = _encode_tool_name(tool_name) + + +@then('_encode_tool_name returns "{expected}"') +def step_encode_tool_name_returns(context: Any, expected: str) -> None: + assert context.encoded_name == expected, ( + f"Expected {expected!r}, got {context.encoded_name!r}" + ) + + +@when('_decode_tool_name is called with "{tool_name}"') +def step_decode_tool_name(context: Any, tool_name: str) -> None: + from cleveragents.reactive.tool_caller import _decode_tool_name + + context.decoded_name = _decode_tool_name(tool_name) + + +@then('_decode_tool_name returns "{expected}"') +def step_decode_tool_name_returns(context: Any, expected: str) -> None: + assert context.decoded_name == expected, ( + f"Expected {expected!r}, got {context.decoded_name!r}" + ) + + +@given('a tool name "{original_name}"') +def step_given_tool_name(context: Any, original_name: str) -> None: + context.roundtrip_name = original_name + + +@when("the tool name is encoded then decoded") +def step_encode_then_decode(context: Any) -> None: + from cleveragents.reactive.tool_caller import _decode_tool_name, _encode_tool_name + + context.roundtrip_result = _decode_tool_name( + _encode_tool_name(context.roundtrip_name) + ) + + +@then('the round-trip name equals "{expected}"') +def step_roundtrip_equals(context: Any, expected: str) -> None: + assert context.roundtrip_result == expected, ( + f"Round-trip expected {expected!r}, got {context.roundtrip_result!r}" + ) + + +# --------------------------------------------------------------------------- +# Scenarios T: _resolve_llm encodes tool names in schemas +# --------------------------------------------------------------------------- + + +@given("a ToolCallingLLMCaller with default actor config") +def step_caller_default_config(context: Any) -> None: + context.tool_caller = ToolCallingLLMCaller( + actor_config={"provider": None, "model": None} + ) + context.mock_llm_t = MagicMock() + context.mock_llm_t.bind_tools.return_value = context.mock_llm_t + + +@when("_resolve_llm is called with a schema containing a namespaced tool name") +def step_resolve_llm_namespaced_schema(context: Any) -> None: + with patch("cleveragents.reactive.tool_caller.get_provider_registry") as mock_reg: + mock_reg.return_value.create_llm.return_value = context.mock_llm_t + context.tool_caller._resolve_llm( + [{"name": "builtin/file-read", "description": "Read a file"}] + ) + + +@then("bind_tools was called with the encoded tool name") +def step_bind_tools_called_with_encoded_name(context: Any) -> None: + assert context.mock_llm_t.bind_tools.called, "Expected bind_tools to be called" + call_args = context.mock_llm_t.bind_tools.call_args[0][0] + assert len(call_args) == 1, f"Expected 1 schema, got {len(call_args)}" + encoded_name = call_args[0]["name"] + assert encoded_name == "builtin_S_file-read", ( + f"Expected encoded name 'builtin_S_file-read', got {encoded_name!r}" + ) + # Verify the original schema was not mutated + assert "/" not in encoded_name, "Encoded name must not contain '/'" + + +# --------------------------------------------------------------------------- +# Scenarios U: invoke decodes tool names from LLM response +# --------------------------------------------------------------------------- + + +@when("invoke is called and the LLM returns a response with encoded tool call names") +def step_invoke_encoded_tool_calls(context: Any) -> None: + mock_llm = MagicMock() + mock_response = MagicMock() + mock_response.content = "I'll call a tool" + mock_response.tool_calls = [ + { + "name": "builtin_S_file-read", + "args": {"path": "test.py"}, + "id": "tc-encoded", + }, + { + "name": "server_C_local_S_my-tool", + "args": {"arg": "val"}, + "id": "tc-server-qualified", + }, + ] + mock_llm.invoke.return_value = mock_response + context.llm_caller._llm = mock_llm + context.llm_response = context.llm_caller.invoke( + prompt="encoded tool call test", + tool_schemas=[], + ) + + +@then("the LLMResponse contains the decoded tool calls") +def step_response_contains_decoded_tool_calls(context: Any) -> None: + assert len(context.llm_response.tool_calls) == 2, ( + f"Expected 2 tool calls, got {len(context.llm_response.tool_calls)}" + ) + names = [tc.name for tc in context.llm_response.tool_calls] + assert "builtin/file-read" in names, ( + f"Expected 'builtin/file-read' in tool call names, got {names}" + ) + assert "server:local/my-tool" in names, ( + f"Expected 'server:local/my-tool' in tool call names, got {names}" + ) diff --git a/features/steps/reactive_application_coverage_boost_steps.py b/features/steps/reactive_application_coverage_boost_steps.py index 093d1ce97..c9cc275c2 100644 --- a/features/steps/reactive_application_coverage_boost_steps.py +++ b/features/steps/reactive_application_coverage_boost_steps.py @@ -668,14 +668,15 @@ def step_register_agents_with_skill_tools_and_llm(context: Context) -> None: context.app._register_agents_from_config() # pylint: disable=protected-access -@then("the LLM agent should remain a SimpleLLMAgent not a SimpleToolAgent") -def step_llm_agent_not_converted(context: Context) -> None: - from cleveragents.reactive.stream_router import SimpleLLMAgent, SimpleToolAgent +@then("the LLM agent should be a ToolCallingAgent not a SimpleLLMAgent") +def step_llm_agent_upgraded_to_tool_calling(context: Context) -> None: + from cleveragents.reactive.stream_router import SimpleToolAgent + from cleveragents.reactive.tool_agent import ToolCallingAgent llm_agent = context.app.stream_router.agents.get("llm_actor") assert llm_agent is not None, "Expected llm_actor to be registered" - assert isinstance(llm_agent, SimpleLLMAgent), ( - f"Expected SimpleLLMAgent but got {type(llm_agent).__name__}" + assert isinstance(llm_agent, ToolCallingAgent), ( + f"Expected ToolCallingAgent but got {type(llm_agent).__name__}" ) tool_agent = context.app.stream_router.agents.get("tool_actor") diff --git a/robot/helper_actor_run_signature.py b/robot/helper_actor_run_signature.py index 2ce7dbd48..0ed014401 100644 --- a/robot/helper_actor_run_signature.py +++ b/robot/helper_actor_run_signature.py @@ -48,6 +48,8 @@ def _make_app(*, result: str) -> MagicMock: app_exec = MagicMock() app_exec.config = SimpleNamespace(global_context={}) app_exec.run_single_shot = AsyncMock(return_value=result) + # last_run_tool_calls is an int property; set to 0 so comparisons work in tests + app_exec.last_run_tool_calls = 0 return app_exec diff --git a/src/cleveragents/cli/commands/actor_run.py b/src/cleveragents/cli/commands/actor_run.py index 14b2d2cf2..0d3bc220e 100644 --- a/src/cleveragents/cli/commands/actor_run.py +++ b/src/cleveragents/cli/commands/actor_run.py @@ -168,6 +168,11 @@ def run( typer.echo(f"Unexpected error: {exc}", err=True) raise typer.Exit(code=3) from exc + # ST-6: Surface tool_calls count from the most recent run + tool_calls_count = app_exec.last_run_tool_calls + if tool_calls_count > 0: + typer.echo(f"Tool Calls: {tool_calls_count}") + if output: output.write_text(result) typer.echo(f"Output written to {output}") diff --git a/src/cleveragents/reactive/application.py b/src/cleveragents/reactive/application.py index d00df3363..9e6f54196 100644 --- a/src/cleveragents/reactive/application.py +++ b/src/cleveragents/reactive/application.py @@ -23,6 +23,13 @@ from cleveragents.reactive.stream_router import ( SimpleLLMAgent, SimpleToolAgent, ) +from cleveragents.reactive.tool_agent import ToolCallingAgent +from cleveragents.tool.builtins import ( + register_file_tools, + register_git_tools, + register_subplan_tool, +) +from cleveragents.tool.registry import ToolRegistry # Skill name format: namespace/short-name, ASCII alphanumeric + ._- only. _SKILL_NAME_RE = re.compile(r"^[\w.-]{1,127}/[\w.-]{1,127}$", re.ASCII) @@ -49,6 +56,7 @@ class ReactiveCleverAgentsApp: dict.fromkeys(self._sanitize_skill_name(n) for n in (skill_names or [])) ) self._resolved_skill_tools: list[dict[str, Any]] = [] + self._last_run_tool_calls: int = 0 self._configure_logging(verbose) self.stream_router = ReactiveStreamRouter() @@ -57,6 +65,14 @@ class ReactiveCleverAgentsApp: self.route_bridge = None self.config: ReactiveConfig | None = None + # ST-4: Shared built-in ToolRegistry populated once at startup. + # ToolCallingAgent instances look up tool names here to build their + # per-run local registry. + self._builtin_registry = ToolRegistry() + register_file_tools(self._builtin_registry) + register_git_tools(self._builtin_registry) + register_subplan_tool(self._builtin_registry) + if self._skill_names: self._resolve_skills() @@ -81,6 +97,11 @@ class ReactiveCleverAgentsApp: """Return the resolved skill tool configurations.""" return list(self._resolved_skill_tools) + @property + def last_run_tool_calls(self) -> int: + """Return the number of tool calls made during the most recent run.""" + return self._last_run_tool_calls + @staticmethod def _sanitize_skill_name(name: str) -> str: """Validate format and strip control characters from a skill name. @@ -178,13 +199,19 @@ class ReactiveCleverAgentsApp: def _make_agent_instance(name: str, agent_cfg: Any) -> Any: tools = agent_cfg.config.get("tools", []) if agent_cfg.config else [] - if self._resolved_skill_tools and tools: + # ST-1: Always merge skill tools regardless of whether the actor + # has a base tools list. The old elif branch silently dropped skill + # tools when the actor config had no base tools — that was a bug. + if self._resolved_skill_tools: tools = list(tools) + self._resolved_skill_tools - elif self._resolved_skill_tools and not tools: - logger.debug( - "Skipping skill tool injection for agent '%s' " - "(agent has no base tools)", - name, + # Routing: + # tools + llm type → ToolCallingAgent (real LLM tool calling) + # tools + non-llm → SimpleToolAgent (string transforms) + # no tools + llm → SimpleLLMAgent (plain LLM, no regression) + # no tools + other → identity lambda + if tools and agent_cfg.type == "llm": + return ToolCallingAgent( + name, agent_cfg.config, tools, self._builtin_registry ) if tools: return SimpleToolAgent(tools, unsafe=self.unsafe) @@ -295,6 +322,18 @@ class ReactiveCleverAgentsApp: ].on_next(msg) ) + def _tally_tool_calls(self) -> None: + """Accumulate tool_calls count from all ToolCallingAgent instances. + + Called at the end of every ``run_single_shot()`` execution to update + ``_last_run_tool_calls`` so the CLI can surface the count. + """ + total = 0 + for agent in self.stream_router.agents.values(): + if isinstance(agent, ToolCallingAgent) and agent.last_result is not None: + total += len(agent.last_result.tool_call_history) + self._last_run_tool_calls = total + def _is_rxpy_stream_present(self) -> bool: if not self.config: return False @@ -384,11 +423,15 @@ class ReactiveCleverAgentsApp: if context_manager and self.config and context_manager.global_context: self.config.global_context.update(context_manager.global_context) + # Reset tool-call counter at the start of each run + self._last_run_tool_calls = 0 + # Try graph route execution first graph_route = self._get_graph_route() if graph_route is not None: executor = GraphExecutor(self.stream_router.agents, self.config) output = executor.execute(prompt, graph_route) + self._tally_tool_calls() return GraphExecutor.strip_routing_prefixes(output) loop = asyncio.get_running_loop() @@ -414,9 +457,11 @@ class ReactiveCleverAgentsApp: raise CleverAgentsException("; ".join(error_container)) if not result_container: + self._tally_tool_calls() return "" output = "\n".join(result_container) + self._tally_tool_calls() return GraphExecutor.strip_routing_prefixes_multiline(output) async def run_with_context( diff --git a/src/cleveragents/reactive/graph_executor.py b/src/cleveragents/reactive/graph_executor.py index 784544f6a..ad89ef7dc 100644 --- a/src/cleveragents/reactive/graph_executor.py +++ b/src/cleveragents/reactive/graph_executor.py @@ -17,6 +17,7 @@ from typing import Any from cleveragents.reactive.config_parser import ReactiveConfig from cleveragents.reactive.route import RouteConfig from cleveragents.reactive.stream_router import SimpleLLMAgent, SimpleToolAgent +from cleveragents.reactive.tool_agent import ToolCallingAgent logger = logging.getLogger(__name__) @@ -138,7 +139,10 @@ class GraphExecutor: @staticmethod def _invoke_agent(agent: Any, message: str, context: dict[str, Any]) -> str: """Execute an agent and return the string result.""" - if isinstance(agent, (SimpleToolAgent | SimpleLLMAgent)): + # ST-5: ToolCallingAgent also accepts context for Jinja2 rendering; + # include it in the isinstance check alongside SimpleToolAgent and + # SimpleLLMAgent so the global context dict is passed through. + if isinstance(agent, (SimpleToolAgent, SimpleLLMAgent, ToolCallingAgent)): result = agent.process(message, context=context) elif hasattr(agent, "process"): result = agent.process(message) diff --git a/src/cleveragents/reactive/tool_agent.py b/src/cleveragents/reactive/tool_agent.py new file mode 100644 index 000000000..6dbdfb73a --- /dev/null +++ b/src/cleveragents/reactive/tool_agent.py @@ -0,0 +1,192 @@ +"""ToolCallingAgent — actor run agent with LLM tool-calling support. + +Implements a reactive agent that drives ``ToolCallingRuntime`` for the +``actor run`` path. When a skill is attached via ``--skill``, this agent +is used instead of ``SimpleLLMAgent`` so that the LLM can actually invoke +tools during its response loop. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from cleveragents.tool.actor_runtime import ToolCallingRuntime, ToolCallRunResult +from cleveragents.tool.registry import ToolRegistry +from cleveragents.tool.router import ProviderFormat +from cleveragents.tool.runner import ToolRunner + +logger = logging.getLogger(__name__) + + +class ToolCallingAgent: + """LLM-backed agent that supports real tool calling via ToolCallingRuntime. + + Unlike ``SimpleLLMAgent`` (which calls ``llm.invoke()`` with no tool + schemas) this agent: + + 1. Builds a per-run ``ToolRegistry`` from the resolved skill tool entries + by looking them up in a shared *builtin_registry*. + 2. Creates a ``ToolRunner`` and a ``ToolCallingLLMCaller`` for that run. + 3. Delegates to ``ToolCallingRuntime.run_tool_loop()`` which handles the + bind-tools / invoke / execute / feed-back loop. + 4. Exposes ``last_result`` so the CLI can surface the ``tool_calls`` count. + + Parameters + ---------- + name: + Actor name (used for logging). + actor_config: + Raw actor configuration dict from the YAML file (provider, model, + system_prompt, temperature, etc.). + resolved_tool_entries: + List of resolved tool entry dicts from ``_resolve_skills()``/ + ``_make_agent_instance()``. Each entry has at least a ``"name"`` + key with the namespaced tool name. + builtin_registry: + Shared ``ToolRegistry`` pre-populated with built-in tools + (file, git, subplan). Each entry's name is looked up here to + obtain the real ``ToolSpec``. + """ + + def __init__( + self, + name: str, + actor_config: dict[str, Any] | None, + resolved_tool_entries: list[dict[str, Any]], + builtin_registry: ToolRegistry, + ) -> None: + self._name = name + self._actor_config: dict[str, Any] = dict(actor_config or {}) + self._resolved_tool_entries = resolved_tool_entries + self._builtin_registry = builtin_registry + self.last_result: ToolCallRunResult | None = None + + # -- Internal helpers ----------------------------------------------------- + + def _build_tool_registry(self) -> ToolRegistry: + """Build a fresh per-run ``ToolRegistry`` from resolved entries. + + Each entry's ``"name"`` is looked up in the shared + ``_builtin_registry``. Entries whose name is not found (e.g. inline + or MCP tools not yet in the builtin registry) are warned and skipped + so the run degrades gracefully rather than crashing. + + Returns + ------- + ToolRegistry + A new registry containing only the successfully resolved tools. + """ + local_registry = ToolRegistry() + seen: set[str] = set() + + for entry in self._resolved_tool_entries: + tool_name: str = entry.get("name", "") if isinstance(entry, dict) else "" + if not tool_name: + continue + if tool_name in seen: + # Deduplicate: the same builtin may appear more than once when + # multiple skills reference it. + continue + spec = self._builtin_registry.get(tool_name) + if spec is None: + logger.warning( + "ToolCallingAgent '%s': tool '%s' not found in builtin " + "registry — skipping (inline/MCP tools not yet supported " + "in actor run tool-calling path)", + self._name, + tool_name, + ) + continue + try: + local_registry.register(spec) + seen.add(tool_name) + except Exception as exc: # pylint: disable=broad-except + logger.debug( + "ToolCallingAgent '%s': skipping '%s' (%s)", + self._name, + tool_name, + exc, + ) + + return local_registry + + # -- Agent interface ------------------------------------------------------ + + @staticmethod + def _resolve_provider_format(actor_config: dict[str, Any] | None) -> ProviderFormat: + """Map an actor config's ``provider`` field to the correct ``ProviderFormat``. + + The format determines the schema key emitted by + ``normalize_tool_schema_for_provider``, which must survive + LangChain's ``convert_to_openai_function()`` call inside + ``bind_tools()``: + + - ``ProviderFormat.LANGCHAIN`` emits ``args_schema`` → **silently + dropped** by ``convert_to_openai_function()`` → ``bind_tools`` + raises ``'parameters'``. + - ``ProviderFormat.ANTHROPIC`` emits ``input_schema`` → preserved + and mapped to ``parameters``. + - ``ProviderFormat.OPENAI`` emits ``parameters`` → preserved + directly. + + Therefore ``ProviderFormat.LANGCHAIN`` must never be used in the + ``ToolCallingRuntime`` → ``bind_tools()`` pipeline. The default + for unrecognised providers is ``ProviderFormat.OPENAI`` since + most providers accept OpenAI-compatible tool schemas. + """ + if not actor_config: + return ProviderFormat.OPENAI + provider: str = (actor_config.get("provider") or "").lower() + if provider == "anthropic": + return ProviderFormat.ANTHROPIC + # openai, google, groq, azure, openrouter, gemini, cohere, + # together, mock, and any unknown provider default to OpenAI + # format since all of these consume OpenAI-compatible tool + # schemas through LangChain's bind_tools(). + return ProviderFormat.OPENAI + + def process( + self, + content: Any, + metadata: dict[str, Any] | None = None, + context: dict[str, Any] | None = None, + ) -> Any: + """Execute the tool-calling loop and return the final text response. + + Parameters + ---------- + content: + The user prompt (string or coercible to string). + metadata: + Optional stream metadata (currently unused). + context: + Optional Jinja2 rendering context (currently unused). + + Returns + ------- + str + The final text response from the LLM after all tool calls. + """ + # Lazy import to avoid circular imports at module load time + from cleveragents.reactive.tool_caller import ToolCallingLLMCaller + + prompt = content if isinstance(content, str) else str(content or "") + local_registry = self._build_tool_registry() + runner = ToolRunner(local_registry) + caller = ToolCallingLLMCaller(self._actor_config) + runtime = ToolCallingRuntime( + registry=local_registry, + runner=runner, + llm_caller=caller, + provider_format=self._resolve_provider_format(self._actor_config), + ) + result = runtime.run_tool_loop(prompt, actor_config=self._actor_config) + self.last_result = result + return result.content + + def process_message_sync( + self, content: Any, metadata: dict[str, Any] | None = None + ) -> Any: + """Synchronous variant for stream-router compatibility.""" + return self.process(content, metadata) diff --git a/src/cleveragents/reactive/tool_caller.py b/src/cleveragents/reactive/tool_caller.py new file mode 100644 index 000000000..ca12d94f0 --- /dev/null +++ b/src/cleveragents/reactive/tool_caller.py @@ -0,0 +1,275 @@ +"""ToolCallingLLMCaller — LLMCaller implementation for actor run tool calling. + +Wraps a LangChain chat model with tool schema binding and multi-turn tool +result threading for use with ``ToolCallingRuntime`` in the reactive actor +run path. + +This caller is stateful within a single ``run_tool_loop()`` invocation: it +accumulates the LangChain message thread across tool-call loop iterations. +Create a fresh instance for each ``process()`` call in ``ToolCallingAgent``. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from cleveragents.application.services.prompt_sanitizer import PromptSanitizer +from cleveragents.tool.actor_runtime import LLMResponse, LLMToolCall + +try: + from langchain_core.messages import AIMessage, HumanMessage, SystemMessage + from langchain_core.messages.tool import ToolMessage +except Exception: # pragma: no cover - optional runtime dependency guard + AIMessage = None # type: ignore[assignment, misc] + HumanMessage = None # type: ignore[assignment, misc] + SystemMessage = None # type: ignore[assignment, misc] + ToolMessage = None # type: ignore[assignment, misc] + +try: + from jinja2.sandbox import SandboxedEnvironment +except Exception: # pragma: no cover - optional runtime dependency guard + SandboxedEnvironment = None # type: ignore[assignment, misc] + +from cleveragents.providers.registry import get_provider_registry + +logger = logging.getLogger(__name__) + +_SANITIZER = PromptSanitizer() + + +def _encode_tool_name(name: str) -> str: + """Encode forbidden characters for Anthropic tool name pattern. + + The Anthropic API requires tool names to match ``^[a-zA-Z0-9_-]{1,128}$``. + CleverAgents tool names use ``:`` (server prefix) and ``/`` (namespace + separator), neither of which is in the allowed character set. + + Because valid CleverAgents tool names (per the internal name regex) may + only contain lowercase letters, digits, ``-``, ``_``, ``:``, and ``/``, + any **uppercase** letter is guaranteed to be unambiguous as an escape + marker. This function replaces: + + * ``:`` → ``_C_`` (C for Colon) + * ``/`` → ``_S_`` (S for Slash) + + These sentinels cannot collide with legitimate tool names because + uppercase letters are forbidden in the internal name regex. + """ + return name.replace(":", "_C_").replace("/", "_S_") + + +def _decode_tool_name(name: str) -> str: + """Reverse :func:`_encode_tool_name`. + + Restores ``_S_`` → ``/`` and ``_C_`` → ``:``. + """ + return name.replace("_S_", "/").replace("_C_", ":") + + +class ToolCallingLLMCaller: + """LLMCaller implementation for actor run tool-calling loop. + + Implements the :class:`~cleveragents.tool.actor_runtime.LLMCaller` + protocol. On the first ``invoke()`` call it binds tool schemas to the + LLM via ``bind_tools()`` and sends the initial system + user messages. + On subsequent calls (when ``tool_results`` is provided) it appends the + previous AI response and ``ToolMessage`` objects for each result, then + invokes again to continue the tool-call conversation. + + Parameters + ---------- + actor_config: + Actor configuration dict (provider, model, temperature, etc.). + When not provided an empty dict is used, which falls through to + ``get_provider_registry().create_llm()`` defaults. + """ + + def __init__(self, actor_config: dict[str, Any] | None = None) -> None: + self._actor_config: dict[str, Any] = dict(actor_config or {}) + self._llm: Any = None + self._accumulated: list[Any] = [] + self._first_call: bool = True + self._last_response: Any = None + self._template_env = ( + SandboxedEnvironment() if SandboxedEnvironment is not None else None + ) + + # -- Internal helpers ----------------------------------------------------- + + def _render_prompt(self, template: str, context: dict[str, Any]) -> str: + """Render a Jinja2 template string against *context*. + + Falls back to returning the raw template if Jinja2 is unavailable or + rendering fails. + """ + if not template: + return "" + if self._template_env is None: + return template + try: + return self._template_env.from_string(template).render( + context=context, **context + ) + except Exception: + return template + + def _resolve_llm(self, tool_schemas: list[dict[str, Any]]) -> Any: + """Resolve and cache the LangChain LLM with tools bound. + + The LLM is resolved once on the first ``invoke()`` call. Subsequent + calls reuse the cached instance so that tool binding (which may create + a new LLM wrapper object) is only performed once per loop run. + """ + if self._llm is not None: + return self._llm + + provider = self._actor_config.get("provider") + model = self._actor_config.get("model") + temperature = self._actor_config.get("temperature") + max_tokens = self._actor_config.get("max_tokens") + max_retries = self._actor_config.get("max_retries") + + llm_kwargs: dict[str, Any] = {} + if temperature is not None: + llm_kwargs["temperature"] = temperature + if max_tokens is not None: + llm_kwargs["max_tokens"] = max_tokens + if max_retries is not None: + llm_kwargs["max_retries"] = max_retries + + registry = get_provider_registry() + # Annotated as Any because create_llm() returns BaseLanguageModel but the + # real runtime object is BaseChatModel which has bind_tools(). + base_llm: Any = registry.create_llm( + provider_type=provider, model_id=model, **llm_kwargs + ) + + if tool_schemas: + try: + # Encode tool names for Anthropic: ':' -> '_C_', '/' -> '_S_' + encoded_schemas = [ + {**s, "name": _encode_tool_name(s["name"])} for s in tool_schemas + ] + self._llm = base_llm.bind_tools(encoded_schemas) + except Exception as exc: + logger.warning("bind_tools failed (%s); falling back to plain LLM", exc) + self._llm = base_llm + else: + self._llm = base_llm + + return self._llm + + # -- LLMCaller protocol --------------------------------------------------- + + def invoke( + self, + prompt: str, + tool_schemas: list[dict[str, Any]], + tool_results: list[dict[str, Any]] | None = None, + actor_config: dict[str, Any] | None = None, + ) -> LLMResponse: + """Send a prompt (with optional tool results) to the LLM. + + On the **first** call (``tool_results is None``): + Builds the initial ``[SystemMessage, HumanMessage]`` pair from the + actor's ``system_prompt`` config and the user prompt, then invokes + the tool-bound LLM. + + On **subsequent** calls (``tool_results`` is a list): + Appends the previous ``AIMessage`` and one ``ToolMessage`` per + result entry, then invokes again. + + Parameters + ---------- + prompt: + The original user prompt text (used only on the first call). + tool_schemas: + Provider-formatted tool schema dicts passed to ``bind_tools()``. + tool_results: + Tool execution results from the previous iteration. Each entry + is a dict with keys ``tool_name``, ``call_id``, ``success``, + ``output``, and optionally ``error``. + actor_config: + Optional per-call actor config override. Falls back to the + instance-level ``actor_config`` passed to ``__init__``. + + Returns + ------- + LLMResponse + Structured response including content and any tool calls. + """ + if HumanMessage is None or SystemMessage is None: + raise RuntimeError("LangChain messages not available for LLM tool calling") + + effective_config = actor_config or self._actor_config + llm = self._resolve_llm(tool_schemas) + + if self._first_call: + system_prompt: str = effective_config.get("system_prompt", "") or "" + + # Apply prompt boundary sanitization (mechanism 2) + if system_prompt: + system_prompt = _SANITIZER.augment_system_prompt(system_prompt) + self._accumulated.append(SystemMessage(content=system_prompt)) + + wrapped_prompt = _SANITIZER.wrap_user_content(prompt) + self._accumulated.append(HumanMessage(content=wrapped_prompt)) + self._first_call = False + + elif tool_results is not None: + # Append the previous AI response so the model has context + if self._last_response is not None: + self._accumulated.append(self._last_response) + + # Append one ToolMessage per result + for tr in tool_results: + tool_result_content = ( + str(tr.get("output", {})) + if tr.get("success") + else tr.get("error", "error") + ) + if ToolMessage is not None: + self._accumulated.append( + ToolMessage( + content=tool_result_content, + tool_call_id=( + tr.get("call_id") or tr.get("tool_name", "unknown") + ), + ) + ) + + response = llm.invoke(self._accumulated) + self._last_response = response + + # Extract content + raw_content = getattr(response, "content", None) + if raw_content is None: + raw_content = str(response) + if isinstance(raw_content, list): + parts: list[str] = [] + for chunk in raw_content: + if isinstance(chunk, dict): + parts.append(str(chunk.get("text") or chunk.get("content", ""))) + else: + parts.append(str(chunk)) + content = " ".join(parts) + else: + content = str(raw_content) + + # Extract tool calls from the LangChain response + tool_calls: list[LLMToolCall] = [] + raw_tool_calls = getattr(response, "tool_calls", None) or [] + for tc in raw_tool_calls: + if isinstance(tc, dict): + raw_name = tc.get("name") or tc.get("id", "") + # Decode tool names from LLM response back to namespaced format + name = _decode_tool_name(raw_name) + args = tc.get("args") or tc.get("arguments") or {} + call_id = tc.get("id", "") + if name: + tool_calls.append( + LLMToolCall(name=name, arguments=args, call_id=call_id) + ) + + return LLMResponse(content=content, tool_calls=tool_calls)