cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

feat(actor): add tool-calling runtime for execution actors
CI / benchmark-publish (pull_request) Has been skipped
Details
CI / lint (pull_request) Failing after 16s
Details
CI / quality (pull_request) Successful in 21s
Details
CI / build (pull_request) Successful in 24s
Details
CI / security (pull_request) Successful in 27s
Details
CI / typecheck (pull_request) Successful in 40s
Details
CI / coverage (pull_request) Has been skipped
Details
CI / benchmark-regression (pull_request) Has been skipped
Details
CI / integration_tests (pull_request) Successful in 2m26s
Details
CI / unit_tests (pull_request) Successful in 5m40s
Details
CI / docker (pull_request) Has been skipped
Details

Browse Source
This commit is contained in:
Jeffrey Phillips Freeman
2026-02-22 08:30:28 +00:00
parent e40dd75478
commit c6d831b5ff
10 changed files with 2171 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/actor_runtime_bench.py
+185
View File
@@ -0,0 +1,185 @@
"""ASV benchmarks for tool-calling actor runtime overhead.
Measures the performance of:
- ToolCallingRuntime construction
- Tool schema export
- Single-iteration tool-call loop (1 tool call)
- Multi-iteration tool-call loop (3 tool calls)
- Max-iteration termination overhead
- ToolActorContext construction and recording
"""
from __future__ import annotations
import sys
from pathlib import Path
from typing import Any
try:
from cleveragents.tool.actor_context import ToolActorContext, ToolCallRecord
from cleveragents.tool.actor_runtime import (
LLMResponse,
LLMToolCall,
ToolCallingRuntime,
)
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
except ModuleNotFoundError:
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from cleveragents.tool.actor_context import ToolActorContext, ToolCallRecord
from cleveragents.tool.actor_runtime import (
LLMResponse,
LLMToolCall,
ToolCallingRuntime,
)
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
def _echo(inputs: dict[str, Any]) -> dict[str, Any]:
return {"echoed": inputs}
class _SingleToolLLM:
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
tool_calls=[LLMToolCall(name="bench/echo", arguments={"x": 1})]
)
return LLMResponse(content="done")
class _AlwaysToolLLM:
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
return LLMResponse(
tool_calls=[LLMToolCall(name="bench/echo", arguments={"x": 1})]
)
class _ImmediateLLM:
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
return LLMResponse(content="immediate")
def _make_registry() -> ToolRegistry:
reg = ToolRegistry()
reg.register(
ToolSpec(name="bench/echo", description="Bench echo", handler=_echo)
)
return reg
class ToolCallingRuntimeConstructionSuite:
"""Benchmark ToolCallingRuntime construction."""
def setup(self) -> None:
self.registry = _make_registry()
self.runner = ToolRunner(self.registry)
self.llm = _ImmediateLLM()
def time_construction(self) -> None:
ToolCallingRuntime(
registry=self.registry,
runner=self.runner,
llm_caller=self.llm,
)
class ToolSchemaExportSuite:
"""Benchmark tool schema export."""
def setup(self) -> None:
self.registry = _make_registry()
self.runner = ToolRunner(self.registry)
self.llm = _ImmediateLLM()
self.runtime = ToolCallingRuntime(
registry=self.registry,
runner=self.runner,
llm_caller=self.llm,
)
def time_export_schemas(self) -> None:
self.runtime.export_tool_schemas()
class SingleToolCallLoopSuite:
"""Benchmark single-iteration tool-call loop."""
def setup(self) -> None:
self.registry = _make_registry()
self.runner = ToolRunner(self.registry)
def time_single_tool_call(self) -> None:
llm = _SingleToolLLM()
runtime = ToolCallingRuntime(
registry=self.registry,
runner=self.runner,
llm_caller=llm,
)
runtime.run_tool_loop(prompt="bench")
class MaxIterationSuite:
"""Benchmark max-iteration termination overhead."""
def setup(self) -> None:
self.registry = _make_registry()
self.runner = ToolRunner(self.registry)
def time_max_iterations_5(self) -> None:
runtime = ToolCallingRuntime(
registry=self.registry,
runner=self.runner,
llm_caller=_AlwaysToolLLM(),
max_iterations=5,
)
runtime.run_tool_loop(prompt="bench")
class ToolActorContextSuite:
"""Benchmark ToolActorContext operations."""
def time_context_construction(self) -> None:
ToolActorContext(plan_id="bench-plan", phase="execute")
def time_record_tool_call(self) -> None:
ctx = ToolActorContext(plan_id="bench-plan")
record = ToolCallRecord(
tool_name="bench/tool",
inputs={"x": 1},
output={"y": 2},
duration_ms=1.0,
)
ctx.record_tool_call(record)
def time_context_summary(self) -> None:
ctx = ToolActorContext(
plan_id="bench-plan",
phase="execute",
sandbox_root="/tmp/sb",
)
ctx.as_summary()
docs/reference/actor_runtime.md
+124
View File
@@ -0,0 +1,124 @@
# Actor Runtime — Tool-Calling Loop
The **ToolCallingRuntime** provides the execution loop for tool-calling
actors. It maps `ToolRegistry` specs to LLM-provider tool schemas,
sends prompts to the LLM, routes tool call requests through
`ToolCallRouter` / `ToolRunner`, and feeds results back to the LLM
until a final response is produced or the iteration limit is reached.
## Architecture
```
ToolCallingRuntime
|-- ToolRegistry (tool spec source)
|-- ToolRunner (4-stage tool execution)
|-- ToolCallRouter (optional; provider format translation)
|-- LLMCaller (LLM invocation protocol)
+-- ToolActorContext (sandbox, resources, history)
```
## Loop Semantics
```
1. Convert ToolRegistry specs -> provider tool schemas
2. Send prompt + tool schemas to LLM via LLMCaller
3. If LLM returns tool calls:
a. Route each call through ToolCallRouter -> ToolRunner
b. Capture metadata (tool name, inputs, outputs, duration, success)
c. Thread sandbox root + resource bindings into tool inputs
d. Feed tool results back to LLM
e. Increment iteration counter; go to step 2
4. If LLM responds without tool calls:
a. Return final response with complete tool call history
5. If max_iterations reached:
a. Return partial result with terminated_by_limit=True
```
## Safety: Max Iterations
The `max_iterations` parameter (default **25**) prevents infinite loops
when the LLM continuously requests tool calls without producing a final
answer. When the limit is reached the runtime returns a
`ToolCallRunResult` with `terminated_by_limit=True` and whatever content
has been accumulated so far.
## Error Semantics
| Error | Condition | Behavior |
|---|---|---|
| `ValueError` | Empty prompt | Raised immediately |
| `TypeError` | Invalid registry/runner type | Raised at construction |
| `ValueError` | `max_iterations < 1` | Raised at construction |
| Tool not found | LLM calls a tool not in registry | `ToolCallRecord` with `success=False` |
| Tool execution error | Handler raises | `ToolCallRecord` with `success=False`, `error` set |
| Max iterations | Loop count exceeds limit | Returns `terminated_by_limit=True` |
## Tool Call Metadata
Every tool call is recorded as a `ToolCallRecord` containing:
| Field | Type | Description |
|---|---|---|
| `tool_name` | `str` | Name of the tool invoked |
| `inputs` | `dict` | Arguments sent to the tool |
| `output` | `dict` | Output returned by the tool |
| `duration_ms` | `float` | Execution time in milliseconds |
| `success` | `bool` | Whether the call succeeded |
| `error` | `str | None` | Error message if failed |
| `iteration` | `int` | Loop iteration when the call was made |
## Sandbox Threading
When a `ToolActorContext` is provided with a `sandbox_root` and/or
`resource_bindings`, these values are threaded into tool inputs as
default keys (`sandbox_root`, `resource_bindings`) so that tools
receive the execution environment context automatically.
## ToolActorContext
The `ToolActorContext` carries:
- **plan_id** — unique plan identifier
- **phase** — current plan phase (e.g. `execute`, `strategize`)
- **sandbox_root** — filesystem path to the sandbox directory
- **automation_profile** — governing automation profile name
- **resource_bindings** — slot-to-resource mappings from the plan
- **project_resources** — additional project resource metadata
- **tool_call_history** — running list of `ToolCallRecord` entries
## Usage Example
```python
from cleveragents.tool import (
ToolCallingRuntime,
ToolRegistry,
ToolRunner,
ToolActorContext,
)
registry = ToolRegistry()
runner = ToolRunner(registry)
# register tools...
runtime = ToolCallingRuntime(
registry=registry,
runner=runner,
llm_caller=my_llm_caller,
max_iterations=10,
)
context = ToolActorContext(
plan_id="plan-123",
phase="execute",
sandbox_root="/tmp/sandbox",
)
result = runtime.run_tool_loop(
prompt="Implement the feature described in the plan.",
context=context,
)
print(result.content)
print(f"Tool calls: {len(result.tool_call_history)}")
print(f"Iterations: {result.iterations}")
```
features/actor_runtime.feature
+225
View File
@@ -0,0 +1,225 @@
Feature: Tool-Calling Actor Runtime
As a developer
I want a tool-calling runtime for execution actors
So that actors can invoke tools through an LLM-driven loop with safety limits
# ---- Single Tool Call ----
Scenario: Tool-call loop with single tool call
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that requests one tool call then responds
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "Use the echo tool"
Then the run result content should be "Final answer after tool call"
And the run result should have 1 tool call records
And the run result iterations should be 2
And the run result should not be terminated by limit
# ---- Multiple Sequential Tool Calls ----
Scenario: Tool-call loop with multiple sequential tool calls
Given a tool registry with an echo tool
And a tool registry with an adder tool
And a tool runner for the actor registry
And a mock LLM caller that requests two sequential tool calls then responds
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "Use both tools"
Then the run result content should be "Done after two tool calls"
And the run result should have 2 tool call records
And the run result iterations should be 3
And the run result should not be terminated by limit
# ---- Max Iteration Safety ----
Scenario: Max-iteration safety termination
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that always requests tool calls
And a tool-calling runtime with max iterations 3
When I run the tool loop with prompt "Keep calling tools"
Then the run result should be terminated by limit
And the run result iterations should be 3
And the run result should have 3 tool call records
# ---- Tool Not Found Error ----
Scenario: Tool call error handling for tool not found
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that requests a nonexistent tool then responds
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "Call missing tool"
Then the run result should have 1 tool call records
And the first tool call record should have success False
And the first tool call record should have an error message
# ---- Tool Execution Error ----
Scenario: Tool call error handling for execution error
Given a tool registry with a failing tool
And a tool runner for the actor registry
And a mock LLM caller that requests the failing tool then responds
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "Call failing tool"
Then the run result should have 1 tool call records
And the first tool call record should have success False
And the first tool call record error should contain "RuntimeError"
# ---- Metadata Capture ----
Scenario: Tool metadata capture verification
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that requests one tool call then responds
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "Echo something"
Then the first tool call record tool name should be "test/echo"
And the first tool call record inputs should not be empty
And the first tool call record output should not be empty
And the first tool call record duration should be greater than 0
And the first tool call record iteration should be 1
# ---- Empty Tool List ----
Scenario: Empty tool list handling
Given an empty actor tool registry
And a tool runner for the actor registry
And a mock LLM caller that responds immediately without tool calls
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "No tools available"
Then the run result content should be "No tools needed"
And the run result should have 0 tool call records
And the run result iterations should be 1
# ---- Sandbox Root Threading ----
Scenario: Sandbox root threading into tool inputs
Given a tool registry with an input-capturing tool
And a tool runner for the actor registry
And a mock LLM caller that requests the capturing tool then responds
And a tool-calling runtime with the mock LLM caller
And an actor context with sandbox root "/tmp/sandbox"
When I run the tool loop with prompt "Use sandbox" and the actor context
Then the captured tool inputs should contain sandbox root "/tmp/sandbox"
# ---- Resource Binding Threading ----
Scenario: Resource bindings threading into tool inputs
Given a tool registry with an input-capturing tool
And a tool runner for the actor registry
And a mock LLM caller that requests the capturing tool then responds
And a tool-calling runtime with the mock LLM caller
And an actor context with resource bindings
When I run the tool loop with prompt "Use resources" and the actor context
Then the captured tool inputs should contain resource bindings
# ---- Constructor Validation ----
Scenario: Runtime rejects invalid max iterations
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that responds immediately without tool calls
Then creating a runtime with max iterations 0 should raise ValueError
Scenario: Runtime rejects empty prompt
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that responds immediately without tool calls
And a tool-calling runtime with the mock LLM caller
Then running the tool loop with empty prompt should raise ValueError
# ---- Actor Context ----
Scenario: ToolActorContext construction and properties
Given an actor context with plan "plan-abc" phase "execute"
Then the actor context plan id should be "plan-abc"
And the actor context phase should be "execute"
And the actor context tool call history should be empty
Scenario: ToolActorContext records tool calls
Given an actor context with plan "plan-rec"
When I record a tool call with name "test/echo" and success True
Then the actor context tool call history should have 1 records
Scenario: ToolActorContext rejects empty plan id
Then creating a tool actor context with empty plan id should raise ValueError
Scenario: ToolActorContext summary includes metadata
Given an actor context with plan "plan-sum" sandbox "/tmp/s"
Then the actor context summary should contain plan id "plan-sum"
And the actor context summary should contain sandbox root "/tmp/s"
# ---- Router Integration ----
Scenario: Tool-call loop with router integration
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a tool call router with plan id "plan-r"
And a mock LLM caller that requests one tool call then responds
And a tool-calling runtime with the mock LLM caller and router
When I run the tool loop with prompt "Use echo with router"
Then the run result content should be "Final answer after tool call"
And the run result should have 1 tool call records
# ---- ToolCallRecord Model ----
Scenario: ToolCallRecord model fields
Given a tool call record with name "test/tool" and duration 42.5
Then the tool call record tool name should be "test/tool"
And the tool call record duration should be 42.5
# ---- Default Context ----
Scenario: Tool loop creates default context when none provided
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that responds immediately without tool calls
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "No context" and no context
Then the run result content should be "No tools needed"
And the run result should have 0 tool call records
# ---- Actor Context Additional Properties ----
Scenario: ToolActorContext exposes all property accessors
Given an actor context with all optional fields
Then the actor context automation profile should be "auto-profile"
And the actor context project resources should have key "repo"
And the actor context metadata should have key "version"
And the actor context clear history should empty the history
Scenario: ToolActorContext rejects non-ToolCallRecord in record_tool_call
Given an actor context with plan "plan-typecheck"
Then recording a non-ToolCallRecord should raise TypeError
# ---- Runtime Constructor Type Validation ----
Scenario: Runtime rejects invalid registry type
Then creating a runtime with a non-ToolRegistry should raise TypeError
Scenario: Runtime rejects invalid runner type
Given a tool registry with an echo tool
Then creating a runtime with a non-ToolRunner should raise TypeError
# ---- Runtime Property Accessors ----
Scenario: Runtime exposes max_iterations and provider_format properties
Given a tool registry with an echo tool
And a tool runner for the actor registry
And a mock LLM caller that responds immediately without tool calls
And a tool-calling runtime with max iterations 5
Then the runtime max iterations should be 5
And the runtime provider format should not be None
# ---- Generic Exception in Tool Execution ----
Scenario: Tool call handles unexpected generic exception
Given a tool registry with a tool that raises a generic exception
And a tool runner for the actor registry
And a mock LLM caller that requests the generic-error tool then responds
And a tool-calling runtime with the mock LLM caller
When I run the tool loop with prompt "Trigger generic error"
Then the run result should have 1 tool call records
And the first tool call record should have success False
And the first tool call record error should contain "KeyError"
features/steps/actor_runtime_steps.py
+710
View File
@@ -0,0 +1,710 @@
"""Step definitions for the Tool-Calling Actor Runtime feature."""
from __future__ import annotations
import contextlib
from typing import Any
from behave import given, then, when
from cleveragents.tool.actor_context import ToolActorContext, ToolCallRecord
from cleveragents.tool.actor_runtime import (
LLMResponse,
LLMToolCall,
ToolCallingRuntime,
ToolCallRunResult,
)
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.router import ToolCallRouter
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
__all__: list[str] = []
# ---------------------------------------------------------------------------
# Mock handlers
# ---------------------------------------------------------------------------
def _rt_echo_handler(inputs: dict[str, Any]) -> dict[str, Any]:
return {"echoed": inputs}
def _rt_adder_handler(inputs: dict[str, Any]) -> dict[str, Any]:
return {"sum": inputs.get("a", 0) + inputs.get("b", 0)}
def _rt_failing_handler(inputs: dict[str, Any]) -> dict[str, Any]:
raise RuntimeError("handler exploded")
def _rt_generic_error_handler(inputs: dict[str, Any]) -> dict[str, Any]:
raise KeyError("missing_key")
_rt_captured_inputs: dict[str, Any] = {}
def _rt_capturing_handler(inputs: dict[str, Any]) -> dict[str, Any]:
_rt_captured_inputs.clear()
_rt_captured_inputs.update(inputs)
return {"captured": True}
# ---------------------------------------------------------------------------
# Mock LLM callers
# ---------------------------------------------------------------------------
class _SingleToolLLM:
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(name="test/echo", arguments={"msg": "hello"})],
)
return LLMResponse(content="Final answer after tool call", tool_calls=[])
class _TwoToolsLLM:
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(name="test/echo", arguments={"msg": "first"})],
)
if self._n == 2:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(name="test/adder", arguments={"a": 1, "b": 2})],
)
return LLMResponse(content="Done after two tool calls", tool_calls=[])
class _AlwaysToolsLLM:
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(name="test/echo", arguments={"x": "loop"})],
)
class _NonexistentToolLLM:
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(name="test/nonexistent", arguments={"x": 1})],
)
return LLMResponse(content="Done after error", tool_calls=[])
class _FailingToolLLM:
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(name="test/fail", arguments={})],
)
return LLMResponse(content="Done after failing tool", tool_calls=[])
class _ImmediateLLM:
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
return LLMResponse(content="No tools needed", tool_calls=[])
class _CapturingToolLLM:
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
content="",
tool_calls=[
LLMToolCall(name="test/capture", arguments={"data": "test"})
],
)
return LLMResponse(content="Done after capture", tool_calls=[])
class _GenericErrorToolLLM:
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(name="test/generic_error", arguments={"x": 1})],
)
return LLMResponse(content="Done after generic error", tool_calls=[])
# ---------------------------------------------------------------------------
# Given steps — registry & runner
# ---------------------------------------------------------------------------
@given("a tool registry with an echo tool")
def step_given_registry_echo(context: Any) -> None:
if not hasattr(context, "registry"):
context.registry = ToolRegistry()
with contextlib.suppress(Exception):
context.registry.register(
ToolSpec(
name="test/echo", description="Echo tool", handler=_rt_echo_handler
)
)
@given("a tool registry with an adder tool")
def step_given_registry_adder(context: Any) -> None:
if not hasattr(context, "registry"):
context.registry = ToolRegistry()
with contextlib.suppress(Exception):
context.registry.register(
ToolSpec(
name="test/adder", description="Adder tool", handler=_rt_adder_handler
)
)
@given("a tool registry with a failing tool")
def step_given_registry_failing(context: Any) -> None:
if not hasattr(context, "registry"):
context.registry = ToolRegistry()
with contextlib.suppress(Exception):
context.registry.register(
ToolSpec(
name="test/fail",
description="Failing tool",
handler=_rt_failing_handler,
)
)
@given("a tool registry with an input-capturing tool")
def step_given_registry_capturing(context: Any) -> None:
if not hasattr(context, "registry"):
context.registry = ToolRegistry()
with contextlib.suppress(Exception):
context.registry.register(
ToolSpec(
name="test/capture",
description="Capturing tool",
handler=_rt_capturing_handler,
)
)
@given("a tool registry with a tool that raises a generic exception")
def step_given_registry_generic_error(context: Any) -> None:
if not hasattr(context, "registry"):
context.registry = ToolRegistry()
with contextlib.suppress(Exception):
context.registry.register(
ToolSpec(
name="test/generic_error",
description="Generic error tool",
handler=_rt_generic_error_handler,
)
)
@given("an empty actor tool registry")
def step_given_empty_registry(context: Any) -> None:
context.registry = ToolRegistry()
@given("a tool runner for the actor registry")
def step_given_runner(context: Any) -> None:
context.runner = ToolRunner(context.registry)
# ---------------------------------------------------------------------------
# Given steps — mock LLM callers
# ---------------------------------------------------------------------------
@given("a mock LLM caller that requests one tool call then responds")
def step_given_mock_llm_single(context: Any) -> None:
context.llm_caller = _SingleToolLLM()
@given("a mock LLM caller that requests two sequential tool calls then responds")
def step_given_mock_llm_two(context: Any) -> None:
context.llm_caller = _TwoToolsLLM()
@given("a mock LLM caller that always requests tool calls")
def step_given_mock_llm_always(context: Any) -> None:
context.llm_caller = _AlwaysToolsLLM()
@given("a mock LLM caller that requests a nonexistent tool then responds")
def step_given_mock_llm_nonexistent(context: Any) -> None:
context.llm_caller = _NonexistentToolLLM()
@given("a mock LLM caller that requests the failing tool then responds")
def step_given_mock_llm_failing(context: Any) -> None:
context.llm_caller = _FailingToolLLM()
@given("a mock LLM caller that responds immediately without tool calls")
def step_given_mock_llm_immediate(context: Any) -> None:
context.llm_caller = _ImmediateLLM()
@given("a mock LLM caller that requests the capturing tool then responds")
def step_given_mock_llm_capturing(context: Any) -> None:
context.llm_caller = _CapturingToolLLM()
@given("a mock LLM caller that requests the generic-error tool then responds")
def step_given_mock_llm_generic_error(context: Any) -> None:
context.llm_caller = _GenericErrorToolLLM()
# ---------------------------------------------------------------------------
# Given steps — runtime construction
# ---------------------------------------------------------------------------
@given("a tool-calling runtime with the mock LLM caller")
def step_given_runtime(context: Any) -> None:
context.runtime = ToolCallingRuntime(
registry=context.registry,
runner=context.runner,
llm_caller=context.llm_caller,
)
@given("a tool-calling runtime with max iterations {n:d}")
def step_given_runtime_max_iter(context: Any, n: int) -> None:
context.runtime = ToolCallingRuntime(
registry=context.registry,
runner=context.runner,
llm_caller=context.llm_caller,
max_iterations=n,
)
@given("a tool-calling runtime with the mock LLM caller and router")
def step_given_runtime_with_router(context: Any) -> None:
context.runtime = ToolCallingRuntime(
registry=context.registry,
runner=context.runner,
llm_caller=context.llm_caller,
router=context.router,
)
# ---------------------------------------------------------------------------
# Given steps — actor context
# ---------------------------------------------------------------------------
@given('an actor context with plan "{plan_id}" phase "{phase}"')
def step_given_actor_ctx_plan_phase(context: Any, plan_id: str, phase: str) -> None:
context.actor_context = ToolActorContext(plan_id=plan_id, phase=phase)
@given('an actor context with plan "{plan_id}" sandbox "{root}"')
def step_given_actor_ctx_plan_sandbox(context: Any, plan_id: str, root: str) -> None:
context.actor_context = ToolActorContext(plan_id=plan_id, sandbox_root=root)
@given('an actor context with plan "{plan_id}"')
def step_given_actor_ctx_plan(context: Any, plan_id: str) -> None:
context.actor_context = ToolActorContext(plan_id=plan_id)
@given('an actor context with sandbox root "{root}"')
def step_given_actor_ctx_sandbox(context: Any, root: str) -> None:
context.actor_context = ToolActorContext(
plan_id="plan-test",
phase="execute",
sandbox_root=root,
)
@given("an actor context with resource bindings")
def step_given_actor_ctx_bindings(context: Any) -> None:
context.actor_context = ToolActorContext(
plan_id="plan-test",
phase="execute",
resource_bindings={"source": {"type": "git-checkout", "path": "/repo"}},
)
@given("an actor context with all optional fields")
def step_given_actor_ctx_all_fields(context: Any) -> None:
context.actor_context = ToolActorContext(
plan_id="plan-full",
phase="execute",
sandbox_root="/tmp/sand",
automation_profile="auto-profile",
resource_bindings={"src": {"path": "/src"}},
project_resources={"repo": "/home/repo"},
metadata={"version": "1.0"},
)
# Pre-add a record so we can test clear_history
context.actor_context.record_tool_call(
ToolCallRecord(
tool_name="test/dummy",
success=True,
duration_ms=1.0,
iteration=1,
)
)
@given('a tool call router with plan id "{plan_id}"')
def step_given_router(context: Any, plan_id: str) -> None:
context.router = ToolCallRouter(
registry=context.registry,
runner=context.runner,
plan_id=plan_id,
)
@given('a tool call record with name "{name}" and duration {duration:g}')
def step_given_tool_call_record(context: Any, name: str, duration: float) -> None:
context.tool_call_record = ToolCallRecord(tool_name=name, duration_ms=duration)
# ---------------------------------------------------------------------------
# When steps
# ---------------------------------------------------------------------------
@when('I run the tool loop with prompt "{prompt}"')
def step_when_run_loop(context: Any, prompt: str) -> None:
context.run_result = context.runtime.run_tool_loop(prompt=prompt)
@when('I run the tool loop with prompt "{prompt}" and the actor context')
def step_when_run_loop_with_context(context: Any, prompt: str) -> None:
context.run_result = context.runtime.run_tool_loop(
prompt=prompt,
context=context.actor_context,
)
@when('I run the tool loop with prompt "{prompt}" and no context')
def step_when_run_loop_no_context(context: Any, prompt: str) -> None:
context.run_result = context.runtime.run_tool_loop(prompt=prompt, context=None)
@when('I record a tool call with name "{name}" and success {success}')
def step_when_record_tool_call(context: Any, name: str, success: str) -> None:
record = ToolCallRecord(tool_name=name, success=success == "True")
context.actor_context.record_tool_call(record)
# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------
@then('the run result content should be "{expected}"')
def step_then_content(context: Any, expected: str) -> None:
result: ToolCallRunResult = context.run_result
assert result.content == expected, f"Expected '{expected}', got '{result.content}'"
@then("the run result should have {count:d} tool call records")
def step_then_tool_call_count(context: Any, count: int) -> None:
result: ToolCallRunResult = context.run_result
actual = len(result.tool_call_history)
assert actual == count, f"Expected {count} records, got {actual}"
@then("the run result iterations should be {expected:d}")
def step_then_iterations(context: Any, expected: int) -> None:
result: ToolCallRunResult = context.run_result
assert result.iterations == expected, (
f"Expected {expected}, got {result.iterations}"
)
@then("the run result should not be terminated by limit")
def step_then_not_terminated(context: Any) -> None:
assert not context.run_result.terminated_by_limit
@then("the run result should be terminated by limit")
def step_then_terminated(context: Any) -> None:
assert context.run_result.terminated_by_limit
@then("the first tool call record should have success {expected}")
def step_then_first_record_success(context: Any, expected: str) -> None:
record = context.run_result.tool_call_history[0]
assert record.success == (expected == "True")
@then("the first tool call record should have an error message")
def step_then_first_record_has_error(context: Any) -> None:
record = context.run_result.tool_call_history[0]
assert record.error is not None and len(record.error) > 0
@then('the first tool call record error should contain "{fragment}"')
def step_then_first_record_error_contains(context: Any, fragment: str) -> None:
record = context.run_result.tool_call_history[0]
assert record.error is not None
assert fragment in record.error, f"Expected '{fragment}' in '{record.error}'"
@then('the first tool call record tool name should be "{name}"')
def step_then_first_record_name(context: Any, name: str) -> None:
assert context.run_result.tool_call_history[0].tool_name == name
@then("the first tool call record inputs should not be empty")
def step_then_first_record_inputs(context: Any) -> None:
assert len(context.run_result.tool_call_history[0].inputs) > 0
@then("the first tool call record output should not be empty")
def step_then_first_record_output(context: Any) -> None:
assert len(context.run_result.tool_call_history[0].output) > 0
@then("the first tool call record duration should be greater than 0")
def step_then_first_record_duration(context: Any) -> None:
assert context.run_result.tool_call_history[0].duration_ms >= 0.0
@then("the first tool call record iteration should be {expected:d}")
def step_then_first_record_iteration(context: Any, expected: int) -> None:
assert context.run_result.tool_call_history[0].iteration == expected
@then('the captured tool inputs should contain sandbox root "{root}"')
def step_then_captured_sandbox(context: Any, root: str) -> None:
assert _rt_captured_inputs.get("sandbox_root") == root
@then("the captured tool inputs should contain resource bindings")
def step_then_captured_bindings(context: Any) -> None:
assert isinstance(_rt_captured_inputs.get("resource_bindings"), dict)
@then("creating a runtime with max iterations 0 should raise ValueError")
def step_then_invalid_max_iter(context: Any) -> None:
try:
ToolCallingRuntime(
registry=context.registry,
runner=context.runner,
llm_caller=context.llm_caller,
max_iterations=0,
)
raise AssertionError("Expected ValueError")
except ValueError:
pass
@then("running the tool loop with empty prompt should raise ValueError")
def step_then_empty_prompt(context: Any) -> None:
try:
context.runtime.run_tool_loop(prompt="")
raise AssertionError("Expected ValueError")
except ValueError:
pass
@then('the actor context plan id should be "{expected}"')
def step_then_actor_plan_id(context: Any, expected: str) -> None:
assert context.actor_context.plan_id == expected
@then('the actor context phase should be "{expected}"')
def step_then_actor_phase(context: Any, expected: str) -> None:
assert context.actor_context.phase == expected
@then("the actor context tool call history should be empty")
def step_then_actor_history_empty(context: Any) -> None:
assert len(context.actor_context.tool_call_history) == 0
@then("the actor context tool call history should have {count:d} records")
def step_then_actor_history_count(context: Any, count: int) -> None:
assert len(context.actor_context.tool_call_history) == count
@then("creating a tool actor context with empty plan id should raise ValueError")
def step_then_empty_plan_id(context: Any) -> None:
try:
ToolActorContext(plan_id="")
raise AssertionError("Expected ValueError")
except ValueError:
pass
@then('the actor context summary should contain plan id "{plan_id}"')
def step_then_summary_plan_id(context: Any, plan_id: str) -> None:
assert context.actor_context.as_summary()["plan_id"] == plan_id
@then('the actor context summary should contain sandbox root "{root}"')
def step_then_summary_sandbox(context: Any, root: str) -> None:
assert context.actor_context.as_summary()["sandbox_root"] == root
@then('the tool call record tool name should be "{name}"')
def step_then_record_name(context: Any, name: str) -> None:
assert context.tool_call_record.tool_name == name
@then("the tool call record duration should be {expected:g}")
def step_then_record_duration(context: Any, expected: float) -> None:
assert context.tool_call_record.duration_ms == expected
# -- Actor Context additional property steps --
@then('the actor context automation profile should be "{expected}"')
def step_then_actor_automation_profile(context: Any, expected: str) -> None:
assert context.actor_context.automation_profile == expected
@then('the actor context project resources should have key "{key}"')
def step_then_actor_project_resources_key(context: Any, key: str) -> None:
assert key in context.actor_context.project_resources
@then('the actor context metadata should have key "{key}"')
def step_then_actor_metadata_key(context: Any, key: str) -> None:
assert key in context.actor_context.metadata
@then("the actor context clear history should empty the history")
def step_then_actor_clear_history(context: Any) -> None:
assert len(context.actor_context.tool_call_history) > 0
context.actor_context.clear_history()
assert len(context.actor_context.tool_call_history) == 0
@then("recording a non-ToolCallRecord should raise TypeError")
def step_then_invalid_record_type(context: Any) -> None:
try:
context.actor_context.record_tool_call("not a record") # type: ignore[arg-type]
raise AssertionError("Expected TypeError")
except TypeError:
pass
# -- Runtime constructor type validation steps --
@then("creating a runtime with a non-ToolRegistry should raise TypeError")
def step_then_invalid_registry_type(context: Any) -> None:
try:
ToolCallingRuntime(
registry="not a registry", # type: ignore[arg-type]
runner=ToolRunner(ToolRegistry()),
llm_caller=_ImmediateLLM(),
)
raise AssertionError("Expected TypeError")
except TypeError:
pass
@then("creating a runtime with a non-ToolRunner should raise TypeError")
def step_then_invalid_runner_type(context: Any) -> None:
try:
ToolCallingRuntime(
registry=context.registry,
runner="not a runner", # type: ignore[arg-type]
llm_caller=_ImmediateLLM(),
)
raise AssertionError("Expected TypeError")
except TypeError:
pass
# -- Runtime property accessors steps --
@then("the runtime max iterations should be {expected:d}")
def step_then_runtime_max_iter(context: Any, expected: int) -> None:
assert context.runtime.max_iterations == expected
@then("the runtime provider format should not be None")
def step_then_runtime_provider_format(context: Any) -> None:
assert context.runtime.provider_format is not None
implementation_plan.md
+18 -18
View File
@@ -2211,24 +2211,24 @@ This section replaces all unfinished work items with the Day 14 plan. Work is or
#### M1: Minimal Local Source-Code Workflow (Target: Day 7, recovery path)
**Parallel Group M1: Tool-Aware Execute + Sandbox Apply**
- [ ] **COMMIT (Owner: Jeff | Group: M1.actor-runtime | Branch: feature/m1-actor-runtime | Planned: Day 14 | Expected: Day 16) - Commit message: "feat(actor): add tool-calling runtime for execution actors"**
- [ ] Git [Jeff]: `git checkout master`
- [ ] Git [Jeff]: `git pull origin master`
- [ ] Git [Jeff]: `git checkout -b feature/m1-actor-runtime`
- [ ] Code [Jeff]: Add tool-calling loop that maps ToolRegistry specs to provider tool schemas and executes through ToolRunner with max-iteration safeguards.
- [ ] Code [Jeff]: Capture tool call metadata (tool name, inputs, outputs, duration, success) into a structured record for downstream ChangeSet/decision usage.
- [ ] Code [Jeff]: Thread sandbox root + resource bindings into tool inputs (default to plan/project resources).
- [ ] Docs [Jeff]: Update `docs/reference/actor_runtime.md` with the tool-call loop and error semantics.
- [ ] Git [Jeff]: `git fetch origin && git merge origin/master` (run before final tests and before commit)
- [ ] Tests (Behave) [Jeff]: Add scenarios for tool-call loop, error handling, and max-iteration termination.
- [ ] Tests (Robot) [Jeff]: Add Robot smoke test running `agents actor run` with a tool-calling actor against a temp repo.
- [ ] Tests (ASV) [Jeff]: Add `benchmarks/actor_runtime_bench.py` for tool-call loop overhead.
- [ ] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%.
- [ ] Quality [Jeff]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes across **entire** code base, do not ignore any failure even if it seems unrelated to this commit, fix it.
- [ ] Git [Jeff]: Perform an appropriate `git add` command to add all the files that should be part of this commit to the git index
- [ ] Git [Jeff]: `git commit -m "feat(actor): add tool-calling runtime for execution actors"`
- [ ] Git [Jeff]: `git push -u origin feature/m1-actor-runtime`
- [ ] Forgejo PR [Jeff]: Open PR from `feature/m1-actor-runtime` to `master` with a suitable and thorough description
- [X] **COMMIT (Owner: Jeff | Group: M1.actor-runtime | Branch: feature/m1-actor-runtime | Planned: Day 14 | Expected: Day 16) - Commit message: "feat(actor): add tool-calling runtime for execution actors"**
- [X] Git [Jeff]: `git checkout master`
- [X] Git [Jeff]: `git pull origin master`
- [X] Git [Jeff]: `git checkout -b feature/m1-actor-runtime`
- [X] Code [Jeff]: Add tool-calling loop that maps ToolRegistry specs to provider tool schemas and executes through ToolRunner with max-iteration safeguards.
- [X] Code [Jeff]: Capture tool call metadata (tool name, inputs, outputs, duration, success) into a structured record for downstream ChangeSet/decision usage.
- [X] Code [Jeff]: Thread sandbox root + resource bindings into tool inputs (default to plan/project resources).
- [X] Docs [Jeff]: Update `docs/reference/actor_runtime.md` with the tool-call loop and error semantics.
- [X] Git [Jeff]: `git fetch origin && git merge origin/master` (run before final tests and before commit)
- [X] Tests (Behave) [Jeff]: Add scenarios for tool-call loop, error handling, and max-iteration termination.
- [X] Tests (Robot) [Jeff]: Add Robot smoke test running `agents actor run` with a tool-calling actor against a temp repo.
- [X] Tests (ASV) [Jeff]: Add `benchmarks/actor_runtime_bench.py` for tool-call loop overhead.
- [X] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%.
- [X] Quality [Jeff]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes across **entire** code base, do not ignore any failure even if it seems unrelated to this commit, fix it.
- [X] Git [Jeff]: Perform an appropriate `git add` command to add all the files that should be part of this commit to the git index
- [X] Git [Jeff]: `git commit -m "feat(actor): add tool-calling runtime for execution actors"`
- [X] Git [Jeff]: `git push -u origin feature/m1-actor-runtime`
- [X] Forgejo PR [Jeff]: Open PR from `feature/m1-actor-runtime` to `master` with a suitable and thorough description (PR #141)
- [ ] **COMMIT (Owner: Jeff | Group: M1.plan-execute | Branch: feature/m1-plan-execute-runtime | Planned: Day 15 | Expected: Day 17) - Commit message: "feat(plan): wire execute phase to actor runtime and changeset capture"**
- [ ] Git [Jeff]: `git checkout master`
robot/actor_runtime.robot
+45
View File
@@ -0,0 +1,45 @@
*** Settings ***
Documentation Smoke tests for tool-calling actor runtime
Resource ${CURDIR}/common.resource
Suite Setup Setup Test Environment
Suite Teardown Cleanup Test Environment
*** Variables ***
${HELPER_SCRIPT} robot/helper_actor_runtime.py
*** Test Cases ***
Actor Runtime Single Tool Call
[Documentation] Run a tool-call loop with a single tool call
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} single_tool cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} single-tool-ok
Actor Runtime Max Iterations
[Documentation] Verify max-iteration safety termination
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} max_iterations cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} max-iterations-ok
Actor Runtime Error Handling
[Documentation] Verify tool error handling in the loop
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} error_handling cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} error-handling-ok
Actor Runtime Context Properties
[Documentation] Verify ToolActorContext properties
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} context_props cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} context-props-ok
Actor Runtime Sandbox Threading
[Documentation] Verify sandbox root threading into tool inputs
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} sandbox_thread cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} sandbox-thread-ok
Actor Runtime Empty Registry
[Documentation] Verify behavior with no tools registered
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} empty_registry cwd=${WORKSPACE}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} empty-registry-ok
robot/helper_actor_runtime.py
+229
View File
@@ -0,0 +1,229 @@
"""Helper script for Robot Framework actor runtime smoke tests."""
from __future__ import annotations
import sys
from pathlib import Path
from typing import Any
# Ensure src is importable when run from workspace root
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from cleveragents.tool.actor_context import ToolActorContext
from cleveragents.tool.actor_runtime import (
LLMResponse,
LLMToolCall,
ToolCallingRuntime,
)
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
def _echo(inputs: dict[str, Any]) -> dict[str, Any]:
return {"echoed": inputs}
def _fail(inputs: dict[str, Any]) -> dict[str, Any]:
raise RuntimeError("boom")
_captured: dict[str, Any] = {}
def _capture(inputs: dict[str, Any]) -> dict[str, Any]:
_captured.clear()
_captured.update(inputs)
return {"ok": True}
class SingleToolLLM:
"""Requests one tool call then returns."""
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
tool_calls=[LLMToolCall(name="test/echo", arguments={"a": 1})]
)
return LLMResponse(content="done")
class AlwaysToolLLM:
"""Always requests tool calls."""
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
return LLMResponse(
tool_calls=[LLMToolCall(name="test/echo", arguments={"x": 1})]
)
class FailToolLLM:
"""Requests failing tool then stops."""
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
tool_calls=[LLMToolCall(name="test/fail", arguments={})]
)
return LLMResponse(content="handled error")
class CaptureToolLLM:
"""Requests capturing tool then stops."""
def __init__(self) -> None:
self._n = 0
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
self._n += 1
if self._n == 1:
return LLMResponse(
tool_calls=[LLMToolCall(name="test/capture", arguments={"d": 1})]
)
return LLMResponse(content="captured")
class ImmediateLLM:
"""Returns immediately without tool calls."""
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
return LLMResponse(content="immediate")
def _make_registry_with_echo() -> ToolRegistry:
reg = ToolRegistry()
reg.register(ToolSpec(name="test/echo", description="Echo", handler=_echo))
return reg
def test_single_tool() -> None:
reg = _make_registry_with_echo()
runner = ToolRunner(reg)
rt = ToolCallingRuntime(
registry=reg, runner=runner, llm_caller=SingleToolLLM()
)
result = rt.run_tool_loop(prompt="go")
assert result.content == "done"
assert len(result.tool_call_history) == 1
assert not result.terminated_by_limit
print("single-tool-ok")
def test_max_iterations() -> None:
reg = _make_registry_with_echo()
runner = ToolRunner(reg)
rt = ToolCallingRuntime(
registry=reg, runner=runner, llm_caller=AlwaysToolLLM(), max_iterations=3
)
result = rt.run_tool_loop(prompt="loop")
assert result.terminated_by_limit
assert result.iterations == 3
print("max-iterations-ok")
def test_error_handling() -> None:
reg = ToolRegistry()
reg.register(ToolSpec(name="test/fail", description="Fail", handler=_fail))
runner = ToolRunner(reg)
rt = ToolCallingRuntime(
registry=reg, runner=runner, llm_caller=FailToolLLM()
)
result = rt.run_tool_loop(prompt="fail")
assert len(result.tool_call_history) == 1
assert not result.tool_call_history[0].success
assert result.tool_call_history[0].error is not None
print("error-handling-ok")
def test_context_props() -> None:
ctx = ToolActorContext(
plan_id="p1", phase="execute", sandbox_root="/sb"
)
assert ctx.plan_id == "p1"
assert ctx.phase == "execute"
assert ctx.sandbox_root == "/sb"
assert len(ctx.tool_call_history) == 0
summary = ctx.as_summary()
assert summary["plan_id"] == "p1"
print("context-props-ok")
def test_sandbox_thread() -> None:
reg = ToolRegistry()
reg.register(
ToolSpec(name="test/capture", description="Capture", handler=_capture)
)
runner = ToolRunner(reg)
ctx = ToolActorContext(
plan_id="p2", phase="execute", sandbox_root="/sandbox/path"
)
rt = ToolCallingRuntime(
registry=reg, runner=runner, llm_caller=CaptureToolLLM()
)
rt.run_tool_loop(prompt="capture", context=ctx)
assert _captured.get("sandbox_root") == "/sandbox/path"
print("sandbox-thread-ok")
def test_empty_registry() -> None:
reg = ToolRegistry()
runner = ToolRunner(reg)
rt = ToolCallingRuntime(
registry=reg, runner=runner, llm_caller=ImmediateLLM()
)
result = rt.run_tool_loop(prompt="nothing")
assert result.content == "immediate"
assert len(result.tool_call_history) == 0
print("empty-registry-ok")
COMMANDS = {
"single_tool": test_single_tool,
"max_iterations": test_max_iterations,
"error_handling": test_error_handling,
"context_props": test_context_props,
"sandbox_thread": test_sandbox_thread,
"empty_registry": test_empty_registry,
}
if __name__ == "__main__":
cmd = sys.argv[1] if len(sys.argv) > 1 else "single_tool"
COMMANDS[cmd]()
src/cleveragents/tool/__init__.py
+19 -1
View File
@@ -4,9 +4,19 @@ Re-exports public types from the runtime, runner, and registry modules
(C0.runtime) and the four-stage tool lifecycle with capability enforcement,
JSON Schema validation, per-plan activation caching, execution tracing,
and cancellation propagation (C5.lifecycle). Also re-exports the
tool call router for provider format translation (C5.router).
tool call router for provider format translation (C5.router) and the
tool-calling actor runtime for execution actors (M1.actor-runtime).
"""
from cleveragents.tool.actor_context import ToolActorContext, ToolCallRecord
from cleveragents.tool.actor_runtime import (
LLMCaller,
LLMResponse,
LLMToolCall,
MaxIterationsExceededError,
ToolCallingRuntime,
ToolCallRunResult,
)
from cleveragents.tool.context import (
BoundResource,
CancellationToken,
@@ -60,16 +70,24 @@ __all__ = [
"CancellationToken",
"Change",
"ChangeOperation",
"LLMCaller",
"LLMResponse",
"LLMToolCall",
"LifecycleToolResult",
"MaxIterationsExceededError",
"NormalizedToolCallResult",
"ProviderFormat",
"StreamingStatus",
"StreamingToolUpdate",
"ToolAccessDeniedError",
"ToolActivationError",
"ToolActorContext",
"ToolCallErrorCategory",
"ToolCallRecord",
"ToolCallRequest",
"ToolCallRouter",
"ToolCallRunResult",
"ToolCallingRuntime",
"ToolCancelledError",
"ToolCheckpointRequiredError",
"ToolDeactivationError",
src/cleveragents/tool/actor_context.py
+165
View File
@@ -0,0 +1,165 @@
"""Tool actor context for CleverAgents execution actors.
The ``ToolActorContext`` carries all the state needed for a tool-calling
actor execution loop: project resources, sandbox configuration, plan
metadata, resource bindings, and a running history of tool calls made
during the current run.
Based on ``docs/specification.md`` and ``implementation_plan.md`` task
M1.actor-runtime.
"""
from __future__ import annotations
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
class ToolCallRecord(BaseModel):
"""Structured record of a single tool call during an actor run.
Captures the tool name, inputs sent, outputs received, timing, and
success status for downstream ChangeSet/decision usage and audit.
"""
tool_name: str = Field(..., min_length=1, description="Name of the tool invoked")
inputs: dict[str, Any] = Field(
default_factory=dict, description="Arguments sent to the tool"
)
output: dict[str, Any] = Field(
default_factory=dict, description="Output returned by the tool"
)
duration_ms: float = Field(0.0, ge=0.0, description="Execution time (ms)")
success: bool = Field(True, description="Whether the call succeeded")
error: str | None = Field(
default=None, description="Error message if the call failed"
)
iteration: int = Field(0, ge=0, description="Loop iteration when the call was made")
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
class ToolActorContext:
"""Context carrier for a tool-calling actor execution.
Holds project resources, sandbox root, plan metadata, resource
bindings from the plan's projects, and the running tool call
history for the current run.
Parameters
----------
plan_id:
Unique identifier for the plan that owns this execution.
phase:
Current plan phase (e.g. ``"execute"``, ``"strategize"``).
sandbox_root:
Filesystem path to the sandbox root directory.
automation_profile:
Name of the automation profile governing this execution.
resource_bindings:
Mapping of slot name to resource metadata from the plan's
projects.
project_resources:
Additional project resource metadata.
metadata:
Arbitrary extra metadata.
"""
def __init__(
self,
*,
plan_id: str,
phase: str = "",
sandbox_root: str = "",
automation_profile: str = "",
resource_bindings: dict[str, Any] | None = None,
project_resources: dict[str, Any] | None = None,
metadata: dict[str, Any] | None = None,
) -> None:
if not plan_id:
raise ValueError("plan_id must not be empty")
self._plan_id = plan_id
self._phase = phase
self._sandbox_root = sandbox_root
self._automation_profile = automation_profile
self._resource_bindings: dict[str, Any] = (
dict(resource_bindings) if resource_bindings else {}
)
self._project_resources: dict[str, Any] = (
dict(project_resources) if project_resources else {}
)
self._metadata: dict[str, Any] = dict(metadata) if metadata else {}
self._tool_call_history: list[ToolCallRecord] = []
# -- Properties -----------------------------------------------------------
@property
def plan_id(self) -> str:
"""Return the plan ID."""
return self._plan_id
@property
def phase(self) -> str:
"""Return the current plan phase."""
return self._phase
@property
def sandbox_root(self) -> str:
"""Return the sandbox root path."""
return self._sandbox_root
@property
def automation_profile(self) -> str:
"""Return the automation profile name."""
return self._automation_profile
@property
def resource_bindings(self) -> dict[str, Any]:
"""Return a copy of the resource bindings."""
return dict(self._resource_bindings)
@property
def project_resources(self) -> dict[str, Any]:
"""Return a copy of the project resources."""
return dict(self._project_resources)
@property
def metadata(self) -> dict[str, Any]:
"""Return a copy of the metadata."""
return dict(self._metadata)
@property
def tool_call_history(self) -> list[ToolCallRecord]:
"""Return the tool call history (read-only copy)."""
return list(self._tool_call_history)
# -- Mutators -------------------------------------------------------------
def record_tool_call(self, record: ToolCallRecord) -> None:
"""Append a tool call record to the history."""
if not isinstance(record, ToolCallRecord):
raise TypeError("record must be a ToolCallRecord")
self._tool_call_history.append(record)
def clear_history(self) -> None:
"""Clear the tool call history."""
self._tool_call_history.clear()
# -- Summary --------------------------------------------------------------
def as_summary(self) -> dict[str, Any]:
"""Return a summary dict for diagnostics."""
return {
"plan_id": self._plan_id,
"phase": self._phase,
"sandbox_root": self._sandbox_root,
"automation_profile": self._automation_profile,
"resource_binding_count": len(self._resource_bindings),
"project_resource_count": len(self._project_resources),
"tool_call_count": len(self._tool_call_history),
}
src/cleveragents/tool/actor_runtime.py
+451
View File
@@ -0,0 +1,451 @@
"""Tool-calling actor runtime for CleverAgents execution actors.
Implements the core tool-call loop that maps ``ToolRegistry`` specs to
LLM provider tool schemas and executes tool calls through
``ToolCallRouter`` / ``ToolRunner`` with iteration safeguards.
## Loop Semantics
```
1. Convert ToolRegistry specs -> provider tool schemas
2. Send prompt + tool schemas to LLM
3. If LLM returns tool calls:
a. Route each call through ToolCallRouter -> ToolRunner
b. Capture metadata (name, inputs, outputs, duration, success)
c. Thread sandbox root + resource bindings into tool inputs
d. Feed tool results back to LLM
e. Repeat from step 2
4. If LLM responds without tool calls or max_iterations reached:
a. Return final response
```
## Safety
- ``max_iterations`` (default 25) prevents infinite loops
- Each iteration is tracked; exceeding the limit produces a
``MaxIterationsExceededError``
Based on ``implementation_plan.md`` task M1.actor-runtime.
"""
from __future__ import annotations
import logging
import time
from typing import Any, Protocol, runtime_checkable
from pydantic import BaseModel, ConfigDict, Field
from cleveragents.tool.actor_context import ToolActorContext, ToolCallRecord
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.router import (
ProviderFormat,
ToolCallRouter,
normalize_tool_schema_for_provider,
)
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolError
logger = logging.getLogger(__name__)
_DEFAULT_MAX_ITERATIONS = 25
# ---------------------------------------------------------------------------
# Errors
# ---------------------------------------------------------------------------
class MaxIterationsExceededError(Exception):
"""Raised when the tool-call loop exceeds the configured iteration limit."""
# ---------------------------------------------------------------------------
# LLM response model
# ---------------------------------------------------------------------------
class LLMToolCall(BaseModel):
"""A single tool call requested by the LLM."""
name: str = Field(..., min_length=1, description="Tool name to invoke")
arguments: dict[str, Any] = Field(
default_factory=dict, description="Arguments for the tool"
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
class LLMResponse(BaseModel):
"""Structured response from an LLM invocation.
If ``tool_calls`` is non-empty the LLM is requesting tool executions.
Otherwise ``content`` contains the final text response.
"""
content: str = Field("", description="Text content of the LLM response")
tool_calls: list[LLMToolCall] = Field(
default_factory=list, description="Tool calls requested by the LLM"
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
# ---------------------------------------------------------------------------
# LLM caller protocol
# ---------------------------------------------------------------------------
@runtime_checkable
class LLMCaller(Protocol):
"""Protocol for invoking an LLM with tool schemas.
Implementations wrap a concrete LLM provider (LangChain, direct API,
or a test double) and translate between the provider's native format
and the ``LLMResponse`` model.
"""
def invoke(
self,
prompt: str,
tool_schemas: list[dict[str, Any]],
tool_results: list[dict[str, Any]] | None = None,
actor_config: dict[str, Any] | None = None,
) -> LLMResponse:
"""Send a prompt (with optional tool results) to the LLM.
Parameters
----------
prompt:
The user/system prompt text.
tool_schemas:
Provider-formatted tool schema dicts.
tool_results:
Results from prior tool calls to feed back to the LLM.
actor_config:
Actor configuration (provider, model, options).
Returns
-------
LLMResponse:
The LLM's response, potentially containing tool calls.
"""
...
# ---------------------------------------------------------------------------
# Run result
# ---------------------------------------------------------------------------
class ToolCallRunResult(BaseModel):
"""Result of a complete tool-call loop execution.
Contains the final LLM response content, the complete tool call
history, iteration count, and whether the loop was terminated by
the iteration safety limit.
"""
content: str = Field("", description="Final LLM response text")
tool_call_history: list[ToolCallRecord] = Field(
default_factory=list, description="All tool calls made during the run"
)
iterations: int = Field(0, ge=0, description="Number of loop iterations")
terminated_by_limit: bool = Field(
False, description="Whether max_iterations was reached"
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
# ---------------------------------------------------------------------------
# ToolCallingRuntime
# ---------------------------------------------------------------------------
class ToolCallingRuntime:
"""Tool-calling runtime for execution actors.
Orchestrates the tool-call loop: converts registry specs to provider
schemas, sends prompts to the LLM, routes tool calls through the
runner, and repeats until the LLM produces a final response or the
iteration limit is reached.
Parameters
----------
registry:
The ``ToolRegistry`` containing available tool specs.
runner:
The ``ToolRunner`` for executing tools.
llm_caller:
An ``LLMCaller`` implementation for invoking the LLM.
router:
Optional ``ToolCallRouter`` for provider format translation.
When not supplied, tool calls are executed directly via the runner.
max_iterations:
Maximum number of tool-call loop iterations (default 25).
provider_format:
Target provider format for schema export (default ``LANGCHAIN``).
"""
def __init__(
self,
*,
registry: ToolRegistry,
runner: ToolRunner,
llm_caller: LLMCaller,
router: ToolCallRouter | None = None,
max_iterations: int = _DEFAULT_MAX_ITERATIONS,
provider_format: ProviderFormat = ProviderFormat.LANGCHAIN,
) -> None:
if not isinstance(registry, ToolRegistry):
raise TypeError("registry must be a ToolRegistry")
if not isinstance(runner, ToolRunner):
raise TypeError("runner must be a ToolRunner")
if max_iterations < 1:
raise ValueError("max_iterations must be at least 1")
self._registry = registry
self._runner = runner
self._llm_caller = llm_caller
self._router = router
self._max_iterations = max_iterations
self._provider_format = provider_format
# -- Properties -----------------------------------------------------------
@property
def max_iterations(self) -> int:
"""Return the configured max iterations."""
return self._max_iterations
@property
def provider_format(self) -> ProviderFormat:
"""Return the target provider format."""
return self._provider_format
# -- Schema export --------------------------------------------------------
def export_tool_schemas(
self,
namespace: str | None = None,
) -> list[dict[str, Any]]:
"""Export tool schemas in the configured provider format.
If a ``ToolCallRouter`` is available, delegates to its
``export_schemas`` method. Otherwise, converts each spec
individually using ``normalize_tool_schema_for_provider``.
Parameters
----------
namespace:
Optional namespace filter.
Returns
-------
list[dict]:
Provider-normalized tool schemas.
"""
if self._router is not None:
return self._router.export_schemas(
self._provider_format, namespace=namespace
)
specs = self._registry.list_tools(namespace=namespace)
return [
normalize_tool_schema_for_provider(spec, self._provider_format)
for spec in specs
]
# -- Tool execution -------------------------------------------------------
def _execute_tool_call(
self,
tool_call: LLMToolCall,
actor_context: ToolActorContext,
iteration: int,
) -> dict[str, Any]:
"""Execute a single tool call and record metadata.
Threads sandbox root and resource bindings into the tool inputs,
executes via the router (if available) or runner, and records
a ``ToolCallRecord`` in the actor context.
Parameters
----------
tool_call:
The LLM-requested tool call.
actor_context:
The actor context carrying sandbox/resource state.
iteration:
Current loop iteration number.
Returns
-------
dict:
Tool result as a dict for feeding back to the LLM.
"""
# Thread sandbox root + resource bindings into inputs
enriched_inputs = dict(tool_call.arguments)
if actor_context.sandbox_root:
enriched_inputs.setdefault("sandbox_root", actor_context.sandbox_root)
if actor_context.resource_bindings:
enriched_inputs.setdefault(
"resource_bindings", actor_context.resource_bindings
)
start = time.monotonic()
success = False
output: dict[str, Any] = {}
error: str | None = None
if self._router is not None:
# Route through ToolCallRouter for provider format handling
payload: dict[str, Any] = {
"name": tool_call.name,
"args": enriched_inputs,
"type": "tool_call",
}
routed_result = self._router.route(payload)
elapsed_ms = (time.monotonic() - start) * 1000.0
success = routed_result.result.success
output = routed_result.result.output
error = routed_result.result.error
else:
# Execute directly via runner, catching ToolError for not-found
try:
result = self._runner.execute(tool_call.name, enriched_inputs)
elapsed_ms = (time.monotonic() - start) * 1000.0
success = result.success
output = result.output
error = result.error
except ToolError as exc:
elapsed_ms = (time.monotonic() - start) * 1000.0
success = False
output = {}
error = str(exc)
except Exception as exc:
elapsed_ms = (time.monotonic() - start) * 1000.0
success = False
output = {}
error = f"{type(exc).__name__}: {exc}"
# Record metadata
record = ToolCallRecord(
tool_name=tool_call.name,
inputs=dict(tool_call.arguments),
output=output,
duration_ms=round(elapsed_ms, 2),
success=success,
error=error,
iteration=iteration,
)
actor_context.record_tool_call(record)
# Build result for LLM feedback
result_dict: dict[str, Any] = {
"tool_name": tool_call.name,
"success": success,
"output": output,
}
if error:
result_dict["error"] = error
return result_dict
# -- Main loop ------------------------------------------------------------
def run_tool_loop(
self,
prompt: str,
actor_config: dict[str, Any] | None = None,
context: ToolActorContext | None = None,
) -> ToolCallRunResult:
"""Execute the tool-calling loop.
Sends the prompt with tool schemas to the LLM, executes any
requested tool calls, feeds results back, and repeats until the
LLM responds without tool calls or ``max_iterations`` is reached.
Parameters
----------
prompt:
The initial prompt text for the LLM.
actor_config:
Optional actor configuration dict (provider, model, etc.).
context:
Optional ``ToolActorContext`` carrying sandbox/resource state.
If not provided, a minimal context is created.
Returns
-------
ToolCallRunResult:
The final result including content, history, and loop metadata.
Raises
------
ValueError:
If prompt is empty.
"""
if not prompt:
raise ValueError("prompt must not be empty")
if context is None:
context = ToolActorContext(plan_id="default", phase="execute")
# Export tool schemas
tool_schemas = self.export_tool_schemas()
tool_results: list[dict[str, Any]] | None = None
final_content = ""
for iteration in range(1, self._max_iterations + 1):
logger.debug(
"Tool-call loop iteration %d/%d",
iteration,
self._max_iterations,
)
# Invoke LLM
llm_response = self._llm_caller.invoke(
prompt=prompt,
tool_schemas=tool_schemas,
tool_results=tool_results,
actor_config=actor_config,
)
# No tool calls -> done
if not llm_response.tool_calls:
final_content = llm_response.content
return ToolCallRunResult(
content=final_content,
tool_call_history=context.tool_call_history,
iterations=iteration,
terminated_by_limit=False,
)
# Execute each tool call
tool_results = []
for tc in llm_response.tool_calls:
result_dict = self._execute_tool_call(tc, context, iteration)
tool_results.append(result_dict)
# Max iterations reached
logger.warning(
"Tool-call loop reached max iterations (%d)", self._max_iterations
)
return ToolCallRunResult(
content=final_content,
tool_call_history=context.tool_call_history,
iterations=self._max_iterations,
terminated_by_limit=True,
)