"""Step definitions for the Tool-Calling Actor Runtime feature.""" from __future__ import annotations import contextlib from typing import Any from behave import given, then, when from cleveragents.infrastructure.events.reactive import ReactiveEventBus from cleveragents.tool.actor_context import ToolActorContext, ToolCallRecord from cleveragents.tool.actor_runtime import ( LLMResponse, LLMToolCall, ToolCallingRuntime, ToolCallRunResult, ) from cleveragents.tool.registry import ToolRegistry from cleveragents.tool.router import ToolCallRouter from cleveragents.tool.runner import ToolRunner from cleveragents.tool.runtime import ToolSpec __all__: list[str] = [] # --------------------------------------------------------------------------- # Mock handlers # --------------------------------------------------------------------------- def _rt_echo_handler(inputs: dict[str, Any]) -> dict[str, Any]: return {"echoed": inputs} def _rt_adder_handler(inputs: dict[str, Any]) -> dict[str, Any]: return {"sum": inputs.get("a", 0) + inputs.get("b", 0)} def _rt_failing_handler(inputs: dict[str, Any]) -> dict[str, Any]: raise RuntimeError("handler exploded") def _rt_generic_error_handler(inputs: dict[str, Any]) -> dict[str, Any]: raise KeyError("missing_key") _rt_captured_inputs: dict[str, Any] = {} def _rt_capturing_handler(inputs: dict[str, Any]) -> dict[str, Any]: _rt_captured_inputs.clear() _rt_captured_inputs.update(inputs) return {"captured": True} # --------------------------------------------------------------------------- # Mock LLM callers # --------------------------------------------------------------------------- class _SingleToolLLM: def __init__(self) -> None: self._n = 0 def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: self._n += 1 if self._n == 1: return LLMResponse( content="", tool_calls=[LLMToolCall(name="test/echo", arguments={"msg": "hello"})], ) return LLMResponse(content="Final answer after tool call", tool_calls=[]) class _TwoToolsLLM: def __init__(self) -> None: self._n = 0 def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: self._n += 1 if self._n == 1: return LLMResponse( content="", tool_calls=[LLMToolCall(name="test/echo", arguments={"msg": "first"})], ) if self._n == 2: return LLMResponse( content="", tool_calls=[LLMToolCall(name="test/adder", arguments={"a": 1, "b": 2})], ) return LLMResponse(content="Done after two tool calls", tool_calls=[]) class _AlwaysToolsLLM: def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: return LLMResponse( content="", tool_calls=[LLMToolCall(name="test/echo", arguments={"x": "loop"})], ) class _NonexistentToolLLM: def __init__(self) -> None: self._n = 0 def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: self._n += 1 if self._n == 1: return LLMResponse( content="", tool_calls=[LLMToolCall(name="test/nonexistent", arguments={"x": 1})], ) return LLMResponse(content="Done after error", tool_calls=[]) class _FailingToolLLM: def __init__(self) -> None: self._n = 0 def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: self._n += 1 if self._n == 1: return LLMResponse( content="", tool_calls=[LLMToolCall(name="test/fail", arguments={})], ) return LLMResponse(content="Done after failing tool", tool_calls=[]) class _ImmediateLLM: def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: return LLMResponse(content="No tools needed", tool_calls=[]) class _CapturingToolLLM: def __init__(self) -> None: self._n = 0 def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: self._n += 1 if self._n == 1: return LLMResponse( content="", tool_calls=[ LLMToolCall(name="test/capture", arguments={"data": "test"}) ], ) return LLMResponse(content="Done after capture", tool_calls=[]) class _GenericErrorToolLLM: def __init__(self) -> None: self._n = 0 def invoke( self, prompt: str, tool_schemas: list[dict[str, Any]], tool_results: list[dict[str, Any]] | None = None, actor_config: dict[str, Any] | None = None, ) -> LLMResponse: self._n += 1 if self._n == 1: return LLMResponse( content="", tool_calls=[LLMToolCall(name="test/generic_error", arguments={"x": 1})], ) return LLMResponse(content="Done after generic error", tool_calls=[]) # --------------------------------------------------------------------------- # Given steps — registry & runner # --------------------------------------------------------------------------- @given("a tool registry with an echo tool") def step_given_registry_echo(context: Any) -> None: if not hasattr(context, "registry"): context.registry = ToolRegistry() with contextlib.suppress(Exception): context.registry.register( ToolSpec( name="test/echo", description="Echo tool", handler=_rt_echo_handler ) ) @given("a tool registry with an adder tool") def step_given_registry_adder(context: Any) -> None: if not hasattr(context, "registry"): context.registry = ToolRegistry() with contextlib.suppress(Exception): context.registry.register( ToolSpec( name="test/adder", description="Adder tool", handler=_rt_adder_handler ) ) @given("a tool registry with a failing tool") def step_given_registry_failing(context: Any) -> None: if not hasattr(context, "registry"): context.registry = ToolRegistry() with contextlib.suppress(Exception): context.registry.register( ToolSpec( name="test/fail", description="Failing tool", handler=_rt_failing_handler, ) ) @given("a tool registry with an input-capturing tool") def step_given_registry_capturing(context: Any) -> None: if not hasattr(context, "registry"): context.registry = ToolRegistry() with contextlib.suppress(Exception): context.registry.register( ToolSpec( name="test/capture", description="Capturing tool", handler=_rt_capturing_handler, ) ) @given("a tool registry with a tool that raises a generic exception") def step_given_registry_generic_error(context: Any) -> None: if not hasattr(context, "registry"): context.registry = ToolRegistry() with contextlib.suppress(Exception): context.registry.register( ToolSpec( name="test/generic_error", description="Generic error tool", handler=_rt_generic_error_handler, ) ) @given("an empty actor tool registry") def step_given_empty_registry(context: Any) -> None: context.registry = ToolRegistry() @given("a tool runner for the actor registry") def step_given_runner(context: Any) -> None: context.runner = ToolRunner(context.registry) # --------------------------------------------------------------------------- # Given steps — mock LLM callers # --------------------------------------------------------------------------- @given("a mock LLM caller that requests one tool call then responds") def step_given_mock_llm_single(context: Any) -> None: context.llm_caller = _SingleToolLLM() @given("a mock LLM caller that requests two sequential tool calls then responds") def step_given_mock_llm_two(context: Any) -> None: context.llm_caller = _TwoToolsLLM() @given("a mock LLM caller that always requests tool calls") def step_given_mock_llm_always(context: Any) -> None: context.llm_caller = _AlwaysToolsLLM() @given("a mock LLM caller that requests a nonexistent tool then responds") def step_given_mock_llm_nonexistent(context: Any) -> None: context.llm_caller = _NonexistentToolLLM() @given("a mock LLM caller that requests the failing tool then responds") def step_given_mock_llm_failing(context: Any) -> None: context.llm_caller = _FailingToolLLM() @given("a mock LLM caller that responds immediately without tool calls") def step_given_mock_llm_immediate(context: Any) -> None: context.llm_caller = _ImmediateLLM() @given("a mock LLM caller that requests the capturing tool then responds") def step_given_mock_llm_capturing(context: Any) -> None: context.llm_caller = _CapturingToolLLM() @given("a mock LLM caller that requests the generic-error tool then responds") def step_given_mock_llm_generic_error(context: Any) -> None: context.llm_caller = _GenericErrorToolLLM() # --------------------------------------------------------------------------- # Given steps — runtime construction # --------------------------------------------------------------------------- @given("a tool-calling runtime with the mock LLM caller") def step_given_runtime(context: Any) -> None: context.runtime = ToolCallingRuntime( registry=context.registry, runner=context.runner, llm_caller=context.llm_caller, event_bus=ReactiveEventBus(), ) @given("a tool-calling runtime with max iterations {n:d}") def step_given_runtime_max_iter(context: Any, n: int) -> None: context.runtime = ToolCallingRuntime( registry=context.registry, runner=context.runner, llm_caller=context.llm_caller, max_iterations=n, event_bus=ReactiveEventBus(), ) @given("a tool-calling runtime with the mock LLM caller and router") def step_given_runtime_with_router(context: Any) -> None: context.runtime = ToolCallingRuntime( registry=context.registry, runner=context.runner, llm_caller=context.llm_caller, router=context.router, event_bus=ReactiveEventBus(), ) # --------------------------------------------------------------------------- # Given steps — actor context # --------------------------------------------------------------------------- @given('an actor context with plan "{plan_id}" phase "{phase}"') def step_given_actor_ctx_plan_phase(context: Any, plan_id: str, phase: str) -> None: context.actor_context = ToolActorContext(plan_id=plan_id, phase=phase) @given('an actor context with plan "{plan_id}" sandbox "{root}"') def step_given_actor_ctx_plan_sandbox(context: Any, plan_id: str, root: str) -> None: context.actor_context = ToolActorContext(plan_id=plan_id, sandbox_root=root) @given('an actor context with plan "{plan_id}"') def step_given_actor_ctx_plan(context: Any, plan_id: str) -> None: context.actor_context = ToolActorContext(plan_id=plan_id) @given('an actor context with sandbox root "{root}"') def step_given_actor_ctx_sandbox(context: Any, root: str) -> None: context.actor_context = ToolActorContext( plan_id="plan-test", phase="execute", sandbox_root=root, ) @given("an actor context with resource bindings") def step_given_actor_ctx_bindings(context: Any) -> None: context.actor_context = ToolActorContext( plan_id="plan-test", phase="execute", resource_bindings={"source": {"type": "git-checkout", "path": "/repo"}}, ) @given("an actor context with all optional fields") def step_given_actor_ctx_all_fields(context: Any) -> None: context.actor_context = ToolActorContext( plan_id="plan-full", phase="execute", sandbox_root="/tmp/sand", automation_profile="auto-profile", resource_bindings={"src": {"path": "/src"}}, project_resources={"repo": "/home/repo"}, metadata={"version": "1.0"}, ) # Pre-add a record so we can test clear_history context.actor_context.record_tool_call( ToolCallRecord( tool_name="test/dummy", success=True, duration_ms=1.0, iteration=1, ) ) @given('a tool call router with plan id "{plan_id}"') def step_given_router(context: Any, plan_id: str) -> None: context.router = ToolCallRouter( registry=context.registry, runner=context.runner, plan_id=plan_id, ) @given('a tool call record with name "{name}" and duration {duration:g}') def step_given_tool_call_record(context: Any, name: str, duration: float) -> None: context.tool_call_record = ToolCallRecord(tool_name=name, duration_ms=duration) # --------------------------------------------------------------------------- # When steps # --------------------------------------------------------------------------- @when('I run the tool loop with prompt "{prompt}"') def step_when_run_loop(context: Any, prompt: str) -> None: context.run_result = context.runtime.run_tool_loop(prompt=prompt) @when('I run the tool loop with prompt "{prompt}" and the actor context') def step_when_run_loop_with_context(context: Any, prompt: str) -> None: context.run_result = context.runtime.run_tool_loop( prompt=prompt, context=context.actor_context, ) @when('I run the tool loop with prompt "{prompt}" and no context') def step_when_run_loop_no_context(context: Any, prompt: str) -> None: context.run_result = context.runtime.run_tool_loop(prompt=prompt, context=None) @when('I record a tool call with name "{name}" and success {success}') def step_when_record_tool_call(context: Any, name: str, success: str) -> None: record = ToolCallRecord(tool_name=name, success=success == "True") context.actor_context.record_tool_call(record) # --------------------------------------------------------------------------- # Then steps # --------------------------------------------------------------------------- @then('the run result content should be "{expected}"') def step_then_content(context: Any, expected: str) -> None: result: ToolCallRunResult = context.run_result assert result.content == expected, f"Expected '{expected}', got '{result.content}'" @then("the run result should have {count:d} tool call records") def step_then_tool_call_count(context: Any, count: int) -> None: result: ToolCallRunResult = context.run_result actual = len(result.tool_call_history) assert actual == count, f"Expected {count} records, got {actual}" @then("the run result iterations should be {expected:d}") def step_then_iterations(context: Any, expected: int) -> None: result: ToolCallRunResult = context.run_result assert result.iterations == expected, ( f"Expected {expected}, got {result.iterations}" ) @then("the run result should not be terminated by limit") def step_then_not_terminated(context: Any) -> None: assert not context.run_result.terminated_by_limit @then("the run result should be terminated by limit") def step_then_terminated(context: Any) -> None: assert context.run_result.terminated_by_limit @then("the first tool call record should have success {expected}") def step_then_first_record_success(context: Any, expected: str) -> None: record = context.run_result.tool_call_history[0] assert record.success == (expected == "True") @then("the first tool call record should have an error message") def step_then_first_record_has_error(context: Any) -> None: record = context.run_result.tool_call_history[0] assert record.error is not None and len(record.error) > 0 @then('the first tool call record error should contain "{fragment}"') def step_then_first_record_error_contains(context: Any, fragment: str) -> None: record = context.run_result.tool_call_history[0] assert record.error is not None assert fragment in record.error, f"Expected '{fragment}' in '{record.error}'" @then('the first tool call record tool name should be "{name}"') def step_then_first_record_name(context: Any, name: str) -> None: assert context.run_result.tool_call_history[0].tool_name == name @then("the first tool call record inputs should not be empty") def step_then_first_record_inputs(context: Any) -> None: assert len(context.run_result.tool_call_history[0].inputs) > 0 @then("the first tool call record output should not be empty") def step_then_first_record_output(context: Any) -> None: assert len(context.run_result.tool_call_history[0].output) > 0 @then("the first tool call record duration should be greater than 0") def step_then_first_record_duration(context: Any) -> None: assert context.run_result.tool_call_history[0].duration_ms >= 0.0 @then("the first tool call record iteration should be {expected:d}") def step_then_first_record_iteration(context: Any, expected: int) -> None: assert context.run_result.tool_call_history[0].iteration == expected @then('the captured tool inputs should contain sandbox root "{root}"') def step_then_captured_sandbox(context: Any, root: str) -> None: assert _rt_captured_inputs.get("sandbox_root") == root @then("the captured tool inputs should contain resource bindings") def step_then_captured_bindings(context: Any) -> None: assert isinstance(_rt_captured_inputs.get("resource_bindings"), dict) @then("creating a runtime with max iterations 0 should raise ValueError") def step_then_invalid_max_iter(context: Any) -> None: try: ToolCallingRuntime( registry=context.registry, runner=context.runner, llm_caller=context.llm_caller, max_iterations=0, ) raise AssertionError("Expected ValueError") except ValueError: pass @then("running the tool loop with empty prompt should raise ValueError") def step_then_empty_prompt(context: Any) -> None: try: context.runtime.run_tool_loop(prompt="") raise AssertionError("Expected ValueError") except ValueError: pass @then('the actor context plan id should be "{expected}"') def step_then_actor_plan_id(context: Any, expected: str) -> None: assert context.actor_context.plan_id == expected @then('the actor context phase should be "{expected}"') def step_then_actor_phase(context: Any, expected: str) -> None: assert context.actor_context.phase == expected @then("the actor context tool call history should be empty") def step_then_actor_history_empty(context: Any) -> None: assert len(context.actor_context.tool_call_history) == 0 @then("the actor context tool call history should have {count:d} records") def step_then_actor_history_count(context: Any, count: int) -> None: assert len(context.actor_context.tool_call_history) == count @then("creating a tool actor context with empty plan id should raise ValueError") def step_then_empty_plan_id(context: Any) -> None: try: ToolActorContext(plan_id="") raise AssertionError("Expected ValueError") except ValueError: pass @then('the actor context summary should contain plan id "{plan_id}"') def step_then_summary_plan_id(context: Any, plan_id: str) -> None: assert context.actor_context.as_summary()["plan_id"] == plan_id @then('the actor context summary should contain sandbox root "{root}"') def step_then_summary_sandbox(context: Any, root: str) -> None: assert context.actor_context.as_summary()["sandbox_root"] == root @then('the tool call record tool name should be "{name}"') def step_then_record_name(context: Any, name: str) -> None: assert context.tool_call_record.tool_name == name @then("the tool call record duration should be {expected:g}") def step_then_record_duration(context: Any, expected: float) -> None: assert context.tool_call_record.duration_ms == expected # -- Actor Context additional property steps -- @then('the actor context automation profile should be "{expected}"') def step_then_actor_automation_profile(context: Any, expected: str) -> None: assert context.actor_context.automation_profile == expected @then('the actor context project resources should have key "{key}"') def step_then_actor_project_resources_key(context: Any, key: str) -> None: assert key in context.actor_context.project_resources @then('the actor context metadata should have key "{key}"') def step_then_actor_metadata_key(context: Any, key: str) -> None: assert key in context.actor_context.metadata @then("the actor context clear history should empty the history") def step_then_actor_clear_history(context: Any) -> None: assert len(context.actor_context.tool_call_history) > 0 context.actor_context.clear_history() assert len(context.actor_context.tool_call_history) == 0 @then("recording a non-ToolCallRecord should raise TypeError") def step_then_invalid_record_type(context: Any) -> None: try: context.actor_context.record_tool_call("not a record") # type: ignore[arg-type] raise AssertionError("Expected TypeError") except TypeError: pass # -- Runtime constructor type validation steps -- @then("creating a runtime with a non-ToolRegistry should raise TypeError") def step_then_invalid_registry_type(context: Any) -> None: try: ToolCallingRuntime( registry="not a registry", # type: ignore[arg-type] runner=ToolRunner(ToolRegistry()), llm_caller=_ImmediateLLM(), ) raise AssertionError("Expected TypeError") except TypeError: pass @then("creating a runtime with a non-ToolRunner should raise TypeError") def step_then_invalid_runner_type(context: Any) -> None: try: ToolCallingRuntime( registry=context.registry, runner="not a runner", # type: ignore[arg-type] llm_caller=_ImmediateLLM(), ) raise AssertionError("Expected TypeError") except TypeError: pass # -- Runtime property accessors steps -- @then("the runtime max iterations should be {expected:d}") def step_then_runtime_max_iter(context: Any, expected: int) -> None: assert context.runtime.max_iterations == expected @then("the runtime provider format should not be None") def step_then_runtime_provider_format(context: Any) -> None: assert context.runtime.provider_format is not None