fix(runtime): add input validation, credential safety, and proper error propagation #40
@@ -0,0 +1,58 @@
|
||||
Feature: Dynamic Router for LangGraph
|
||||
As a developer
|
||||
I want dynamic routers to handle content-based routing and condition evaluation
|
||||
So that graph workflows can route messages based on content patterns
|
||||
|
||||
Background:
|
||||
Given the dynamic router test context is initialized (drc)
|
||||
|
||||
Scenario: create_dynamic_router_config builds config from patterns dict
|
||||
Given a patterns dict with routing patterns (drc)
|
||||
When create_dynamic_router_config is called (drc)
|
||||
Then a config with type dynamic_router and routes list should be returned (drc)
|
||||
|
||||
Scenario: ContentBasedCondition evaluates to true when pattern matches
|
||||
Given a ContentBasedCondition with pattern "GOTO_TARGET" (drc)
|
||||
And a graph state with last message containing "GOTO_TARGET" (drc)
|
||||
When evaluate is called on the condition (drc)
|
||||
Then the result should be True (drc)
|
||||
|
||||
Scenario: ContentBasedCondition evaluates to false when pattern does not match
|
||||
Given a ContentBasedCondition with pattern "GOTO_TARGET" (drc)
|
||||
And a graph state with last message not containing pattern (drc)
|
||||
When evaluate is called on the condition (drc)
|
||||
Then the result should be False (drc)
|
||||
|
||||
Scenario: ContentBasedCondition evaluates to false when messages list is empty
|
||||
Given a ContentBasedCondition with pattern "GOTO_TARGET" (drc)
|
||||
And a graph state with empty messages list (drc)
|
||||
When evaluate is called on the condition (drc)
|
||||
Then the result should be False (drc)
|
||||
|
||||
Scenario: ContentBasedCondition handles dict messages correctly
|
||||
Given a ContentBasedCondition with pattern "MATCH_ME" (drc)
|
||||
And a graph state with last message as dict containing "MATCH_ME" (drc)
|
||||
When evaluate is called on the condition (drc)
|
||||
Then the result should be True (drc)
|
||||
|
||||
Scenario: ContentBasedCondition handles raw string messages not in dict wrapper
|
||||
Given a ContentBasedCondition with pattern "ABSENT" (drc)
|
||||
And a graph state with raw non-dict string messages (drc)
|
||||
When evaluate is called on the condition (drc)
|
||||
Then the result should be False (drc)
|
||||
|
||||
Scenario: extend_graph_with_router does not redirect non-routing nodes
|
||||
Given a graph config with non-routing edges (drc)
|
||||
When extend_graph_with_router is called (drc)
|
||||
Then non-routing edges should be preserved unchanged (drc)
|
||||
|
||||
Scenario: register_content_conditions can be called on a graph
|
||||
Given a mock graph instance (drc)
|
||||
When register_content_conditions is called (drc)
|
||||
Then the function should complete without error (drc)
|
||||
|
||||
Scenario: DynamicRouterNode handles non-dict messages
|
||||
Given a DynamicRouterNode with routing patterns (drc)
|
||||
And a graph state with string messages (drc)
|
||||
When execute is called on the router (drc)
|
||||
Then routing should work with string message content (drc)
|
||||
@@ -0,0 +1,12 @@
|
||||
Feature: ProgressBarManager Coverage
|
||||
As a developer
|
||||
I want the progress bar manager to resolve state from remaining counts and contextual snapshots
|
||||
So that progress rendering is robust across all update patterns
|
||||
|
||||
Background:
|
||||
Given the progress test context is initialized (prg)
|
||||
|
||||
Scenario: ProgressBarManager resolves current from remaining count
|
||||
Given the progress manager update is called with total 10 and remaining 8 (prg)
|
||||
When the progress snapshot is retrieved (prg)
|
||||
Then the snapshot should have total 10 and remaining 8 (prg)
|
||||
@@ -0,0 +1,12 @@
|
||||
Feature: Route Graph Config Conversion Coverage
|
||||
As a developer
|
||||
I want RouteConfig to correctly convert MESSAGE_ROUTER node rules to metadata
|
||||
So that graph configuration conversions preserve routing rule information
|
||||
|
||||
Background:
|
||||
Given the route graph config test context is initialized (rgc)
|
||||
|
||||
Scenario: MESSAGE_ROUTER rules added to node metadata during to_graph_config
|
||||
Given a RouteConfig with MESSAGE_ROUTER node having rules (rgc)
|
||||
When to_graph_config is called (rgc)
|
||||
Then the MESSAGE_ROUTER rules should appear in node metadata (rgc)
|
||||
@@ -0,0 +1,83 @@
|
||||
Feature: Runtime Executor Extended Coverage
|
||||
As a developer
|
||||
I want the runtime executor to handle additional edge cases in graph, tool, and multi-actor execution
|
||||
So that all code paths in the runtime module are exercised
|
||||
|
||||
Background:
|
||||
Given the runtime extended test context is initialized (rxe)
|
||||
|
||||
Scenario: execute detects graph type from routes key in CleverAgents v2.0 format
|
||||
Given a config dict with routes key but no type field (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test v2 graph" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _normalize_graph_config handles v2.0 routes with dict nodes
|
||||
Given a config dict with type graph and v2.0 routes format with dict nodes (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test v2 dict nodes" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _normalize_graph_config handles v2.0 routes with list nodes
|
||||
Given a config dict with type graph and v2.0 routes format with list nodes (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test v2 list nodes" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_llm builds conversation history from messages
|
||||
Given a config dict with type llm provider model and system_prompt in config block (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
And conversation history messages are provided (rxe)
|
||||
When I execute the extended runtime actor with message "test conversation" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_llm picks up provider from config block when not at top level
|
||||
Given a config dict with type llm and provider model in nested config block (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test config block" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_graph handles agent name from node ID in v2.0 actors block
|
||||
Given a config dict with type graph and v2.0 routes with actors block (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test agents block" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_graph handles exception during agent creation warning
|
||||
Given a config dict with type graph and route with agents block having missing agent (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test missing agent" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_graph merges global context from config with conversation history
|
||||
Given a config dict with type graph and route with global context (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
And conversation history messages are provided (rxe)
|
||||
When I execute the extended runtime actor with message "test global context" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_tool builds conversation history from messages
|
||||
Given a config dict with type tool and tools list (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
And conversation history messages are provided (rxe)
|
||||
When I execute the extended runtime actor with message "test tool with history" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_multi_actor uses default actor from cleveragents block
|
||||
Given a multi-actor config dict with cleveragents default_actor (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test default actor" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
And the node usage IDs should be prefixed with the default actor name (rxe)
|
||||
|
||||
Scenario: _execute_graph handles actor context without global key
|
||||
Given a config dict with type graph and route with actor context without global key (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test actor context" (rxe)
|
||||
Then the execution should return an ActorResult (rxe)
|
||||
|
||||
Scenario: _execute_graph handles missing credential provider gracefully
|
||||
Given a config dict with type llm and unknown provider (rxe)
|
||||
And credentials dict with openai provider (rxe)
|
||||
When I execute the extended runtime actor with message "test missing creds" (rxe)
|
||||
Then a ConfigurationError should be raised about missing credentials (rxe)
|
||||
@@ -0,0 +1,47 @@
|
||||
Feature: Runtime Tokens Estimation
|
||||
As a developer
|
||||
I want token estimation to work correctly with both tiktoken and fallback heuristics
|
||||
So that token counting is reliable regardless of the tiktoken availability
|
||||
|
||||
Background:
|
||||
Given the runtime tokens test context is initialized (rtc)
|
||||
|
||||
Scenario: estimate_tokens uses tiktoken for gpt-4 model with tiktoken available
|
||||
Given tiktoken is available (rtc)
|
||||
And a mock tiktoken encoding is configured (rtc)
|
||||
When estimate_tokens is called with prompt "Hello world" response "Hi" model "gpt-4" provider "openai" (rtc)
|
||||
Then prompt tokens and completion tokens should be returned from tiktoken (rtc)
|
||||
|
||||
Scenario: estimate_tokens uses cl100k_base encoding for non-gpt models with tiktoken available
|
||||
Given tiktoken is available (rtc)
|
||||
And a mock tiktoken encoding is configured (rtc)
|
||||
When estimate_tokens is called with prompt "Test" response "OK" model "claude-3" provider "anthropic" (rtc)
|
||||
Then the cl100k_base encoding should be used for token estimation (rtc)
|
||||
|
||||
Scenario: estimate_tokens falls back to heuristic when tiktoken is not available
|
||||
Given tiktoken is not available (rtc)
|
||||
When estimate_tokens is called with prompt "Hello this is a longer test prompt" response "Short" model "gpt-4" provider "openai" (rtc)
|
||||
Then token counts should be estimated using the character heuristic (rtc)
|
||||
|
||||
Scenario: estimate_tokens falls back to heuristic when tiktoken raises exception
|
||||
Given tiktoken is available (rtc)
|
||||
And tiktoken encoding raises an exception (rtc)
|
||||
When estimate_tokens is called with prompt "Hello" response "World" model "gpt-4" provider "openai" (rtc)
|
||||
Then token counts should fall back to heuristic estimation (rtc)
|
||||
|
||||
Scenario: estimate_graph_tokens uses cl100k_base encoding when tiktoken is available
|
||||
Given tiktoken is available (rtc)
|
||||
And a mock tiktoken encoding is configured (rtc)
|
||||
When estimate_graph_tokens is called with prompt "Graph prompt" response "Graph response" (rtc)
|
||||
Then graph token counts should be returned from tiktoken (rtc)
|
||||
|
||||
Scenario: estimate_graph_tokens falls back to heuristic when tiktoken is not available
|
||||
Given tiktoken is not available (rtc)
|
||||
When estimate_graph_tokens is called with prompt "Graph prompt long text here" response "Graph response" (rtc)
|
||||
Then graph token counts should be estimated using the character heuristic (rtc)
|
||||
|
||||
Scenario: estimate_graph_tokens falls back to heuristic when tiktoken raises exception
|
||||
Given tiktoken is available (rtc)
|
||||
And tiktoken encoding raises an exception (rtc)
|
||||
When estimate_graph_tokens is called with prompt "Test" response "Result" (rtc)
|
||||
Then graph token counts should fall back to heuristic estimation (rtc)
|
||||
@@ -182,7 +182,7 @@ async def step_execute_graph_actor(context: Any, message: str) -> None:
|
||||
context._captured_factory_credentials = captured_factory_credentials
|
||||
|
||||
try:
|
||||
mock_execute = AsyncMock(return_value="Mock graph response")
|
||||
mock_execute = AsyncMock(return_value=("Mock graph response", {}))
|
||||
with (
|
||||
patch.object(AgentFactory, "__init__", capturing_factory_init),
|
||||
patch.object(PureLangGraph, "execute", mock_execute),
|
||||
|
||||
@@ -60,7 +60,7 @@ async def step_execute_graph_actor_cleanup_error(context: Any) -> None:
|
||||
|
||||
context.executor_raised_exception = None
|
||||
try:
|
||||
mock_execute = AsyncMock(return_value="Mock graph response")
|
||||
mock_execute = AsyncMock(return_value=("Mock graph response", {}))
|
||||
with patch.object(PureLangGraph, "execute", mock_execute):
|
||||
context.executor_result = await context.executor.execute(
|
||||
"Hello cleanup error"
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Step definitions for dynamic_router.py coverage tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveractors.langgraph.dynamic_router import (
|
||||
ContentBasedCondition,
|
||||
DynamicRouterNode,
|
||||
RoutePattern,
|
||||
create_dynamic_router_config,
|
||||
extend_graph_with_router,
|
||||
register_content_conditions,
|
||||
)
|
||||
|
||||
|
||||
@given("the dynamic router test context is initialized (drc)")
|
||||
def step_drc_init(context: Context) -> None:
|
||||
context.drc_result = None
|
||||
context.drc_error = None
|
||||
context.drc_state = None
|
||||
context.drc_condition = None
|
||||
context.drc_router = None
|
||||
context.drc_graph_config = None
|
||||
|
||||
|
||||
@given("a patterns dict with routing patterns (drc)")
|
||||
def step_patterns_dict(context: Context) -> None:
|
||||
context.drc_patterns = {"GOTO_X": "target_x", "GOTO_Y": "target_y"}
|
||||
|
||||
|
||||
@when("create_dynamic_router_config is called (drc)")
|
||||
def step_create_router_config(context: Context) -> None:
|
||||
context.drc_result = create_dynamic_router_config(context.drc_patterns)
|
||||
|
||||
|
||||
@then("a config with type dynamic_router and routes list should be returned (drc)")
|
||||
def step_then_router_config(context: Context) -> None:
|
||||
assert context.drc_result is not None
|
||||
assert context.drc_result["type"] == "dynamic_router"
|
||||
assert isinstance(context.drc_result["routes"], list)
|
||||
assert len(context.drc_result["routes"]) == 2
|
||||
for route in context.drc_result["routes"]:
|
||||
assert isinstance(route, RoutePattern)
|
||||
|
||||
|
||||
@given('a ContentBasedCondition with pattern "{pattern}" (drc)')
|
||||
def step_condition_with_pattern(context: Context, pattern: str) -> None:
|
||||
context.drc_condition = ContentBasedCondition(pattern=pattern)
|
||||
|
||||
|
||||
@given('a graph state with last message containing "{content}" (drc)')
|
||||
def step_state_with_content(context: Context, content: str) -> None:
|
||||
context.drc_state = {"messages": [{"role": "user", "content": content}]}
|
||||
|
||||
|
||||
@given("a graph state with last message not containing pattern (drc)")
|
||||
def step_state_without_pattern(context: Context) -> None:
|
||||
context.drc_state = {
|
||||
"messages": [{"role": "user", "content": "nothing to see here"}]
|
||||
}
|
||||
|
||||
|
||||
@given("a graph state with empty messages list (drc)")
|
||||
def step_state_empty_messages(context: Context) -> None:
|
||||
context.drc_state = {"messages": []}
|
||||
|
||||
|
||||
@given('a graph state with last message as dict containing "{content}" (drc)')
|
||||
def step_state_dict_with_content(context: Context, content: str) -> None:
|
||||
context.drc_state = {"messages": [{"content": content}]}
|
||||
|
||||
|
||||
@when("evaluate is called on the condition (drc)")
|
||||
def step_evaluate_condition(context: Context) -> None:
|
||||
context.drc_result = context.drc_condition.evaluate(context.drc_state)
|
||||
|
||||
|
||||
@then("the result should be True (drc)")
|
||||
def step_then_true(context: Context) -> None:
|
||||
assert context.drc_result is True
|
||||
|
||||
|
||||
@then("the result should be False (drc)")
|
||||
def step_then_false(context: Context) -> None:
|
||||
assert context.drc_result is False
|
||||
|
||||
|
||||
@given("a graph config with non-routing edges (drc)")
|
||||
def step_non_routing_edges(context: Context) -> None:
|
||||
context.drc_graph_config = {
|
||||
"nodes": {"node_a": {"type": "function"}, "node_b": {"type": "function"}},
|
||||
"edges": [{"source": "node_a", "target": "node_b"}],
|
||||
}
|
||||
|
||||
|
||||
@when("extend_graph_with_router is called (drc)")
|
||||
def step_extend_with_router(context: Context) -> None:
|
||||
import copy
|
||||
|
||||
context.drc_result = extend_graph_with_router(
|
||||
copy.deepcopy(context.drc_graph_config)
|
||||
)
|
||||
|
||||
|
||||
@then("non-routing edges should be preserved unchanged (drc)")
|
||||
def step_then_preserved_edges(context: Context) -> None:
|
||||
original_edges = {
|
||||
(e["source"], e["target"]) for e in context.drc_graph_config["edges"]
|
||||
}
|
||||
result_edges = {(e["source"], e["target"]) for e in context.drc_result["edges"]}
|
||||
for orig in original_edges:
|
||||
assert orig in result_edges, f"Edge {orig} should be preserved"
|
||||
|
||||
|
||||
@given("a mock graph instance (drc)")
|
||||
def step_mock_graph(context: Context) -> None:
|
||||
context.drc_mock_graph = MagicMock()
|
||||
|
||||
|
||||
@when("register_content_conditions is called (drc)")
|
||||
def step_register_conditions(context: Context) -> None:
|
||||
register_content_conditions(context.drc_mock_graph)
|
||||
context.drc_result = "completed"
|
||||
|
||||
|
||||
@then("the function should complete without error (drc)")
|
||||
def step_then_no_error(context: Context) -> None:
|
||||
assert context.drc_result == "completed"
|
||||
|
||||
|
||||
@given("a DynamicRouterNode with routing patterns (drc)")
|
||||
def step_dynamic_router_node(context: Context) -> None:
|
||||
routes = [
|
||||
RoutePattern(pattern="GOTO_TARGET", target="target_node"),
|
||||
RoutePattern(pattern="GOTO_OTHER", target="other_node"),
|
||||
]
|
||||
context.drc_router = DynamicRouterNode(routes=routes)
|
||||
|
||||
|
||||
@given("a graph state with string messages (drc)")
|
||||
def step_state_string_messages(context: Context) -> None:
|
||||
context.drc_state = {"messages": ["prefix GOTO_TARGET: rest of message"]}
|
||||
|
||||
|
||||
@when("execute is called on the router (drc)")
|
||||
async def step_execute_router(context: Context):
|
||||
context.drc_result = await context.drc_router.execute(context.drc_state)
|
||||
|
||||
|
||||
@then("routing should work with string message content (drc)")
|
||||
def step_then_routing_works(context: Context) -> None:
|
||||
assert context.drc_result is not None
|
||||
assert context.drc_result.get("next_node") == "target_node"
|
||||
|
||||
|
||||
@given("a graph state with raw non-dict string messages (drc)")
|
||||
def step_state_raw_strings(context: Context) -> None:
|
||||
context.drc_state = {"messages": ["raw message one", "raw message two"]}
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Step definitions for ProgressBarManager coverage tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveractors.core.progress import ProgressBarManager
|
||||
|
||||
|
||||
@given("the progress test context is initialized (prg)")
|
||||
def step_prg_init(context: Context) -> None:
|
||||
context.prg_result = None
|
||||
|
||||
|
||||
@given("the progress manager update is called with total 10 and remaining 8 (prg)")
|
||||
def step_prg_update(context: Context) -> None:
|
||||
ProgressBarManager.update(
|
||||
stage="writing",
|
||||
total=10,
|
||||
remaining=8,
|
||||
current=2,
|
||||
)
|
||||
|
||||
|
||||
@when("the progress snapshot is retrieved (prg)")
|
||||
def step_prg_snapshot(context: Context) -> None:
|
||||
context.prg_result = ProgressBarManager.update(
|
||||
stage="writing",
|
||||
total=10,
|
||||
remaining=8,
|
||||
current=3,
|
||||
)
|
||||
|
||||
|
||||
@then("the snapshot should have total 10 and remaining 8 (prg)")
|
||||
def step_prg_check(context: Context) -> None:
|
||||
assert context.prg_result is not None
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Step definitions for RouteConfig graph conversion coverage tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveractors.reactive.route import RouteConfig, RouteType
|
||||
|
||||
|
||||
@given("the route graph config test context is initialized (rgc)")
|
||||
def step_rgc_init(context: Context) -> None:
|
||||
context.rgc_route_config = None
|
||||
context.rgc_result = None
|
||||
|
||||
|
||||
@given("a RouteConfig with MESSAGE_ROUTER node having rules (rgc)")
|
||||
def step_rgc_config(context: Context) -> None:
|
||||
context.rgc_route_config = RouteConfig(
|
||||
name="test_graph",
|
||||
type=RouteType.GRAPH,
|
||||
nodes={
|
||||
"router": {
|
||||
"type": "MESSAGE_ROUTER",
|
||||
"rules": [{"pattern": "test", "target": "target"}],
|
||||
"metadata": {"extra": "value"},
|
||||
},
|
||||
"target": {"type": "function"},
|
||||
},
|
||||
edges=[],
|
||||
entry_point="router",
|
||||
)
|
||||
|
||||
|
||||
@when("to_graph_config is called (rgc)")
|
||||
def step_rgc_to_graph(context: Context) -> None:
|
||||
context.rgc_result = context.rgc_route_config.to_graph_config()
|
||||
|
||||
|
||||
@then("the MESSAGE_ROUTER rules should appear in node metadata (rgc)")
|
||||
def step_rgc_check(context: Context) -> None:
|
||||
assert context.rgc_result is not None
|
||||
found = False
|
||||
for _name, node in context.rgc_result.nodes.items():
|
||||
if node.metadata and "rules" in node.metadata:
|
||||
found = True
|
||||
assert found, "Expected MESSAGE_ROUTER rules in node metadata"
|
||||
@@ -361,12 +361,15 @@ async def step_execute_actor(context, msg):
|
||||
context.test_executor = executor
|
||||
|
||||
with (
|
||||
patch("cleveractors.runtime.TemplateRenderer") as mock_renderer,
|
||||
patch("cleveractors.runtime.ToolAgent") as mock_tool,
|
||||
patch("cleveractors.runtime.AgentFactory") as mock_factory,
|
||||
patch("cleveractors.runtime.PureLangGraph") as mock_pure_graph,
|
||||
patch("cleveractors.runtime.estimate_tokens", return_value=(100, 50)),
|
||||
patch("cleveractors.runtime.estimate_graph_tokens", return_value=(100, 50)),
|
||||
patch("cleveractors.templates.renderer.TemplateRenderer") as mock_renderer,
|
||||
patch("cleveractors.agents.tool.ToolAgent") as mock_tool,
|
||||
patch("cleveractors.agents.llm.LLMAgent") as mock_llm,
|
||||
patch("cleveractors.agents.factory.AgentFactory") as mock_factory,
|
||||
patch("cleveractors.langgraph.pure_graph.PureLangGraph") as mock_pure_graph,
|
||||
patch("cleveractors.runtime._estimate_tokens", return_value=(100, 50)),
|
||||
patch(
|
||||
"cleveractors.runtime_tokens.estimate_graph_tokens", return_value=(100, 50)
|
||||
),
|
||||
):
|
||||
mock_renderer_instance = MagicMock()
|
||||
mock_renderer.return_value = mock_renderer_instance
|
||||
@@ -384,6 +387,8 @@ async def step_execute_actor(context, msg):
|
||||
)
|
||||
mock_llm_instance.cleanup = AsyncMock()
|
||||
|
||||
mock_llm.return_value = mock_llm_instance
|
||||
|
||||
mock_tool_instance = MagicMock()
|
||||
mock_tool_instance.process_message = AsyncMock(
|
||||
return_value="Mock tool response"
|
||||
@@ -391,7 +396,9 @@ async def step_execute_actor(context, msg):
|
||||
mock_tool.return_value = mock_tool_instance
|
||||
|
||||
mock_graph_instance = MagicMock()
|
||||
mock_graph_instance.execute = AsyncMock(return_value="Mock graph response")
|
||||
mock_graph_instance.execute = AsyncMock(
|
||||
return_value=("Mock graph response", {})
|
||||
)
|
||||
mock_pure_graph.return_value = mock_graph_instance
|
||||
|
||||
mock_factory_instance = MagicMock()
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
"""Step definitions for Runtime Executor Extended Coverage BDD tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.api.async_step import async_run_until_complete
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveractors.core.exceptions import ConfigurationError
|
||||
from cleveractors.runtime import Executor, create_executor
|
||||
|
||||
|
||||
@given("the runtime extended test context is initialized (rxe)")
|
||||
def step_rxe_init(context: Context) -> None:
|
||||
context.rxe_config = None
|
||||
context.rxe_credentials = {}
|
||||
context.rxe_result = None
|
||||
context.rxe_error = None
|
||||
context.rxe_messages = None
|
||||
|
||||
|
||||
@given("credentials dict with openai provider (rxe)")
|
||||
def step_rxe_creds_openai(context: Context) -> None:
|
||||
context.rxe_credentials = {"openai": {"api_key": "test-key"}}
|
||||
|
||||
|
||||
@given("conversation history messages are provided (rxe)")
|
||||
def step_rxe_messages(context: Context) -> None:
|
||||
context.rxe_messages = [
|
||||
{"role": "user", "content": "previous message"},
|
||||
{"role": "assistant", "content": "previous response"},
|
||||
]
|
||||
|
||||
|
||||
# ── Config Given steps ──
|
||||
|
||||
|
||||
@given("a config dict with routes key but no type field (rxe)")
|
||||
def step_rxe_routes_no_type(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_v2_graph",
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": {"start": {"type": "start"}, "end": {"type": "end"}},
|
||||
"edges": [{"source": "start", "target": "end"}],
|
||||
"entry_point": "start",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2.0 routes format with dict nodes (rxe)")
|
||||
def step_rxe_v2_dict_nodes(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_v2_dict",
|
||||
"type": "graph",
|
||||
"actors": {
|
||||
"node_a": {
|
||||
"name": "node_a",
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
"type": "llm",
|
||||
},
|
||||
},
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": {"node_a": {}, "node_b": {"type": "function"}},
|
||||
"edges": [
|
||||
{"source": "node_a", "target": "node_b"},
|
||||
{"source": "node_b", "target": "end"},
|
||||
],
|
||||
"entry_point": "node_a",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2.0 routes format with list nodes (rxe)")
|
||||
def step_rxe_v2_list_nodes(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_v2_list",
|
||||
"type": "graph",
|
||||
"actors": {
|
||||
"node_a": {
|
||||
"name": "node_a",
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
"type": "llm",
|
||||
},
|
||||
},
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": [
|
||||
{"id": "node_a", "type": "agent", "agent": "node_a"},
|
||||
{"id": "node_b", "type": "function"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "node_a", "target": "node_b"},
|
||||
{"source": "node_b", "target": "end"},
|
||||
],
|
||||
"entry_point": "node_a",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given(
|
||||
"a config dict with type llm provider model and system_prompt in config block (rxe)"
|
||||
)
|
||||
def step_rxe_llm_config_block(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_llm",
|
||||
"type": "llm",
|
||||
"config": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
"system_prompt": "You are helpful",
|
||||
"temperature": 0.5,
|
||||
"max_tokens": 500,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type llm and provider model in nested config block (rxe)")
|
||||
def step_rxe_llm_nested(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_llm",
|
||||
"type": "llm",
|
||||
"config": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2.0 routes with actors block (rxe)")
|
||||
def step_rxe_graph_actors(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_graph_actors",
|
||||
"type": "graph",
|
||||
"actors": {
|
||||
"node_a": {
|
||||
"name": "node_a",
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
"type": "llm",
|
||||
},
|
||||
},
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": {"node_a": {}, "node_b": {"type": "function"}},
|
||||
"edges": [
|
||||
{"source": "node_a", "target": "node_b"},
|
||||
{"source": "node_b", "target": "end"},
|
||||
],
|
||||
"entry_point": "node_a",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type graph and route with invalid node type (rxe)")
|
||||
def step_rxe_invalid_node_type(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_bad_node",
|
||||
"type": "graph",
|
||||
"actors": {
|
||||
"node_a": {
|
||||
"name": "node_a",
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
"type": "llm",
|
||||
},
|
||||
},
|
||||
"route": {
|
||||
"nodes": [
|
||||
{"id": "node_a", "agent": "node_a", "type": "weird_type"},
|
||||
{"id": "node_b", "type": "function"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "node_a", "target": "node_b"},
|
||||
{"source": "node_b", "target": "end"},
|
||||
],
|
||||
"entry_node": "node_a",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given(
|
||||
"a config dict with type graph and route with agents block having missing agent (rxe)"
|
||||
)
|
||||
def step_rxe_missing_agent(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_missing_agent",
|
||||
"type": "graph",
|
||||
"actors": {
|
||||
"node_b": {
|
||||
"name": "node_b",
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
"type": "llm",
|
||||
},
|
||||
},
|
||||
"route": {
|
||||
"nodes": [
|
||||
{"id": "node_a", "agent": "nonexistent_agent"},
|
||||
{"id": "node_b", "type": "function"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "node_a", "target": "node_b"},
|
||||
{"source": "node_b", "target": "end"},
|
||||
],
|
||||
"entry_node": "node_a",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type graph and route with global context (rxe)")
|
||||
def step_rxe_global_context(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_global_ctx",
|
||||
"type": "graph",
|
||||
"context": {
|
||||
"global": {"stage_order": ["intro", "body", "conclusion"]},
|
||||
},
|
||||
"route": {
|
||||
"nodes": [{"id": "start", "type": "start"}, {"id": "end", "type": "end"}],
|
||||
"edges": [{"source": "start", "target": "end"}],
|
||||
"entry_node": "start",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given(
|
||||
"a config dict with type graph and route with actor context without global key (rxe)"
|
||||
)
|
||||
def step_rxe_actor_context_no_global(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_actor_ctx",
|
||||
"type": "graph",
|
||||
"context": {"stage_order": ["intro", "body"]},
|
||||
"route": {
|
||||
"nodes": [{"id": "start", "type": "start"}, {"id": "end", "type": "end"}],
|
||||
"edges": [{"source": "start", "target": "end"}],
|
||||
"entry_node": "start",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type tool and tools list (rxe)")
|
||||
def step_rxe_tool_config(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_tool",
|
||||
"type": "tool",
|
||||
"tools": ["echo"],
|
||||
}
|
||||
|
||||
|
||||
@given("a multi-actor config dict with cleveragents default_actor (rxe)")
|
||||
def step_rxe_multi_default(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_multi",
|
||||
"type": "multi_actor",
|
||||
"cleveragents": {"default_actor": "actor_llm"},
|
||||
"actors": {
|
||||
"actor_llm": {
|
||||
"name": "actor_llm",
|
||||
"type": "llm",
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
},
|
||||
"actor_tool": {
|
||||
"name": "actor_tool",
|
||||
"type": "tool",
|
||||
"tools": ["echo"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@given("a config dict with type llm and unknown provider (rxe)")
|
||||
def step_rxe_unknown_provider(context: Context) -> None:
|
||||
context.rxe_config = {
|
||||
"name": "test_unknown",
|
||||
"type": "llm",
|
||||
"provider": "unknown_provider",
|
||||
"model": "unsupported-model",
|
||||
}
|
||||
|
||||
|
||||
# ── When step ──
|
||||
|
||||
|
||||
@when('I execute the extended runtime actor with message "{message}" (rxe)')
|
||||
@async_run_until_complete
|
||||
async def step_rxe_execute(context: Context, message: str) -> None:
|
||||
mock_llm_inst = MagicMock()
|
||||
mock_llm_inst.process_message = AsyncMock(return_value="Mock LLM response")
|
||||
mock_llm_inst.cleanup = AsyncMock()
|
||||
|
||||
mock_tool_inst = MagicMock()
|
||||
mock_tool_inst.process_message = AsyncMock(return_value="Mock tool response")
|
||||
mock_tool_inst.cleanup = AsyncMock()
|
||||
|
||||
mock_graph_inst = MagicMock()
|
||||
mock_graph_inst.execute = AsyncMock(return_value=("Mock graph response", {}))
|
||||
mock_graph_inst.dispose = AsyncMock()
|
||||
|
||||
mock_factory_inst = MagicMock()
|
||||
mock_factory_inst.create_agent = MagicMock(return_value=mock_llm_inst)
|
||||
|
||||
with (
|
||||
patch("cleveractors.templates.renderer.TemplateRenderer") as mock_renderer,
|
||||
patch("cleveractors.agents.tool.ToolAgent") as mock_tool,
|
||||
patch("cleveractors.agents.llm.LLMAgent") as mock_llm,
|
||||
patch("cleveractors.agents.factory.AgentFactory") as mock_factory,
|
||||
patch("cleveractors.langgraph.pure_graph.PureLangGraph") as mock_pure_graph,
|
||||
patch("cleveractors.langgraph.pure_graph.PureGraphConfig") as mock_pg_config,
|
||||
patch("cleveractors.runtime._estimate_tokens", return_value=(100, 50)),
|
||||
):
|
||||
mock_renderer_inst = MagicMock()
|
||||
mock_renderer.return_value = mock_renderer_inst
|
||||
mock_llm.return_value = mock_llm_inst
|
||||
mock_tool.return_value = mock_tool_inst
|
||||
mock_pure_graph.return_value = mock_graph_inst
|
||||
mock_pg_config.return_value = MagicMock()
|
||||
mock_factory.return_value = mock_factory_inst
|
||||
|
||||
try:
|
||||
executor = Executor(
|
||||
config_dict=context.rxe_config,
|
||||
credentials=context.rxe_credentials,
|
||||
limits={},
|
||||
pricing={},
|
||||
)
|
||||
context.rxe_result = await executor.execute(
|
||||
message, messages=context.rxe_messages
|
||||
)
|
||||
context.rxe_error = None
|
||||
except ConfigurationError as exc:
|
||||
context.rxe_error = exc
|
||||
context.rxe_result = None
|
||||
|
||||
|
||||
# ── Then steps ──
|
||||
|
||||
|
||||
@then("the execution should return an ActorResult (rxe)")
|
||||
def step_then_actor_result_rxe(context: Context) -> None:
|
||||
assert context.rxe_result is not None, (
|
||||
f"Expected ActorResult but got error: {context.rxe_error}"
|
||||
)
|
||||
assert context.rxe_result.response is not None
|
||||
|
||||
|
||||
@then("the node usage IDs should be prefixed with the default actor name (rxe)")
|
||||
def step_then_prefixed_ids_rxe(context: Context) -> None:
|
||||
assert context.rxe_result is not None
|
||||
assert len(context.rxe_result.nodes) > 0
|
||||
for node in context.rxe_result.nodes:
|
||||
assert "." in node.node_id, f"Expected prefixed node ID, got {node.node_id}"
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised about missing credentials (rxe)")
|
||||
def step_then_missing_creds_rxe(context: Context) -> None:
|
||||
assert context.rxe_error is not None
|
||||
assert isinstance(context.rxe_error, ConfigurationError)
|
||||
err_msg = str(context.rxe_error).lower()
|
||||
assert "credential" in err_msg or "provider" in err_msg
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Step definitions for runtime_tokens.py coverage tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
|
||||
@given("the runtime tokens test context is initialized (rtc)")
|
||||
def step_rtc_init(context: Context) -> None:
|
||||
context.rtc_tiktoken_avail = True
|
||||
context.rtc_mock_encoding = None
|
||||
context.rtc_prompt_tokens = None
|
||||
context.rtc_completion_tokens = None
|
||||
context.rtc_used_cl100k = False
|
||||
|
||||
|
||||
@given("tiktoken is available (rtc)")
|
||||
def step_tiktoken_avail(context: Context) -> None:
|
||||
context.rtc_tiktoken_avail = True
|
||||
context.rtc_enc_raises = False
|
||||
|
||||
|
||||
@given("tiktoken is not available (rtc)")
|
||||
def step_tiktoken_not_avail(context: Context) -> None:
|
||||
context.rtc_tiktoken_avail = False
|
||||
|
||||
|
||||
@given("a mock tiktoken encoding is configured (rtc)")
|
||||
def step_mock_encoding(context: Context) -> None:
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.encode.return_value = [1, 2, 3, 4, 5]
|
||||
context.rtc_mock_encoding = mock_enc
|
||||
|
||||
|
||||
@given("tiktoken encoding raises an exception (rtc)")
|
||||
def step_tiktoken_raises(context: Context) -> None:
|
||||
context.rtc_tiktoken_avail = True
|
||||
context.rtc_enc_raises = True
|
||||
|
||||
|
||||
@when(
|
||||
'estimate_tokens is called with prompt "{prompt}" response "{response}" model "{model}" provider "{provider}" (rtc)'
|
||||
)
|
||||
def step_estimate_tokens(
|
||||
context: Context, prompt: str, response: str, model: str, provider: str
|
||||
) -> None:
|
||||
# We need to patch cleveractors.runtime_tokens to control tiktoken availability
|
||||
import cleveractors.runtime_tokens as rt
|
||||
|
||||
if context.rtc_tiktoken_avail and not context.rtc_enc_raises:
|
||||
# Mock tiktoken to be available with our mock encoding
|
||||
mock_tiktoken = MagicMock()
|
||||
mock_tiktoken.encoding_for_model = MagicMock(
|
||||
return_value=context.rtc_mock_encoding
|
||||
)
|
||||
mock_tiktoken.get_encoding = MagicMock(return_value=context.rtc_mock_encoding)
|
||||
# Check if cl100k_base was requested
|
||||
orig_get_encoding = mock_tiktoken.get_encoding
|
||||
|
||||
def _track_get_encoding(name):
|
||||
if name == "cl100k_base":
|
||||
context.rtc_used_cl100k = True
|
||||
return context.rtc_mock_encoding
|
||||
|
||||
mock_tiktoken.get_encoding = _track_get_encoding
|
||||
|
||||
with (
|
||||
patch.object(rt, "_TIKTOKEN_AVAILABLE", True),
|
||||
patch.object(rt, "_tiktoken", mock_tiktoken),
|
||||
):
|
||||
context.rtc_prompt_tokens, context.rtc_completion_tokens = (
|
||||
rt.estimate_tokens(prompt, response, model, provider)
|
||||
)
|
||||
elif context.rtc_tiktoken_avail and context.rtc_enc_raises:
|
||||
mock_tiktoken = MagicMock()
|
||||
mock_tiktoken.encoding_for_model = MagicMock(
|
||||
side_effect=ValueError("encoding error")
|
||||
)
|
||||
mock_tiktoken.get_encoding = MagicMock(side_effect=ValueError("encoding error"))
|
||||
|
||||
with (
|
||||
patch.object(rt, "_TIKTOKEN_AVAILABLE", True),
|
||||
patch.object(rt, "_tiktoken", mock_tiktoken),
|
||||
):
|
||||
context.rtc_prompt_tokens, context.rtc_completion_tokens = (
|
||||
rt.estimate_tokens(prompt, response, model, provider)
|
||||
)
|
||||
else:
|
||||
with (
|
||||
patch.object(rt, "_TIKTOKEN_AVAILABLE", False),
|
||||
patch.object(rt, "_tiktoken", None),
|
||||
):
|
||||
context.rtc_prompt_tokens, context.rtc_completion_tokens = (
|
||||
rt.estimate_tokens(prompt, response, model, provider)
|
||||
)
|
||||
|
||||
|
||||
@when(
|
||||
'estimate_graph_tokens is called with prompt "{prompt}" response "{response}" (rtc)'
|
||||
)
|
||||
def step_estimate_graph_tokens(context: Context, prompt: str, response: str) -> None:
|
||||
import cleveractors.runtime_tokens as rt
|
||||
|
||||
if context.rtc_tiktoken_avail and not context.rtc_enc_raises:
|
||||
mock_tiktoken = MagicMock()
|
||||
mock_tiktoken.get_encoding = MagicMock(return_value=context.rtc_mock_encoding)
|
||||
|
||||
with (
|
||||
patch.object(rt, "_TIKTOKEN_AVAILABLE", True),
|
||||
patch.object(rt, "_tiktoken", mock_tiktoken),
|
||||
):
|
||||
context.rtc_prompt_tokens, context.rtc_completion_tokens = (
|
||||
rt.estimate_graph_tokens(prompt, response)
|
||||
)
|
||||
elif context.rtc_tiktoken_avail and context.rtc_enc_raises:
|
||||
mock_tiktoken = MagicMock()
|
||||
mock_tiktoken.get_encoding = MagicMock(side_effect=ValueError("encoding error"))
|
||||
|
||||
with (
|
||||
patch.object(rt, "_TIKTOKEN_AVAILABLE", True),
|
||||
patch.object(rt, "_tiktoken", mock_tiktoken),
|
||||
):
|
||||
context.rtc_prompt_tokens, context.rtc_completion_tokens = (
|
||||
rt.estimate_graph_tokens(prompt, response)
|
||||
)
|
||||
else:
|
||||
with (
|
||||
patch.object(rt, "_TIKTOKEN_AVAILABLE", False),
|
||||
patch.object(rt, "_tiktoken", None),
|
||||
):
|
||||
context.rtc_prompt_tokens, context.rtc_completion_tokens = (
|
||||
rt.estimate_graph_tokens(prompt, response)
|
||||
)
|
||||
|
||||
|
||||
@then("prompt tokens and completion tokens should be returned from tiktoken (rtc)")
|
||||
def step_then_tiktoken_result(context: Context) -> None:
|
||||
assert context.rtc_prompt_tokens is not None
|
||||
assert context.rtc_completion_tokens is not None
|
||||
assert context.rtc_prompt_tokens > 0
|
||||
assert context.rtc_completion_tokens > 0
|
||||
|
||||
|
||||
@then("the cl100k_base encoding should be used for token estimation (rtc)")
|
||||
def step_then_cl100k_used(context: Context) -> None:
|
||||
assert context.rtc_prompt_tokens is not None
|
||||
assert context.rtc_completion_tokens is not None
|
||||
assert context.rtc_used_cl100k, "Expected cl100k_base encoding to be used"
|
||||
|
||||
|
||||
@then("token counts should be estimated using the character heuristic (rtc)")
|
||||
def step_then_heuristic_tokens(context: Context) -> None:
|
||||
assert context.rtc_prompt_tokens is not None
|
||||
assert context.rtc_completion_tokens is not None
|
||||
assert context.rtc_prompt_tokens > 0
|
||||
assert context.rtc_completion_tokens > 0
|
||||
|
||||
|
||||
@then("token counts should fall back to heuristic estimation (rtc)")
|
||||
def step_then_fallback_heuristic(context: Context) -> None:
|
||||
assert context.rtc_prompt_tokens is not None
|
||||
assert context.rtc_completion_tokens is not None
|
||||
assert context.rtc_prompt_tokens > 0
|
||||
assert context.rtc_completion_tokens > 0
|
||||
|
||||
|
||||
@then("graph token counts should be returned from tiktoken (rtc)")
|
||||
def step_then_graph_tiktoken(context: Context) -> None:
|
||||
assert context.rtc_prompt_tokens is not None
|
||||
assert context.rtc_completion_tokens is not None
|
||||
assert context.rtc_prompt_tokens > 0
|
||||
assert context.rtc_completion_tokens > 0
|
||||
|
||||
|
||||
@then("graph token counts should be estimated using the character heuristic (rtc)")
|
||||
def step_then_graph_heuristic(context: Context) -> None:
|
||||
assert context.rtc_prompt_tokens is not None
|
||||
assert context.rtc_completion_tokens is not None
|
||||
assert context.rtc_prompt_tokens > 0
|
||||
assert context.rtc_completion_tokens > 0
|
||||
|
||||
|
||||
@then("graph token counts should fall back to heuristic estimation (rtc)")
|
||||
def step_then_graph_fallback(context: Context) -> None:
|
||||
assert context.rtc_prompt_tokens is not None
|
||||
assert context.rtc_completion_tokens is not None
|
||||
assert context.rtc_prompt_tokens > 0
|
||||
assert context.rtc_completion_tokens > 0
|
||||
@@ -0,0 +1,643 @@
|
||||
"""Step definitions for validation/_actor.py coverage tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveractors.core.exceptions import ConfigurationError
|
||||
from cleveractors.validation._actor import (
|
||||
_infer_actor_type,
|
||||
_validate_graph_actor,
|
||||
_validate_llm_actor,
|
||||
_validate_multi_actor,
|
||||
_validate_tool_actor,
|
||||
validate_actor_config,
|
||||
)
|
||||
|
||||
|
||||
@given("the validation actor test context is initialized (vac)")
|
||||
def step_vac_init(context: Context) -> None:
|
||||
context.vac_config = {}
|
||||
context.vac_limits = {}
|
||||
context.vac_error = None
|
||||
|
||||
|
||||
def _set_config_dict(context: Context, config: dict[str, Any]) -> None:
|
||||
context.vac_config = config
|
||||
|
||||
|
||||
def _set_limits(context: Context, limits: dict[str, Any]) -> None:
|
||||
context.vac_limits = limits
|
||||
|
||||
|
||||
# ── validate_actor_config dispatch ──
|
||||
|
||||
|
||||
@given("a config dict with type graph and route in legacy format (vac)")
|
||||
def step_graph_route_legacy(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"route": {"nodes": [{"id": "a"}], "edges": [], "entry_node": "a"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type llm provider and model (vac)")
|
||||
def step_llm_valid(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "llm",
|
||||
"name": "test_llm",
|
||||
"provider": "openai",
|
||||
"model": "gpt-3.5-turbo",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type tool and tools list (vac)")
|
||||
def step_tool_valid(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "tool",
|
||||
"name": "test_tool",
|
||||
"tools": ["echo"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type multi_actor and actors mapping (vac)")
|
||||
def step_multi_valid(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "multi_actor",
|
||||
"actors": {
|
||||
"actor1": {
|
||||
"type": "llm",
|
||||
"name": "a1",
|
||||
"model": "gpt-4",
|
||||
"provider": "openai",
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── _infer_actor_type implicit detection ──
|
||||
|
||||
|
||||
@given("a config dict without type but with top-level routes key (vac)")
|
||||
def step_no_type_routes(context: Context) -> None:
|
||||
_set_config_dict(context, {"routes": {}})
|
||||
|
||||
|
||||
@given("a config dict without type but with top-level actors key (vac)")
|
||||
def step_no_type_actors(context: Context) -> None:
|
||||
_set_config_dict(context, {"actors": {}})
|
||||
|
||||
|
||||
@given(
|
||||
"a config dict without type but with routes containing spec-level stream entry (vac)"
|
||||
)
|
||||
def step_no_type_routes_spec(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{"routes": {"main": {"type": "stream"}}},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with integer type field (vac)")
|
||||
def step_int_type(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": 42})
|
||||
|
||||
|
||||
@given("a config dict with unknown actor type (vac)")
|
||||
def step_unknown_type(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "banana"})
|
||||
|
||||
|
||||
@given("a config dict without type routes or actors (vac)")
|
||||
def step_no_type_no_routes_no_actors(context: Context) -> None:
|
||||
_set_config_dict(context, {"name": "orphan"})
|
||||
|
||||
|
||||
# ── _validate_graph_actor legacy ──
|
||||
|
||||
|
||||
@given(
|
||||
"a config dict with type graph and legacy route with nodes edges entry_node (vac)"
|
||||
)
|
||||
def step_legacy_valid(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"route": {
|
||||
"nodes": [{"id": "start"}, {"id": "end"}],
|
||||
"edges": [{"source": "start", "target": "end"}],
|
||||
"entry_node": "start",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph and legacy route without edges (vac)")
|
||||
def step_legacy_no_edges(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"route": {
|
||||
"nodes": [{"id": "start"}],
|
||||
"entry_node": "start",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph and legacy route without entry_node (vac)")
|
||||
def step_legacy_no_entry(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"route": {
|
||||
"nodes": [{"id": "start"}],
|
||||
"edges": [],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph and legacy route with 100 nodes (vac)")
|
||||
def step_legacy_too_many_nodes(context: Context) -> None:
|
||||
nodes = [{"id": f"node_{i}"} for i in range(100)]
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"route": {
|
||||
"nodes": nodes,
|
||||
"edges": [],
|
||||
"entry_node": "node_0",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@when("platform limits with max_total_nodes {limit:d} (vac)")
|
||||
def step_limits_max_nodes(context: Context, limit: int) -> None:
|
||||
_set_limits(context, {"max_total_nodes": limit})
|
||||
|
||||
|
||||
@given("platform limits with max_total_nodes 10 (vac)")
|
||||
def step_limits_max_nodes_10(context: Context) -> None:
|
||||
_set_limits(context, {"max_total_nodes": 10})
|
||||
|
||||
|
||||
@given("platform limits with max_total_nodes 3 (vac)")
|
||||
def step_limits_max_nodes_3(context: Context) -> None:
|
||||
_set_limits(context, {"max_total_nodes": 3})
|
||||
|
||||
|
||||
@given("platform limits with max_total_nodes 100 (vac)")
|
||||
def step_limits_max_nodes_100(context: Context) -> None:
|
||||
_set_limits(context, {"max_total_nodes": 100})
|
||||
|
||||
|
||||
# ── _validate_graph_actor v2.0 ──
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2 routes with nodes edges entry_point (vac)")
|
||||
def step_v2_valid(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": {"a": {}, "b": {}},
|
||||
"edges": [{"source": "a", "target": "b"}],
|
||||
"entry_point": "a",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2 routes without edges (vac)")
|
||||
def step_v2_no_edges(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": {"a": {}},
|
||||
"entry_point": "a",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2 routes without entry_point (vac)")
|
||||
def step_v2_no_entry_point(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": {"a": {}},
|
||||
"edges": [],
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2 routes with 200 nodes (vac)")
|
||||
def step_v2_too_many_nodes(context: Context) -> None:
|
||||
nodes = {f"node_{i}": {} for i in range(200)}
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"routes": {
|
||||
"main": {
|
||||
"nodes": nodes,
|
||||
"edges": [],
|
||||
"entry_point": "node_0",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2 routes with non-dict main (vac)")
|
||||
def step_v2_non_dict_main(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "graph",
|
||||
"routes": "not a dict",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type graph but no route or routes key (vac)")
|
||||
def step_graph_no_route_or_routes(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "graph"})
|
||||
|
||||
|
||||
# ── _validate_llm_actor ──
|
||||
|
||||
|
||||
@given("a config dict with type llm but no name field (vac)")
|
||||
def step_llm_no_name(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "llm", "provider": "openai", "model": "gpt-4"})
|
||||
|
||||
|
||||
@given("a config dict with type llm name but no provider (vac)")
|
||||
def step_llm_no_provider(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "llm", "name": "test", "model": "gpt-4"})
|
||||
|
||||
|
||||
@given("a config dict with type llm name provider but no model (vac)")
|
||||
def step_llm_no_model(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "llm", "name": "test", "provider": "openai"})
|
||||
|
||||
|
||||
@given("a config dict with type llm name and provider model in config block (vac)")
|
||||
def step_llm_config_block(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "llm",
|
||||
"name": "test",
|
||||
"config": {"provider": "openai", "model": "gpt-4"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── _validate_tool_actor ──
|
||||
|
||||
|
||||
@given("a config dict with type tool but no name field (vac)")
|
||||
def step_tool_no_name(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "tool", "tools": ["echo"]})
|
||||
|
||||
|
||||
@given("a config dict with type tool name but no tools (vac)")
|
||||
def step_tool_no_tools(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "tool", "name": "test"})
|
||||
|
||||
|
||||
@given("a config dict with type tool name and tools in config block (vac)")
|
||||
def step_tool_config_block(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "tool",
|
||||
"name": "test",
|
||||
"config": {"tools": ["echo"]},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── _validate_multi_actor ──
|
||||
|
||||
|
||||
@given("a config dict with type multi_actor and empty actors (vac)")
|
||||
def step_multi_empty_actors(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "multi_actor", "actors": {}})
|
||||
|
||||
|
||||
@given("a config dict with type multi_actor and non-dict actors (vac)")
|
||||
def step_multi_non_dict_actors(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "multi_actor", "actors": "not dict"})
|
||||
|
||||
|
||||
@given("a config dict with type multi_actor containing many graph actors (vac)")
|
||||
def step_multi_many_graph_actors(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "multi_actor",
|
||||
"actors": {
|
||||
f"g{i}": {
|
||||
"type": "graph",
|
||||
"route": {
|
||||
"nodes": [{"id": f"n{j}"} for j in range(5)],
|
||||
"edges": [],
|
||||
"entry_node": "n0",
|
||||
},
|
||||
}
|
||||
for i in range(5)
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@given("a config dict with type multi_actor containing few graph actors (vac)")
|
||||
def step_multi_few_graph_actors(context: Context) -> None:
|
||||
_set_config_dict(
|
||||
context,
|
||||
{
|
||||
"type": "multi_actor",
|
||||
"actors": {
|
||||
"g1": {
|
||||
"type": "graph",
|
||||
"route": {
|
||||
"nodes": [{"id": "a"}, {"id": "b"}],
|
||||
"edges": [],
|
||||
"entry_node": "a",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── When steps ──
|
||||
|
||||
|
||||
def _call_and_capture(context: Context, fn, *args):
|
||||
try:
|
||||
fn(*args)
|
||||
context.vac_error = None
|
||||
except ConfigurationError as exc:
|
||||
context.vac_error = exc
|
||||
except Exception as exc:
|
||||
context.vac_error = exc
|
||||
|
||||
|
||||
@when("validate_actor_config is called (vac)")
|
||||
def step_when_validate_actor_config(context: Context) -> None:
|
||||
_call_and_capture(
|
||||
context,
|
||||
validate_actor_config,
|
||||
context.vac_config,
|
||||
context.vac_limits,
|
||||
)
|
||||
|
||||
|
||||
@when("_infer_actor_type is called (vac)")
|
||||
def step_when_infer_actor_type(context: Context) -> None:
|
||||
try:
|
||||
context.vac_result = _infer_actor_type(context.vac_config)
|
||||
context.vac_error = None
|
||||
except ConfigurationError as exc:
|
||||
context.vac_error = exc
|
||||
context.vac_result = None
|
||||
|
||||
|
||||
@when("_validate_graph_actor is called (vac)")
|
||||
def step_when_validate_graph_actor(context: Context) -> None:
|
||||
_call_and_capture(
|
||||
context,
|
||||
_validate_graph_actor,
|
||||
context.vac_config,
|
||||
context.vac_limits,
|
||||
)
|
||||
|
||||
|
||||
@when("_validate_llm_actor is called (vac)")
|
||||
def step_when_validate_llm_actor(context: Context) -> None:
|
||||
_call_and_capture(context, _validate_llm_actor, context.vac_config)
|
||||
|
||||
|
||||
@when("_validate_tool_actor is called (vac)")
|
||||
def step_when_validate_tool_actor(context: Context) -> None:
|
||||
_call_and_capture(context, _validate_tool_actor, context.vac_config)
|
||||
|
||||
|
||||
@when("_validate_multi_actor is called (vac)")
|
||||
def step_when_validate_multi_actor(context: Context) -> None:
|
||||
_call_and_capture(
|
||||
context,
|
||||
_validate_multi_actor,
|
||||
context.vac_config,
|
||||
context.vac_limits,
|
||||
)
|
||||
|
||||
|
||||
# ── Then steps ──
|
||||
|
||||
|
||||
@then("no error should be raised for valid {config_type} config (vac)")
|
||||
def step_then_no_error(context: Context, config_type: str) -> None:
|
||||
assert context.vac_error is None, (
|
||||
f"Expected no error for {config_type} but got {context.vac_error}"
|
||||
)
|
||||
|
||||
|
||||
@then("the inferred type should be {expected_type} (vac)")
|
||||
def step_then_inferred_type(context: Context, expected_type: str) -> None:
|
||||
assert context.vac_error is None, f"Expected no error but got {context.vac_error}"
|
||||
assert context.vac_result == expected_type, (
|
||||
f"Expected {expected_type} but got {context.vac_result}"
|
||||
)
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised mentioning missing agents key (vac)")
|
||||
def step_then_missing_agents(context: Context) -> None:
|
||||
assert context.vac_error is not None, "Expected ConfigurationError but got none"
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "agents" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for non-string type (vac)")
|
||||
def step_then_non_string_type(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "string" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for unknown type (vac)")
|
||||
def step_then_unknown_type(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "unknown" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing type (vac)")
|
||||
def step_then_missing_type(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing edges (vac)")
|
||||
def step_then_missing_edges(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "edge" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing entry_node (vac)")
|
||||
def step_then_missing_entry_node(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "entry_node" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for exceeding max nodes (vac)")
|
||||
def step_then_exceeds_max_nodes(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "node" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing edges in routes main (vac)")
|
||||
def step_then_v2_missing_edges(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "edge" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then(
|
||||
"a ConfigurationError should be raised for missing entry_point in routes main (vac)"
|
||||
)
|
||||
def step_then_v2_missing_entry(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "entry_point" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for non-mapping routes main (vac)")
|
||||
def step_then_non_dict_main(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "mapping" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing route or routes (vac)")
|
||||
def step_then_missing_route_or_routes(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert (
|
||||
"route" in str(context.vac_error).lower()
|
||||
or "routes" in str(context.vac_error).lower()
|
||||
)
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing name in llm (vac)")
|
||||
def step_then_llm_no_name(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "name" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing provider (vac)")
|
||||
def step_then_missing_provider(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "provider" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing model (vac)")
|
||||
def step_then_missing_model(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "model" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing name in tool (vac)")
|
||||
def step_then_tool_no_name(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "name" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for missing tools (vac)")
|
||||
def step_then_missing_tools(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "tool" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for empty actors mapping (vac)")
|
||||
def step_then_empty_actors(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "actor" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
@then("a ConfigurationError should be raised for exceeding total nodes (vac)")
|
||||
def step_then_exceed_total_nodes(context: Context) -> None:
|
||||
assert context.vac_error is not None
|
||||
assert isinstance(context.vac_error, ConfigurationError)
|
||||
assert "node" in str(context.vac_error).lower()
|
||||
|
||||
|
||||
# ── Additional edge case Given steps ──
|
||||
|
||||
|
||||
@given("platform limits with string max_total_nodes value (vac)")
|
||||
def step_vac_string_max_nodes(context: Context) -> None:
|
||||
_set_limits(context, {"max_total_nodes": "not_an_int"})
|
||||
|
||||
|
||||
@given("a config dict with type graph and v2 routes having non-dict main value (vac)")
|
||||
def step_vac_v2_non_dict_main(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "graph", "routes": {"main": "not_a_dict"}})
|
||||
|
||||
|
||||
@given(
|
||||
"a config dict with type llm name without provider and string config block (vac)"
|
||||
)
|
||||
def step_vac_llm_string_config_no_provider(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "llm", "name": "test", "config": "not_a_dict"})
|
||||
|
||||
|
||||
@given("a config dict with type tool name and string config block (vac)")
|
||||
def step_vac_tool_string_config(context: Context) -> None:
|
||||
_set_config_dict(context, {"type": "tool", "name": "test", "config": "not_a_dict"})
|
||||
@@ -0,0 +1,205 @@
|
||||
Feature: Validation Actor Runtime Config Validation
|
||||
As a developer
|
||||
I want actor-level runtime configs to be validated correctly for graph, LLM, tool, and multi_actor types
|
||||
So that only well-formed individual actor configs are accepted by the validation layer
|
||||
|
||||
Background:
|
||||
Given the validation actor test context is initialized (vac)
|
||||
|
||||
# ── validate_actor_config dispatch ──
|
||||
|
||||
Scenario: validate_actor_config dispatches to graph validator for type graph
|
||||
Given a config dict with type graph and route in legacy format (vac)
|
||||
When validate_actor_config is called (vac)
|
||||
Then no error should be raised for valid graph config (vac)
|
||||
|
||||
Scenario: validate_actor_config dispatches to llm validator for type llm
|
||||
Given a config dict with type llm provider and model (vac)
|
||||
When validate_actor_config is called (vac)
|
||||
Then no error should be raised for valid llm config (vac)
|
||||
|
||||
Scenario: validate_actor_config dispatches to tool validator for type tool
|
||||
Given a config dict with type tool and tools list (vac)
|
||||
When validate_actor_config is called (vac)
|
||||
Then no error should be raised for valid tool config (vac)
|
||||
|
||||
Scenario: validate_actor_config dispatches to multi_actor validator for type multi_actor
|
||||
Given a config dict with type multi_actor and actors mapping (vac)
|
||||
When validate_actor_config is called (vac)
|
||||
Then no error should be raised for valid multi_actor config (vac)
|
||||
|
||||
# ── _infer_actor_type implicit detection ──
|
||||
|
||||
Scenario: _infer_actor_type detects graph from top-level routes key
|
||||
Given a config dict without type but with top-level routes key (vac)
|
||||
When _infer_actor_type is called (vac)
|
||||
Then the inferred type should be graph (vac)
|
||||
|
||||
Scenario: _infer_actor_type detects multi_actor from top-level actors key
|
||||
Given a config dict without type but with top-level actors key (vac)
|
||||
When _infer_actor_type is called (vac)
|
||||
Then the inferred type should be multi_actor (vac)
|
||||
|
||||
Scenario: _infer_actor_type raises when routes has spec-level route types
|
||||
Given a config dict without type but with routes containing spec-level stream entry (vac)
|
||||
When _infer_actor_type is called (vac)
|
||||
Then a ConfigurationError should be raised mentioning missing agents key (vac)
|
||||
|
||||
Scenario: _infer_actor_type raises for non-string type field
|
||||
Given a config dict with integer type field (vac)
|
||||
When _infer_actor_type is called (vac)
|
||||
Then a ConfigurationError should be raised for non-string type (vac)
|
||||
|
||||
Scenario: _infer_actor_type raises for unknown actor type
|
||||
Given a config dict with unknown actor type (vac)
|
||||
When _infer_actor_type is called (vac)
|
||||
Then a ConfigurationError should be raised for unknown type (vac)
|
||||
|
||||
Scenario: _infer_actor_type raises when no type and no routes or actors
|
||||
Given a config dict without type routes or actors (vac)
|
||||
When _infer_actor_type is called (vac)
|
||||
Then a ConfigurationError should be raised for missing type (vac)
|
||||
|
||||
# ── _validate_graph_actor legacy format ──
|
||||
|
||||
Scenario: _validate_graph_actor validates legacy format successfully
|
||||
Given a config dict with type graph and legacy route with nodes edges entry_node (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then no error should be raised for valid legacy graph config (vac)
|
||||
|
||||
Scenario: _validate_graph_actor rejects legacy format without edges
|
||||
Given a config dict with type graph and legacy route without edges (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing edges (vac)
|
||||
|
||||
Scenario: _validate_graph_actor rejects legacy format without entry_node
|
||||
Given a config dict with type graph and legacy route without entry_node (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing entry_node (vac)
|
||||
|
||||
Scenario: _validate_graph_actor rejects legacy format exceeding max nodes
|
||||
Given a config dict with type graph and legacy route with 100 nodes (vac)
|
||||
And platform limits with max_total_nodes 10 (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for exceeding max nodes (vac)
|
||||
|
||||
# ── _validate_graph_actor v2.0 format ──
|
||||
|
||||
Scenario: _validate_graph_actor validates v2.0 format successfully
|
||||
Given a config dict with type graph and v2 routes with nodes edges entry_point (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then no error should be raised for valid v2 graph config (vac)
|
||||
|
||||
Scenario: _validate_graph_actor rejects v2 format without edges
|
||||
Given a config dict with type graph and v2 routes without edges (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing edges in routes main (vac)
|
||||
|
||||
Scenario: _validate_graph_actor rejects v2 format without entry_point
|
||||
Given a config dict with type graph and v2 routes without entry_point (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing entry_point in routes main (vac)
|
||||
|
||||
Scenario: _validate_graph_actor rejects v2 format exceeding max nodes
|
||||
Given a config dict with type graph and v2 routes with 200 nodes (vac)
|
||||
And platform limits with max_total_nodes 10 (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for exceeding max nodes (vac)
|
||||
|
||||
Scenario: _validate_graph_actor rejects v2 format with non-dict routes main
|
||||
Given a config dict with type graph and v2 routes with non-dict main (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for non-mapping routes main (vac)
|
||||
|
||||
# ── _validate_graph_actor neither format ──
|
||||
|
||||
Scenario: _validate_graph_actor rejects config without route or routes
|
||||
Given a config dict with type graph but no route or routes key (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing route or routes (vac)
|
||||
|
||||
# ── _validate_llm_actor ──
|
||||
|
||||
Scenario: _validate_llm_actor rejects config without name
|
||||
Given a config dict with type llm but no name field (vac)
|
||||
When _validate_llm_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing name in llm (vac)
|
||||
|
||||
Scenario: _validate_llm_actor rejects config without provider
|
||||
Given a config dict with type llm name but no provider (vac)
|
||||
When _validate_llm_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing provider (vac)
|
||||
|
||||
Scenario: _validate_llm_actor rejects config without model
|
||||
Given a config dict with type llm name provider but no model (vac)
|
||||
When _validate_llm_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing model (vac)
|
||||
|
||||
Scenario: _validate_llm_actor accepts config with provider and model in config block
|
||||
Given a config dict with type llm name and provider model in config block (vac)
|
||||
When _validate_llm_actor is called (vac)
|
||||
Then no error should be raised for valid llm config (vac)
|
||||
|
||||
# ── _validate_tool_actor ──
|
||||
|
||||
Scenario: _validate_tool_actor rejects config without name
|
||||
Given a config dict with type tool but no name field (vac)
|
||||
When _validate_tool_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing name in tool (vac)
|
||||
|
||||
Scenario: _validate_tool_actor rejects config without tools
|
||||
Given a config dict with type tool name but no tools (vac)
|
||||
When _validate_tool_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing tools (vac)
|
||||
|
||||
Scenario: _validate_tool_actor accepts config with tools in config block
|
||||
Given a config dict with type tool name and tools in config block (vac)
|
||||
When _validate_tool_actor is called (vac)
|
||||
Then no error should be raised for valid tool config (vac)
|
||||
|
||||
# ── _validate_multi_actor ──
|
||||
|
||||
Scenario: _validate_multi_actor rejects config with empty actors
|
||||
Given a config dict with type multi_actor and empty actors (vac)
|
||||
When _validate_multi_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for empty actors mapping (vac)
|
||||
|
||||
Scenario: _validate_multi_actor rejects config with non-dict actors
|
||||
Given a config dict with type multi_actor and non-dict actors (vac)
|
||||
When _validate_multi_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for empty actors mapping (vac)
|
||||
|
||||
Scenario: _validate_multi_actor rejects config exceeding total nodes limit
|
||||
Given a config dict with type multi_actor containing many graph actors (vac)
|
||||
And platform limits with max_total_nodes 3 (vac)
|
||||
When _validate_multi_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for exceeding total nodes (vac)
|
||||
|
||||
Scenario: _validate_multi_actor accepts config within node limits
|
||||
Given a config dict with type multi_actor containing few graph actors (vac)
|
||||
And platform limits with max_total_nodes 100 (vac)
|
||||
When _validate_multi_actor is called (vac)
|
||||
Then no error should be raised for valid multi_actor config (vac)
|
||||
|
||||
# ── Additional edge cases ──
|
||||
|
||||
Scenario: _validate_graph_actor handles non-int max_total_nodes fallback
|
||||
Given a config dict with type graph and legacy route with nodes edges entry_node (vac)
|
||||
And platform limits with string max_total_nodes value (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then no error should be raised for valid legacy graph config (vac)
|
||||
|
||||
Scenario: _validate_graph_actor v2 format with non-dict routes main raises error
|
||||
Given a config dict with type graph and v2 routes having non-dict main value (vac)
|
||||
When _validate_graph_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for non-mapping routes main (vac)
|
||||
|
||||
Scenario: _validate_llm_actor non-dict config block falls back to empty dict
|
||||
Given a config dict with type llm name without provider and string config block (vac)
|
||||
When _validate_llm_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing provider (vac)
|
||||
|
||||
Scenario: _validate_tool_actor non-dict config block falls back to empty dict
|
||||
Given a config dict with type tool name and string config block (vac)
|
||||
When _validate_tool_actor is called (vac)
|
||||
Then a ConfigurationError should be raised for missing tools (vac)
|
||||
@@ -520,9 +520,7 @@ class Node: # pylint: disable=too-many-instance-attributes
|
||||
self.logger.warning(
|
||||
f"Message router node {self.name} has no rules configured"
|
||||
)
|
||||
# Default to workflow_controller for CleverAgents v2.0 graphs
|
||||
# that rely on edge conditions rather than router rules
|
||||
return {"metadata": {"next_node": "workflow_controller"}}
|
||||
return {}
|
||||
|
||||
# Convert rules data to RouteRule objects
|
||||
rules = []
|
||||
|
||||
+54
-13
@@ -15,7 +15,11 @@ import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from cleveractors.core.exceptions import ConfigurationError
|
||||
from cleveractors.core.exceptions import (
|
||||
AgentCreationError,
|
||||
ConfigurationError,
|
||||
ExecutionError,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -67,6 +71,14 @@ class Executor:
|
||||
limits: dict[str, Any],
|
||||
pricing: dict[str, Any],
|
||||
):
|
||||
if not isinstance(config_dict, dict):
|
||||
raise ConfigurationError("config_dict must be a dict")
|
||||
if credentials is not None and not isinstance(credentials, dict):
|
||||
raise ConfigurationError("credentials must be a dict")
|
||||
if not isinstance(limits, dict):
|
||||
raise ConfigurationError("limits must be a dict")
|
||||
if not isinstance(pricing, dict):
|
||||
raise ConfigurationError("pricing must be a dict")
|
||||
self.config = config_dict
|
||||
self.credentials = credentials
|
||||
self.limits = limits
|
||||
@@ -158,11 +170,17 @@ class Executor:
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
creds = (
|
||||
self.credentials.get(provider)
|
||||
or self.credentials.get("openai_compatible")
|
||||
or {}
|
||||
)
|
||||
creds = {}
|
||||
if self.credentials is not None:
|
||||
creds = (
|
||||
self.credentials.get(provider)
|
||||
or self.credentials.get("openai_compatible")
|
||||
or {}
|
||||
)
|
||||
if not creds:
|
||||
raise ConfigurationError(
|
||||
f"missing credentials for provider: {provider}"
|
||||
)
|
||||
if creds.get("api_key"):
|
||||
agent_config["api_key"] = creds["api_key"]
|
||||
if creds.get("base_url"):
|
||||
@@ -185,11 +203,17 @@ class Executor:
|
||||
completion_tokens = 0
|
||||
|
||||
try:
|
||||
# Use the agent's process_message
|
||||
response = await agent.process_message(message, context)
|
||||
except (ConfigurationError, ExecutionError, AgentCreationError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception("LLM agent execution failed")
|
||||
raise ConfigurationError(f"LLM execution failed: {exc}") from exc
|
||||
raise ExecutionError(f"LLM execution failed: {exc}") from None
|
||||
finally:
|
||||
try:
|
||||
await agent.cleanup()
|
||||
except Exception as exc:
|
||||
logger.debug("Agent cleanup failed: %s", exc)
|
||||
|
||||
# Estimate tokens if the agent didn't track them
|
||||
prompt_tokens, completion_tokens = _estimate_tokens(
|
||||
@@ -317,7 +341,9 @@ class Executor:
|
||||
# Build agents with credential injection
|
||||
renderer = self._template_renderer()
|
||||
factory = AgentFactory(
|
||||
config=self._build_factory_config(), template_renderer=renderer
|
||||
config=self._build_factory_config(),
|
||||
template_renderer=renderer,
|
||||
credentials=self.credentials,
|
||||
)
|
||||
|
||||
# Pre-create agents referenced by nodes
|
||||
@@ -330,6 +356,8 @@ class Executor:
|
||||
if agent_name and agent_name not in agents:
|
||||
try:
|
||||
agents[agent_name] = factory.create_agent(agent_name)
|
||||
except (ConfigurationError, ExecutionError, AgentCreationError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to create agent %s: %s", agent_name, exc)
|
||||
|
||||
@@ -356,9 +384,18 @@ class Executor:
|
||||
conversation_history=conversation_history,
|
||||
initial_state=state,
|
||||
)
|
||||
except (ConfigurationError, ExecutionError, AgentCreationError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception("Graph execution failed")
|
||||
raise ConfigurationError(f"Graph execution failed: {exc}") from exc
|
||||
raise ExecutionError(f"Graph execution failed: {exc}") from exc
|
||||
finally:
|
||||
for name, agent in agents.items():
|
||||
try:
|
||||
if hasattr(agent, "cleanup"):
|
||||
await agent.cleanup()
|
||||
except Exception as exc:
|
||||
logger.debug("Agent %s cleanup failed: %s", name, exc)
|
||||
|
||||
# For graph execution, we don't have per-node token tracking yet
|
||||
# so we estimate based on the final response
|
||||
@@ -406,9 +443,11 @@ class Executor:
|
||||
|
||||
try:
|
||||
response = await agent.process_message(message, context)
|
||||
except (ConfigurationError, ExecutionError, AgentCreationError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception("Tool agent execution failed")
|
||||
raise ConfigurationError(f"Tool execution failed: {exc}") from exc
|
||||
raise ExecutionError(f"Tool execution failed: {exc}") from exc
|
||||
|
||||
# Tools don't consume LLM tokens
|
||||
node_usage = NodeUsage(
|
||||
@@ -552,8 +591,10 @@ def _estimate_tokens(
|
||||
prompt_tokens = len(enc.encode(prompt))
|
||||
completion_tokens = len(enc.encode(response))
|
||||
return prompt_tokens, completion_tokens
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Token estimation via tiktoken failed for %s: %s", model, type(exc).__name__
|
||||
)
|
||||
|
||||
# Fallback: ~4 chars per token for English text
|
||||
prompt_tokens = max(1, len(prompt) // 4)
|
||||
|
||||
Reference in New Issue
Block a user