aditya
4ae7f2d6fe
- Add missing node descriptions in Behave test scenarios - Fix context_view and parallel execution step definitions - Update llm_with_tools.yaml max_context_tokens (10000 → 16000) - Add context.error compatibility for error assertions Results: Behave 25/25, Robot 16/16, ASV 7/7 all newly added tests passing.
433 lines
18 KiB
Python
433 lines
18 KiB
Python
"""Step definitions for actor YAML examples validation.
|
|
|
|
Tests for features/actor_examples.feature — validates that all actor examples
|
|
are correct, loadable, and demonstrate the expected patterns.
|
|
|
|
Note: Most validation steps are reused from actor_schema_steps.py.
|
|
Only examples-specific steps are defined here.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from behave import given, then # type: ignore[import-untyped]
|
|
from behave.runner import Context # type: ignore[import-untyped]
|
|
|
|
# ────────────────────────────────────────────────────────────
|
|
# Examples-Specific Given Steps
|
|
# ────────────────────────────────────────────────────────────
|
|
|
|
|
|
@given("the CleverAgents actor schema validation is available")
|
|
def step_given_schema_available(context: Context) -> None:
|
|
"""Verify that ActorConfigSchema is importable and ready."""
|
|
from cleveragents.actor.schema import ActorConfigSchema
|
|
|
|
context.schema_available = ActorConfigSchema is not None
|
|
assert context.schema_available
|
|
|
|
|
|
@given('the actor examples YAML file "{file_path}"')
|
|
def step_given_actor_examples_yaml_file(context: Context, file_path: str) -> None:
|
|
"""Load actor YAML file path for validation (examples-specific)."""
|
|
context.actor_yaml_file = file_path
|
|
context.actor_yaml_string = None
|
|
|
|
|
|
@given("the examples/actors directory")
|
|
def step_given_examples_directory(context: Context) -> None:
|
|
"""Set examples directory path."""
|
|
project_root = Path(__file__).resolve().parents[2]
|
|
context.examples_dir = project_root / "examples" / "actors"
|
|
assert context.examples_dir.exists(), (
|
|
f"Examples directory not found: {context.examples_dir}"
|
|
)
|
|
|
|
|
|
@given('the documentation file "{doc_path}"')
|
|
def step_given_documentation_file(context: Context, doc_path: str) -> None:
|
|
"""Load documentation file for verification."""
|
|
project_root = Path(__file__).resolve().parents[2]
|
|
context.doc_file = project_root / doc_path
|
|
|
|
|
|
# Generic step for all "Given an actor YAML string ..." patterns
|
|
@given("an actor YAML string representing a strategist actor:")
|
|
@given("an actor YAML string representing an executor actor:")
|
|
@given("an actor YAML string representing a reviewer actor:")
|
|
@given("an actor YAML string with tool-only configuration:")
|
|
@given("an actor YAML string with git tools:")
|
|
@given("an actor YAML string representing a validator actor:")
|
|
@given("an actor YAML string with linear graph workflow:")
|
|
@given("an actor YAML string with conditional routing examples:")
|
|
@given("an actor YAML string with tool nodes examples:")
|
|
@given("an actor YAML string with subgraph examples:")
|
|
@given("an actor YAML string with retry logic:")
|
|
@given("an actor YAML string with multi-level hierarchy:")
|
|
@given("an actor YAML string missing required name field for examples:")
|
|
@given("an actor YAML string with invalid name for examples:")
|
|
@given("an actor YAML string for LLM without model for examples:")
|
|
@given("an actor YAML string for TOOL without tools for examples:")
|
|
@given("an actor YAML string for GRAPH without route for examples:")
|
|
@given("an actor YAML string with cyclic graph for examples:")
|
|
def step_given_actor_yaml_string_pattern(context: Context) -> None:
|
|
"""Load actor YAML string from docstring (generic for all patterns)."""
|
|
context.actor_yaml_string = context.text
|
|
context.actor_yaml_file = None
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────
|
|
# Additional Then Steps for Actor Examples (not in actor_schema_steps.py)
|
|
# ────────────────────────────────────────────────────────────
|
|
|
|
|
|
@then('the actor config description should contain "{expected_text}"')
|
|
def step_then_description_contains(context: Context, expected_text: str) -> None:
|
|
"""Verify actor description contains text."""
|
|
assert context.actor_config is not None
|
|
assert expected_text.lower() in context.actor_config.description.lower()
|
|
|
|
|
|
@then('the actor config context_view should be "{expected_view}"')
|
|
def step_then_config_context_view_is(context: Context, expected_view: str) -> None:
|
|
"""Verify context_view matches (with 'config' in step name)."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.context_view == expected_view
|
|
|
|
|
|
@then("the actor config should have read-only tools")
|
|
def step_then_has_readonly_tools(context: Context) -> None:
|
|
"""Verify actor has read-only tools."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.tools is not None
|
|
# Check for typical read-only tool patterns
|
|
readonly_patterns = ["read", "list", "get", "show", "view"]
|
|
tools_str = " ".join(str(t) for t in context.actor_config.tools).lower()
|
|
assert any(p in tools_str for p in readonly_patterns)
|
|
|
|
|
|
@then("the actor config should have write-capable tools")
|
|
def step_then_has_write_tools(context: Context) -> None:
|
|
"""Verify actor has write-capable tools."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.tools is not None
|
|
write_patterns = ["write", "create", "update", "delete", "modify"]
|
|
tools_str = " ".join(str(t) for t in context.actor_config.tools).lower()
|
|
assert any(p in tools_str for p in write_patterns)
|
|
|
|
|
|
@then("the actor config should have linting tools")
|
|
def step_then_has_linting_tools(context: Context) -> None:
|
|
"""Verify actor has linting tools."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.tools is not None
|
|
lint_patterns = ["lint", "format", "check", "ruff"]
|
|
tools_str = " ".join(str(t) for t in context.actor_config.tools).lower()
|
|
assert any(p in tools_str for p in lint_patterns)
|
|
|
|
|
|
@then("the actor config should have validation tools")
|
|
def step_then_has_validation_tools(context: Context) -> None:
|
|
"""Verify actor has validation tools."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.tools is not None
|
|
validation_patterns = ["validate", "verify", "test", "check"]
|
|
tools_str = " ".join(str(t) for t in context.actor_config.tools).lower()
|
|
assert any(p in tools_str for p in validation_patterns)
|
|
|
|
|
|
@then("the actor config should not have model field")
|
|
def step_then_no_model(context: Context) -> None:
|
|
"""Verify actor doesn't have model field."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.model is None or context.actor_config.model == ""
|
|
|
|
|
|
@then("the actor config should not have system_prompt field")
|
|
def step_then_no_system_prompt(context: Context) -> None:
|
|
"""Verify actor doesn't have system_prompt field."""
|
|
assert context.actor_config is not None
|
|
assert (
|
|
context.actor_config.system_prompt is None
|
|
or context.actor_config.system_prompt == ""
|
|
)
|
|
|
|
|
|
@then('the actor route entry_node should be "{expected_node}"')
|
|
def step_then_entry_node_is(context: Context, expected_node: str) -> None:
|
|
"""Verify entry node ID."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert context.actor_config.route.entry_node == expected_node
|
|
|
|
|
|
@then("the actor route should have {count:d} exit node")
|
|
@then("the actor route should have {count:d} exit nodes")
|
|
def step_then_route_exit_node_count(context: Context, count: int) -> None:
|
|
"""Verify number of exit nodes."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert len(context.actor_config.route.exit_nodes) == count
|
|
|
|
|
|
@then("the actor route should have conditional node")
|
|
def step_then_has_conditional_node(context: Context) -> None:
|
|
"""Verify route has conditional node."""
|
|
from cleveragents.actor.schema import NodeType
|
|
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert any(n.type == NodeType.CONDITIONAL for n in context.actor_config.route.nodes)
|
|
|
|
|
|
@then("the actor route should have tool node")
|
|
def step_then_has_tool_node(context: Context) -> None:
|
|
"""Verify route has tool node."""
|
|
from cleveragents.actor.schema import NodeType
|
|
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert any(n.type == NodeType.TOOL for n in context.actor_config.route.nodes)
|
|
|
|
|
|
@then("the actor route should have agent nodes")
|
|
def step_then_has_agent_nodes(context: Context) -> None:
|
|
"""Verify route has agent nodes."""
|
|
from cleveragents.actor.schema import NodeType
|
|
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert any(n.type == NodeType.AGENT for n in context.actor_config.route.nodes)
|
|
|
|
|
|
@then("the actor route should have subgraph nodes")
|
|
def step_then_has_subgraph_nodes(context: Context) -> None:
|
|
"""Verify route has subgraph nodes."""
|
|
from cleveragents.actor.schema import NodeType
|
|
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert any(n.type == NodeType.SUBGRAPH for n in context.actor_config.route.nodes)
|
|
|
|
|
|
@then("the actor route should have multiple exit nodes")
|
|
def step_then_has_multiple_exit_nodes(context: Context) -> None:
|
|
"""Verify route has multiple exit nodes."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert len(context.actor_config.route.exit_nodes) > 1
|
|
|
|
|
|
@then("the actor route should have retry loop")
|
|
def step_then_has_retry_loop(context: Context) -> None:
|
|
"""Verify route has retry logic (nodes/edges for retry)."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
# Check for nodes with "retry" in name or description
|
|
node_texts = [
|
|
n.name.lower() + (n.description or "").lower()
|
|
for n in context.actor_config.route.nodes
|
|
]
|
|
assert any("retry" in text for text in node_texts)
|
|
|
|
|
|
@then("the actor route should have escalation path")
|
|
def step_then_has_escalation_path(context: Context) -> None:
|
|
"""Verify route has escalation path."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
node_texts = [
|
|
n.name.lower() + (n.description or "").lower()
|
|
for n in context.actor_config.route.nodes
|
|
]
|
|
assert any("escalat" in text for text in node_texts)
|
|
|
|
|
|
@then("the actor route should have multiple subgraph levels")
|
|
def step_then_has_multiple_subgraph_levels(context: Context) -> None:
|
|
"""Verify route has multiple levels of subgraphs."""
|
|
from cleveragents.actor.schema import NodeType
|
|
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
subgraph_count = sum(
|
|
1 for n in context.actor_config.route.nodes if n.type == NodeType.SUBGRAPH
|
|
)
|
|
assert subgraph_count >= 2
|
|
|
|
|
|
@then("the actor route should have parallel execution config")
|
|
@then("the actor route should support parallel execution")
|
|
def step_then_has_parallel_execution(context: Context) -> None:
|
|
"""Verify route supports parallel execution."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
|
|
# Check for two things:
|
|
# 1. Nodes with parallel config
|
|
# 2. Nodes with multiple outgoing edges (parallel branching)
|
|
from collections import defaultdict
|
|
|
|
# Check for parallel config in subgraph nodes
|
|
has_parallel_config = False
|
|
for node in context.actor_config.route.nodes:
|
|
if node.type == "subgraph" and hasattr(node, "config") and node.config:
|
|
# Config might be a dict or an object
|
|
if isinstance(node.config, dict):
|
|
has_parallel_config = node.config.get("parallel", False)
|
|
else:
|
|
has_parallel_config = getattr(node.config, "parallel", False)
|
|
if has_parallel_config:
|
|
break
|
|
|
|
# Check for nodes with multiple outgoing edges
|
|
edge_counts = defaultdict(int)
|
|
for edge in context.actor_config.route.edges:
|
|
edge_counts[edge.from_node] += 1
|
|
has_multiple_edges = any(count > 1 for count in edge_counts.values())
|
|
|
|
# Either condition satisfies parallel execution
|
|
assert has_parallel_config or has_multiple_edges, (
|
|
f"Route should have parallel execution via config or multiple outgoing edges "
|
|
f"(found parallel_config={has_parallel_config}, multiple_edges={has_multiple_edges})"
|
|
)
|
|
|
|
|
|
@then("the actor memory max_messages should be at least {minimum:d}")
|
|
def step_then_memory_max_messages_at_least(context: Context, minimum: int) -> None:
|
|
"""Verify memory.max_messages is at least minimum."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.memory is not None
|
|
assert context.actor_config.memory.max_messages >= minimum
|
|
|
|
|
|
@then("the actor context max_context_tokens should be {expected:d}")
|
|
def step_then_context_max_tokens(context: Context, expected: int) -> None:
|
|
"""Verify context.max_context_tokens value."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.context is not None
|
|
assert context.actor_config.context.max_context_tokens == expected
|
|
|
|
|
|
@then('the actor config name should match pattern "{pattern}"')
|
|
def step_then_name_matches_pattern(context: Context, pattern: str) -> None:
|
|
"""Verify actor name matches regex pattern."""
|
|
import re
|
|
|
|
assert context.actor_config is not None
|
|
assert re.match(pattern, context.actor_config.name)
|
|
|
|
|
|
@then("the actor route should have at least {minimum:d} nodes")
|
|
def step_then_route_has_min_nodes(context: Context, minimum: int) -> None:
|
|
"""Verify route has minimum number of nodes."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert len(context.actor_config.route.nodes) >= minimum
|
|
|
|
|
|
@then("the actor route should have at least {minimum:d} edges")
|
|
def step_then_route_has_min_edges(context: Context, minimum: int) -> None:
|
|
"""Verify route has minimum number of edges."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert len(context.actor_config.route.edges) >= minimum
|
|
|
|
|
|
@then("the actor route should have an entry_node")
|
|
def step_then_route_has_entry_node(context: Context) -> None:
|
|
"""Verify route has entry_node."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert context.actor_config.route.entry_node is not None
|
|
|
|
|
|
@then("the actor route should have exit_nodes")
|
|
def step_then_route_has_exit_nodes(context: Context) -> None:
|
|
"""Verify route has exit_nodes."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.route is not None
|
|
assert context.actor_config.route.exit_nodes is not None
|
|
assert len(context.actor_config.route.exit_nodes) > 0
|
|
|
|
|
|
@then("the actor config should have environment variables")
|
|
def step_then_has_env_vars(context: Context) -> None:
|
|
"""Verify actor has environment variables."""
|
|
assert context.actor_config is not None
|
|
assert context.actor_config.env_vars is not None
|
|
assert len(context.actor_config.env_vars) > 0
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────
|
|
# Examples-Specific Then Steps
|
|
# ────────────────────────────────────────────────────────────
|
|
|
|
|
|
@then("there should be exactly {count:d} example YAML files")
|
|
def step_then_count_example_files(context: Context, count: int) -> None:
|
|
"""Verify the number of example YAML files."""
|
|
yaml_files = list(context.examples_dir.glob("*.yaml"))
|
|
assert len(yaml_files) == count, (
|
|
f"Expected {count} YAML files in examples/actors/, "
|
|
f"found {len(yaml_files)}: {[f.name for f in yaml_files]}"
|
|
)
|
|
|
|
|
|
@then('the example files should include "{filename}"')
|
|
def step_then_example_file_exists(context: Context, filename: str) -> None:
|
|
"""Verify specific example file exists."""
|
|
file_path = context.examples_dir / filename
|
|
assert file_path.exists(), f"Expected example file not found: {file_path}"
|
|
|
|
|
|
@then('the documentation should reference "{filename}"')
|
|
def step_then_doc_references_file(context: Context, filename: str) -> None:
|
|
"""Verify documentation references the example file."""
|
|
assert hasattr(context, "doc_file"), "Documentation file not loaded"
|
|
content = context.doc_file.read_text()
|
|
assert filename in content, f"Documentation should reference {filename}"
|
|
|
|
|
|
@then("the documentation should have examples for strategist actors")
|
|
def step_then_doc_has_strategist_examples(context: Context) -> None:
|
|
"""Verify documentation has strategist examples."""
|
|
content = context.doc_file.read_text()
|
|
assert "strategist" in content.lower() or "strategy" in content.lower()
|
|
|
|
|
|
@then("the documentation should have examples for executor actors")
|
|
def step_then_doc_has_executor_examples(context: Context) -> None:
|
|
"""Verify documentation has executor examples."""
|
|
content = context.doc_file.read_text()
|
|
assert "executor" in content.lower() or "execution" in content.lower()
|
|
|
|
|
|
@then("the documentation should have examples for reviewer actors")
|
|
def step_then_doc_has_reviewer_examples(context: Context) -> None:
|
|
"""Verify documentation has reviewer examples."""
|
|
content = context.doc_file.read_text()
|
|
assert "reviewer" in content.lower() or "review" in content.lower()
|
|
|
|
|
|
@then("the documentation should have examples for tool-only actors")
|
|
def step_then_doc_has_tool_only_examples(context: Context) -> None:
|
|
"""Verify documentation has tool-only examples."""
|
|
content = context.doc_file.read_text()
|
|
assert "tool-only" in content.lower() or "tool collection" in content.lower()
|
|
|
|
|
|
@then("the documentation should have examples for validation actors")
|
|
def step_then_doc_has_validation_examples(context: Context) -> None:
|
|
"""Verify documentation has validation examples."""
|
|
content = context.doc_file.read_text()
|
|
assert "validation" in content.lower() or "validator" in content.lower()
|
|
|
|
|
|
@then("the documentation should have examples for hierarchical graphs")
|
|
def step_then_doc_has_hierarchical_examples(context: Context) -> None:
|
|
"""Verify documentation has hierarchical graph examples."""
|
|
content = context.doc_file.read_text()
|
|
assert "hierarchical" in content.lower() or "subgraph" in content.lower()
|