forked from HAL9000/cleveragents-core
642 lines
22 KiB
Python
642 lines
22 KiB
Python
"""
|
|
Unit tests for agents/factory.py
|
|
|
|
Tests the AgentFactory class.
|
|
"""
|
|
|
|
import pytest
|
|
from unittest.mock import Mock, patch
|
|
|
|
from cleveragents.agents.factory import AgentFactory
|
|
from cleveragents.agents.base import Agent
|
|
from cleveragents.agents.llm import LLMAgent
|
|
from cleveragents.agents.tool import ToolAgent
|
|
from cleveragents.core.exceptions import AgentCreationError, ConfigurationError
|
|
from cleveragents.templates.renderer import TemplateRenderer
|
|
|
|
|
|
class TestAgentFactory:
|
|
"""Test suite for the AgentFactory class."""
|
|
|
|
@pytest.fixture
|
|
def template_renderer(self):
|
|
"""Create a basic template renderer."""
|
|
return TemplateRenderer()
|
|
|
|
def _create_factory_with_llm_and_tool_agents(self, template_renderer):
|
|
"""Helper method to create factory with LLM and tool agents config."""
|
|
config = {
|
|
"agents": {
|
|
"agent1": {"type": "llm", "config": {"model": "gpt-4"}},
|
|
"agent2": {"type": "tool", "config": {"tools": ["echo"]}},
|
|
}
|
|
}
|
|
return AgentFactory(config, template_renderer)
|
|
|
|
@pytest.fixture
|
|
def simple_config(self):
|
|
"""Create a simple configuration with agents."""
|
|
return {
|
|
"agents": {
|
|
"researcher": {
|
|
"type": "llm",
|
|
"provider": "openai",
|
|
"model": "gpt-4",
|
|
},
|
|
"analyzer": {
|
|
"type": "tool",
|
|
"tools": ["file_read", "file_write"],
|
|
},
|
|
}
|
|
}
|
|
|
|
def test_factory_initialization(self, simple_config, template_renderer):
|
|
"""Test AgentFactory initialization."""
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
assert factory.config == simple_config
|
|
assert factory.template_renderer == template_renderer
|
|
assert "llm" in factory.agent_types
|
|
assert "tool" in factory.agent_types
|
|
assert factory.agents == {}
|
|
|
|
def test_factory_get_agent_types(self, simple_config, template_renderer):
|
|
"""Test getting registered agent types."""
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
agent_types = factory.get_agent_types()
|
|
|
|
assert "llm" in agent_types
|
|
assert "tool" in agent_types
|
|
assert agent_types["llm"] == LLMAgent
|
|
assert agent_types["tool"] == ToolAgent
|
|
|
|
# Verify it's a copy
|
|
agent_types["new_type"] = Agent
|
|
assert "new_type" not in factory.agent_types
|
|
|
|
def test_factory_register_agent_type(self, simple_config, template_renderer):
|
|
"""Test registering a new agent type."""
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
class CustomAgent(Agent):
|
|
def get_capabilities(self):
|
|
return ["custom"]
|
|
|
|
async def process_message(self, message: str, context=None) -> str:
|
|
return "processed"
|
|
|
|
factory.register_agent_type("custom", CustomAgent)
|
|
|
|
assert "custom" in factory.agent_types
|
|
assert factory.agent_types["custom"] == CustomAgent
|
|
|
|
def test_factory_register_invalid_agent_type(self, simple_config, template_renderer):
|
|
"""Test that registering non-Agent class raises error."""
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
class NotAnAgent:
|
|
pass
|
|
|
|
with pytest.raises(ConfigurationError) as exc_info:
|
|
factory.register_agent_type("invalid", NotAnAgent)
|
|
|
|
assert "must inherit from Agent" in str(exc_info.value)
|
|
|
|
@patch('cleveragents.agents.factory.LLMAgent')
|
|
def test_factory_create_agent(self, mock_llm_agent, simple_config, template_renderer):
|
|
"""Test creating an agent."""
|
|
mock_agent = Mock()
|
|
mock_llm_agent.return_value = mock_agent
|
|
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
agent = factory.create_agent("researcher")
|
|
|
|
assert agent == mock_agent
|
|
mock_llm_agent.assert_called_once()
|
|
# Verify agent is cached
|
|
assert "researcher" in factory.agents
|
|
assert factory.agents["researcher"] == mock_agent
|
|
|
|
@patch('cleveragents.agents.factory.LLMAgent')
|
|
def test_factory_create_agent_cached(self, mock_llm_agent, simple_config, template_renderer):
|
|
"""Test that agents are cached and reused."""
|
|
mock_agent = Mock()
|
|
mock_llm_agent.return_value = mock_agent
|
|
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
# Create agent first time
|
|
agent1 = factory.create_agent("researcher")
|
|
|
|
# Create same agent second time
|
|
agent2 = factory.create_agent("researcher")
|
|
|
|
# Should be the same instance
|
|
assert agent1 == agent2
|
|
# Should only be called once due to caching
|
|
mock_llm_agent.assert_called_once()
|
|
|
|
def test_factory_create_nonexistent_agent(self, simple_config, template_renderer):
|
|
"""Test creating an agent that doesn't exist in config."""
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
with pytest.raises(AgentCreationError) as exc_info:
|
|
factory.create_agent("nonexistent")
|
|
|
|
assert "nonexistent" in str(exc_info.value)
|
|
assert "configuration" in str(exc_info.value).lower()
|
|
|
|
def test_factory_create_agent_missing_type_defaults_to_llm(self, template_renderer, monkeypatch):
|
|
"""Test creating an agent with missing type defaults to LLM."""
|
|
# Set fake API key to allow LLM agent creation
|
|
monkeypatch.setenv("OPENAI_API_KEY", "fake-key-for-testing")
|
|
|
|
config = {
|
|
"agents": {
|
|
"test_agent": {
|
|
"model": "gpt-4",
|
|
# Missing 'type' - should default to 'llm'
|
|
}
|
|
}
|
|
}
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
# Mock the ChatOpenAI class to avoid actual API calls
|
|
with patch('cleveragents.agents.llm.ChatOpenAI') as mock_chat:
|
|
mock_chat.return_value = Mock()
|
|
|
|
agent = factory.create_agent("test_agent")
|
|
|
|
# Should create an LLM agent by default
|
|
assert isinstance(agent, LLMAgent)
|
|
assert agent.name == "test_agent"
|
|
|
|
def test_factory_create_agent_unknown_type(self, template_renderer):
|
|
"""Test creating an agent with unknown type."""
|
|
config = {
|
|
"agents": {
|
|
"broken": {
|
|
"type": "unknown_type",
|
|
}
|
|
}
|
|
}
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(AgentCreationError) as exc_info:
|
|
factory.create_agent("broken")
|
|
|
|
assert "unknown_type" in str(exc_info.value)
|
|
|
|
@patch('cleveragents.agents.factory.LLMAgent')
|
|
def test_factory_agents_cache(self, mock_llm_agent, simple_config, template_renderer):
|
|
"""Test that agents are properly cached after creation."""
|
|
mock_agent = Mock()
|
|
mock_llm_agent.return_value = mock_agent
|
|
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
# Initially cache should be empty
|
|
assert len(factory.agents) == 0
|
|
|
|
# Create an agent
|
|
factory.create_agent("researcher")
|
|
|
|
# Cache should now contain the agent
|
|
assert len(factory.agents) == 1
|
|
assert "researcher" in factory.agents
|
|
assert factory.agents["researcher"] == mock_agent
|
|
|
|
def test_factory_with_stream_router(self, simple_config, template_renderer):
|
|
"""Test factory with stream router."""
|
|
mock_router = Mock()
|
|
factory = AgentFactory(simple_config, template_renderer, stream_router=mock_router)
|
|
|
|
assert factory.stream_router == mock_router
|
|
|
|
def test_factory_with_langgraph_bridge(self, simple_config, template_renderer):
|
|
"""Test factory with langgraph bridge."""
|
|
mock_bridge = Mock()
|
|
factory = AgentFactory(simple_config, template_renderer, langgraph_bridge=mock_bridge)
|
|
|
|
assert factory.langgraph_bridge == mock_bridge
|
|
|
|
@patch('cleveragents.agents.factory.LLMAgent')
|
|
@patch('cleveragents.agents.factory.ToolAgent')
|
|
def test_factory_create_all_configured_agents(self, mock_tool, mock_llm, template_renderer):
|
|
"""Test creating all agents from configuration."""
|
|
config = {
|
|
"agents": {
|
|
"llm1": {"type": "llm", "model": "gpt-4"},
|
|
"llm2": {"type": "llm", "model": "claude-3"},
|
|
"tool1": {"type": "tool", "tools": ["echo"]},
|
|
}
|
|
}
|
|
|
|
mock_llm_instance = Mock()
|
|
mock_tool_instance = Mock()
|
|
mock_llm.return_value = mock_llm_instance
|
|
mock_tool.return_value = mock_tool_instance
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
# Create all agents
|
|
factory.create_agent("llm1")
|
|
factory.create_agent("llm2")
|
|
factory.create_agent("tool1")
|
|
|
|
# Verify all were created
|
|
assert len(factory.agents) == 3
|
|
assert "llm1" in factory.agents
|
|
assert "llm2" in factory.agents
|
|
assert "tool1" in factory.agents
|
|
|
|
def test_factory_no_agents_config(self, template_renderer):
|
|
"""Test factory with no agents section in config."""
|
|
config = {}
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(AgentCreationError):
|
|
factory.create_agent("any_agent")
|
|
|
|
@patch('cleveragents.agents.factory.LLMAgent')
|
|
def test_factory_agent_creation_error(self, mock_llm, simple_config, template_renderer):
|
|
"""Test that agent creation errors are properly wrapped."""
|
|
mock_llm.side_effect = Exception("Creation failed")
|
|
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
with pytest.raises(AgentCreationError) as exc_info:
|
|
factory.create_agent("researcher")
|
|
|
|
assert "researcher" in str(exc_info.value)
|
|
|
|
def test_factory_config_property(self, simple_config, template_renderer):
|
|
"""Test that factory stores config correctly."""
|
|
factory = AgentFactory(simple_config, template_renderer)
|
|
|
|
assert factory.config == simple_config
|
|
assert "agents" in factory.config
|
|
|
|
@patch('cleveragents.agents.factory.LLMAgent')
|
|
def test_factory_multiple_agent_types(self, mock_llm, template_renderer):
|
|
"""Test factory handles multiple agent types in config."""
|
|
config = {
|
|
"agents": {
|
|
"agent1": {"type": "llm"},
|
|
"agent2": {"type": "llm"},
|
|
"agent3": {"type": "llm"},
|
|
}
|
|
}
|
|
|
|
mock_llm.return_value = Mock()
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
# Create all three agents
|
|
for name in ["agent1", "agent2", "agent3"]:
|
|
factory.create_agent(name)
|
|
|
|
# All should be cached
|
|
assert len(factory.agents) == 3
|
|
|
|
|
|
def test_factory_create_composite_agent(self, template_renderer):
|
|
"""Test creating a composite agent through factory."""
|
|
config = {
|
|
"agents": {
|
|
"composite1": {
|
|
"type": "composite",
|
|
"config": {
|
|
"routing": {}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(
|
|
config,
|
|
template_renderer,
|
|
stream_router=Mock(),
|
|
langgraph_bridge=Mock()
|
|
)
|
|
|
|
agent = factory.create_agent("composite1")
|
|
|
|
from cleveragents.agents.composite import CompositeAgent
|
|
assert isinstance(agent, CompositeAgent)
|
|
assert agent.name == "composite1"
|
|
|
|
def test_factory_create_composite_with_nested_agents(self, template_renderer, monkeypatch):
|
|
"""Test creating a composite agent with nested agent components."""
|
|
monkeypatch.setenv("OPENAI_API_KEY", "fake-key")
|
|
|
|
config = {
|
|
"agents": {
|
|
"composite1": {
|
|
"type": "composite",
|
|
"config": {
|
|
"components": {
|
|
"agents": {
|
|
"nested_llm": {
|
|
"type": "llm",
|
|
"config": {"model": "gpt-4"}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(
|
|
config,
|
|
template_renderer,
|
|
stream_router=Mock(),
|
|
langgraph_bridge=Mock()
|
|
)
|
|
|
|
with patch('cleveragents.agents.llm.ChatOpenAI'):
|
|
agent = factory.create_agent("composite1")
|
|
|
|
from cleveragents.agents.composite import CompositeAgent
|
|
assert isinstance(agent, CompositeAgent)
|
|
assert "nested_llm" in agent.agents
|
|
|
|
def test_factory_create_composite_with_legacy_config(self, template_renderer, monkeypatch):
|
|
"""Test creating a composite agent with legacy configuration."""
|
|
monkeypatch.setenv("OPENAI_API_KEY", "fake-key")
|
|
|
|
config = {
|
|
"agents": {
|
|
"llm1": {
|
|
"type": "llm",
|
|
"config": {"model": "gpt-4"}
|
|
},
|
|
"composite1": {
|
|
"type": "composite",
|
|
"config": {
|
|
"agents": ["llm1"] # Legacy format
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(
|
|
config,
|
|
template_renderer,
|
|
stream_router=Mock(),
|
|
langgraph_bridge=Mock()
|
|
)
|
|
|
|
with patch('cleveragents.agents.llm.ChatOpenAI'):
|
|
# First create the referenced agent
|
|
factory.create_agent("llm1")
|
|
# Then create composite
|
|
composite = factory.create_agent("composite1")
|
|
|
|
from cleveragents.agents.composite import CompositeAgent
|
|
assert isinstance(composite, CompositeAgent)
|
|
assert "llm1" in composite.agents
|
|
|
|
def test_factory_create_agents_from_config(self, template_renderer, monkeypatch):
|
|
"""Test creating all agents from config."""
|
|
monkeypatch.setenv("OPENAI_API_KEY", "fake-key")
|
|
|
|
factory = self._create_factory_with_llm_and_tool_agents(template_renderer)
|
|
|
|
with patch('cleveragents.agents.llm.ChatOpenAI'):
|
|
agents = factory.create_agents_from_config()
|
|
|
|
assert "agent1" in agents
|
|
assert "agent2" in agents
|
|
assert len(agents) == 2
|
|
|
|
def test_factory_create_agents_from_config_stores_in_cache(self, template_renderer, monkeypatch):
|
|
"""Test that create_agents_from_config stores agents in cache."""
|
|
monkeypatch.setenv("OPENAI_API_KEY", "fake-key")
|
|
|
|
factory = self._create_factory_with_llm_and_tool_agents(template_renderer)
|
|
|
|
with patch('cleveragents.agents.llm.ChatOpenAI'):
|
|
agents = factory.create_agents_from_config()
|
|
|
|
# Check that all agents are now in the agents dict (cached)
|
|
assert "agent1" in factory.agents
|
|
assert "agent2" in factory.agents
|
|
assert factory.agents == agents
|
|
|
|
def test_factory_agent_creation_error_handling(self, template_renderer):
|
|
"""Test that agent creation errors are properly wrapped."""
|
|
config = {
|
|
"agents": {
|
|
"broken_agent": {
|
|
"type": "invalid_type", # Use an invalid agent type
|
|
"config": {"model": "test"}
|
|
}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
# Agent creation will fail with unknown agent type
|
|
with pytest.raises(AgentCreationError) as exc_info:
|
|
factory.create_agent("broken_agent")
|
|
|
|
assert "broken_agent" in str(exc_info.value) or "invalid_type" in str(exc_info.value)
|
|
|
|
def test_factory_register_multiple_agent_types(self, template_renderer):
|
|
"""Test registering multiple custom agent types."""
|
|
factory = AgentFactory({}, template_renderer)
|
|
|
|
class CustomAgent1(Agent):
|
|
def __init__(self, name, config, template_renderer):
|
|
super().__init__(name, config, template_renderer)
|
|
|
|
async def process_message(self, message, context=None):
|
|
return "custom1"
|
|
|
|
def get_capabilities(self):
|
|
return ["custom1"]
|
|
|
|
class CustomAgent2(Agent):
|
|
def __init__(self, name, config, template_renderer):
|
|
super().__init__(name, config, template_renderer)
|
|
|
|
async def process_message(self, message, context=None):
|
|
return "custom2"
|
|
|
|
def get_capabilities(self):
|
|
return ["custom2"]
|
|
|
|
factory.register_agent_type("custom1", CustomAgent1)
|
|
factory.register_agent_type("custom2", CustomAgent2)
|
|
|
|
assert "custom1" in factory.get_agent_types()
|
|
assert "custom2" in factory.get_agent_types()
|
|
|
|
def test_factory_with_router_and_bridge(self, template_renderer):
|
|
"""Test factory initialization with stream router and langgraph bridge."""
|
|
mock_router = Mock()
|
|
mock_bridge = Mock()
|
|
|
|
factory = AgentFactory(
|
|
{},
|
|
template_renderer,
|
|
stream_router=mock_router,
|
|
langgraph_bridge=mock_bridge
|
|
)
|
|
|
|
assert factory.stream_router is mock_router
|
|
assert factory.langgraph_bridge is mock_bridge
|
|
|
|
def test_factory_agents_property_returns_copy(self, template_renderer, monkeypatch):
|
|
"""Test that agents property returns current agents."""
|
|
monkeypatch.setenv("OPENAI_API_KEY", "fake-key")
|
|
|
|
config = {
|
|
"agents": {
|
|
"agent1": {"type": "llm", "config": {"model": "gpt-4"}}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with patch('cleveragents.agents.llm.ChatOpenAI'):
|
|
factory.create_agent("agent1")
|
|
|
|
agents = factory.agents
|
|
assert "agent1" in agents
|
|
|
|
def test_factory_validate_configuration_valid(self, template_renderer):
|
|
"""Test validating a valid configuration."""
|
|
config = {
|
|
"agents": {
|
|
"agent1": {"type": "llm", "config": {"model": "gpt-4"}},
|
|
"agent2": {"type": "tool", "config": {"tools": ["echo"]}}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
# Should not raise
|
|
factory.validate_configuration()
|
|
|
|
def test_factory_validate_configuration_invalid_agents_not_dict(self, template_renderer):
|
|
"""Test validation fails when agents is not a dictionary."""
|
|
config = {"agents": "not a dict"}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(ConfigurationError) as exc_info:
|
|
factory.validate_configuration()
|
|
|
|
assert "must be a dictionary" in str(exc_info.value)
|
|
|
|
def test_factory_validate_configuration_agent_config_not_dict(self, template_renderer):
|
|
"""Test validation fails when agent config is not a dictionary."""
|
|
config = {"agents": {"agent1": "not a dict"}}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(ConfigurationError) as exc_info:
|
|
factory.validate_configuration()
|
|
|
|
assert "agent1" in str(exc_info.value)
|
|
|
|
def test_factory_validate_configuration_missing_type(self, template_renderer):
|
|
"""Test validation fails when agent type is missing."""
|
|
config = {"agents": {"agent1": {"config": {"model": "gpt-4"}}}}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(ConfigurationError) as exc_info:
|
|
factory.validate_configuration()
|
|
|
|
assert "must specify a type" in str(exc_info.value)
|
|
|
|
def test_factory_validate_configuration_unknown_type(self, template_renderer):
|
|
"""Test validation fails for unknown agent type."""
|
|
config = {"agents": {"agent1": {"type": "unknown_type"}}}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(ConfigurationError) as exc_info:
|
|
factory.validate_configuration()
|
|
|
|
assert "unknown_type" in str(exc_info.value)
|
|
|
|
def test_factory_validate_configuration_invalid_agent_config_not_dict(self, template_renderer):
|
|
"""Test validation fails when agent's config field is not a dictionary."""
|
|
config = {"agents": {"agent1": {"type": "llm", "config": "not a dict"}}}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(ConfigurationError) as exc_info:
|
|
factory.validate_configuration()
|
|
|
|
assert "'config'" in str(exc_info.value)
|
|
assert "agent1" in str(exc_info.value)
|
|
|
|
def test_factory_get_agent_metadata_llm(self, template_renderer):
|
|
"""Test getting metadata for an LLM agent."""
|
|
config = {
|
|
"agents": {
|
|
"llm_agent": {
|
|
"type": "llm",
|
|
"config": {
|
|
"provider": "openai",
|
|
"model": "gpt-4",
|
|
"temperature": 0.8
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
metadata = factory.get_agent_metadata("llm_agent")
|
|
|
|
assert metadata["name"] == "llm_agent"
|
|
assert metadata["type"] == "llm"
|
|
assert metadata["provider"] == "openai"
|
|
assert metadata["model"] == "gpt-4"
|
|
assert metadata["temperature"] == 0.8
|
|
|
|
def test_factory_get_agent_metadata_tool(self, template_renderer):
|
|
"""Test getting metadata for a tool agent."""
|
|
config = {
|
|
"agents": {
|
|
"tool_agent": {
|
|
"type": "tool",
|
|
"config": {
|
|
"tools": ["echo", "math"],
|
|
"allow_shell": True,
|
|
"safe_mode": False
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
metadata = factory.get_agent_metadata("tool_agent")
|
|
|
|
assert metadata["name"] == "tool_agent"
|
|
assert metadata["type"] == "tool"
|
|
assert metadata["tools"] == ["echo", "math"]
|
|
assert metadata["allow_shell"] is True
|
|
assert metadata["safe_mode"] is False
|
|
|
|
def test_factory_get_agent_metadata_nonexistent(self, template_renderer):
|
|
"""Test getting metadata for nonexistent agent raises error."""
|
|
config = {"agents": {"agent1": {"type": "llm"}}}
|
|
|
|
factory = AgentFactory(config, template_renderer)
|
|
|
|
with pytest.raises(AgentCreationError) as exc_info:
|
|
factory.get_agent_metadata("nonexistent")
|
|
|
|
assert "nonexistent" in str(exc_info.value)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|
|
|