test: add detailed tests for Tool, WebSearchTool, and ToolAgent
This commit is contained in:
@@ -0,0 +1,309 @@
|
||||
import ast
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import AsyncMock
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import aiohttp
|
||||
from behave import given
|
||||
from behave import then
|
||||
from behave import when
|
||||
|
||||
from cleveragents.agents.tool import AgentCreationError
|
||||
from cleveragents.agents.tool import ConfigurationError
|
||||
from cleveragents.agents.tool import ExecutionError
|
||||
from cleveragents.agents.tool import Tool
|
||||
from cleveragents.agents.tool import ToolAgent
|
||||
from cleveragents.agents.tool import WebSearchTool
|
||||
from cleveragents.templates.renderer import TemplateRenderer
|
||||
|
||||
|
||||
# Helper to run async functions in behave steps
|
||||
def run_async(func):
|
||||
return asyncio.run(func())
|
||||
|
||||
|
||||
@given("I have a base Tool instance")
|
||||
def step_impl(context):
|
||||
context.tool = Tool(name="base_tool", description="A base tool.")
|
||||
|
||||
|
||||
@when('I try to execute the tool with input "{input_data}"')
|
||||
def step_impl(context, input_data):
|
||||
async def execute_tool():
|
||||
context.error = None
|
||||
try:
|
||||
await context.tool.execute(input_data)
|
||||
except Exception as e:
|
||||
context.error = e
|
||||
|
||||
run_async(execute_tool)
|
||||
|
||||
|
||||
@then("an NotImplementedError should be raised")
|
||||
def step_impl(context):
|
||||
assert isinstance(context.error, NotImplementedError)
|
||||
|
||||
|
||||
@given('no "GOOGLE_SEARCH_API_KEY" environment variable is set')
|
||||
def step_impl(context):
|
||||
if "GOOGLE_SEARCH_API_KEY" in os.environ:
|
||||
context.original_api_key = os.environ.pop("GOOGLE_SEARCH_API_KEY")
|
||||
else:
|
||||
context.original_api_key = None
|
||||
|
||||
|
||||
@given("a WebSearchTool configuration without an API key but with a CX")
|
||||
def step_impl(context):
|
||||
context.tool_config = {"cx": "test_cx"}
|
||||
|
||||
|
||||
@when("I try to create a WebSearchTool with this configuration")
|
||||
def step_impl(context):
|
||||
context.error = None
|
||||
try:
|
||||
WebSearchTool(config=context.tool_config)
|
||||
except Exception as e:
|
||||
context.error = e
|
||||
|
||||
if hasattr(context, "original_api_key") and context.original_api_key is not None:
|
||||
os.environ["GOOGLE_SEARCH_API_KEY"] = context.original_api_key
|
||||
|
||||
|
||||
@then('a ConfigurationError should be raised with a message containing "{message}"')
|
||||
def step_impl(context, message):
|
||||
assert isinstance(context.error, ConfigurationError)
|
||||
assert message in str(context.error)
|
||||
|
||||
|
||||
@given('a "GOOGLE_SEARCH_API_KEY" environment variable is set to "{value}"')
|
||||
def step_impl(context, value):
|
||||
context.original_api_key = os.environ.get("GOOGLE_SEARCH_API_KEY")
|
||||
os.environ["GOOGLE_SEARCH_API_KEY"] = value
|
||||
|
||||
|
||||
@given('no "GOOGLE_CX" environment variable is set')
|
||||
def step_impl(context):
|
||||
if "GOOGLE_CX" in os.environ:
|
||||
context.original_cx = os.environ.pop("GOOGLE_CX")
|
||||
else:
|
||||
context.original_cx = None
|
||||
|
||||
|
||||
@given("a WebSearchTool configuration without a CX")
|
||||
def step_impl(context):
|
||||
context.tool_config = {}
|
||||
|
||||
|
||||
@then('an AgentCreationError should be raised with a message containing "{message}"')
|
||||
def step_impl(context, message):
|
||||
assert isinstance(context.error, AgentCreationError)
|
||||
assert message in str(context.error)
|
||||
|
||||
if hasattr(context, "original_api_key") and context.original_api_key is not None:
|
||||
os.environ["GOOGLE_SEARCH_API_KEY"] = context.original_api_key
|
||||
elif "GOOGLE_SEARCH_API_KEY" in os.environ:
|
||||
del os.environ["GOOGLE_SEARCH_API_KEY"]
|
||||
|
||||
if hasattr(context, "original_cx") and context.original_cx is not None:
|
||||
os.environ["GOOGLE_CX"] = context.original_cx
|
||||
elif "GOOGLE_CX" in os.environ:
|
||||
del os.environ["GOOGLE_CX"]
|
||||
|
||||
|
||||
@given("a configured WebSearchTool")
|
||||
def step_impl(context):
|
||||
context.tool = WebSearchTool(config={"api_key": "test_key", "cx": "test_cx"})
|
||||
|
||||
|
||||
def patch_aiohttp_session(context, status=200, json_data=None, error=None, text=""):
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = status
|
||||
mock_response.json = AsyncMock(return_value=json_data)
|
||||
mock_response.text = AsyncMock(return_value=text)
|
||||
|
||||
get_cm = AsyncMock()
|
||||
get_cm.__aenter__.return_value = mock_response
|
||||
session_instance = AsyncMock()
|
||||
if error:
|
||||
session_instance.get = MagicMock(side_effect=error)
|
||||
else:
|
||||
session_instance.get = MagicMock(return_value=get_cm)
|
||||
mock_session_cm = AsyncMock()
|
||||
mock_session_cm.__aenter__.return_value = session_instance
|
||||
|
||||
context.aiohttp_patch = patch("aiohttp.ClientSession", return_value=mock_session_cm)
|
||||
context.aiohttp_patch.start()
|
||||
|
||||
|
||||
@given("the Google Search API will return a 403 error")
|
||||
def step_impl(context):
|
||||
patch_aiohttp_session(context, status=403, text="Forbidden")
|
||||
|
||||
|
||||
@when('I execute the web_search tool with query "{query}"')
|
||||
def step_impl(context, query):
|
||||
async def execute_tool():
|
||||
context.error = None
|
||||
context.result = None
|
||||
try:
|
||||
context.result = await context.tool.execute(query)
|
||||
except Exception as e:
|
||||
context.error = e
|
||||
finally:
|
||||
if hasattr(context, "aiohttp_patch"):
|
||||
context.aiohttp_patch.stop()
|
||||
|
||||
run_async(execute_tool)
|
||||
|
||||
|
||||
@then('an ExecutionError should be raised with a message containing "{message}"')
|
||||
def step_impl(context, message):
|
||||
assert isinstance(
|
||||
context.error, ExecutionError
|
||||
), f"Expected ExecutionError, but got {type(context.error)}"
|
||||
assert message in str(context.error)
|
||||
|
||||
|
||||
@given("the Google Search API will raise a connection error")
|
||||
def step_impl(context):
|
||||
patch_aiohttp_session(context, error=aiohttp.ClientError("Connection failed"))
|
||||
|
||||
|
||||
@given("the Google Search API will raise an unexpected error")
|
||||
def step_impl(context):
|
||||
patch_aiohttp_session(context, error=ValueError("Unexpected error"))
|
||||
|
||||
|
||||
@given("the Google Search API will return no results")
|
||||
def step_impl(context):
|
||||
patch_aiohttp_session(context, status=200, json_data={"items": []})
|
||||
|
||||
|
||||
@then('the result must be "{expected_result}"')
|
||||
def step_impl(context, expected_result):
|
||||
assert context.result == expected_result
|
||||
|
||||
|
||||
@given("a ToolAgent configuration with an invalid tool entry: {invalid_config}")
|
||||
def step_impl(context, invalid_config):
|
||||
try:
|
||||
# The feature file has strings like "'{\"name\": \"\"}'" or "'not_a_dictionary'".
|
||||
# We need to evaluate the outer quotes to get the inner string.
|
||||
config_str = ast.literal_eval(invalid_config)
|
||||
try:
|
||||
# Try to parse the result as JSON
|
||||
config_data = json.loads(config_str)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
# If it's not JSON (e.g., 'not_a_dictionary'), use the string itself
|
||||
config_data = config_str
|
||||
except (ValueError, SyntaxError):
|
||||
config_data = invalid_config
|
||||
|
||||
context.agent_config = {"config": {"tools": [config_data]}}
|
||||
context.template_renderer = TemplateRenderer()
|
||||
|
||||
|
||||
@when("I try to create a ToolAgent with this configuration")
|
||||
def step_impl(context):
|
||||
context.error = None
|
||||
try:
|
||||
ToolAgent(
|
||||
name="test_agent",
|
||||
config=context.agent_config,
|
||||
template_renderer=context.template_renderer,
|
||||
)
|
||||
except Exception as e:
|
||||
context.error = e
|
||||
|
||||
|
||||
@given("a ToolAgent is created with no tools")
|
||||
def step_impl(context):
|
||||
config = {"config": {"tools": []}}
|
||||
context.agent = ToolAgent(
|
||||
name="no_tool_agent", config=config, template_renderer=TemplateRenderer()
|
||||
)
|
||||
|
||||
|
||||
@when('I try to process a message "{message}" with the agent')
|
||||
def step_impl(context, message):
|
||||
async def process_message():
|
||||
context.error = None
|
||||
try:
|
||||
await context.agent.process(message)
|
||||
except Exception as e:
|
||||
context.error = e
|
||||
|
||||
run_async(process_message)
|
||||
|
||||
|
||||
@given('a ToolAgent with a "{tool_name}" tool')
|
||||
def step_impl(context, tool_name):
|
||||
config = {"config": {"tools": [{"name": tool_name}]}}
|
||||
context.agent = ToolAgent(
|
||||
name="single_tool_agent", config=config, template_renderer=TemplateRenderer()
|
||||
)
|
||||
|
||||
|
||||
@when('I process the following message "{message}" with the agent')
|
||||
def step_impl(context, message):
|
||||
async def process_message():
|
||||
context.error = None
|
||||
context.response = None
|
||||
try:
|
||||
context.response = await context.agent.process(message)
|
||||
except Exception as e:
|
||||
context.error = e
|
||||
|
||||
run_async(process_message)
|
||||
|
||||
|
||||
@then('the response must be "{expected_response}"')
|
||||
def step_impl(context, expected_response):
|
||||
assert context.response == expected_response
|
||||
|
||||
|
||||
@when("I try to parse a tool request with the agent")
|
||||
def step_impl(context):
|
||||
context.error = None
|
||||
try:
|
||||
# This will fail because there are no tools, causing next(iter) to raise StopIteration
|
||||
context.agent._parse_tool_request("any message")
|
||||
except Exception as e:
|
||||
context.error = e
|
||||
|
||||
|
||||
@given('a ToolAgent with "calculator" and "web_search" tools')
|
||||
def step_impl(context):
|
||||
config = {
|
||||
"config": {
|
||||
"tools": [
|
||||
{"name": "calculator"},
|
||||
{"name": "web_search", "config": {"api_key": "fake", "cx": "fake"}},
|
||||
]
|
||||
}
|
||||
}
|
||||
context.agent = ToolAgent(
|
||||
name="multi_tool_agent", config=config, template_renderer=TemplateRenderer()
|
||||
)
|
||||
# Mock the web_search tool's execute method to avoid real calls
|
||||
context.agent.tools["web_search"].execute = AsyncMock(
|
||||
return_value="web search result"
|
||||
)
|
||||
|
||||
|
||||
@when("I obtain the agent's capabilities")
|
||||
def step_impl(context):
|
||||
context.capabilities = context.agent.get_capabilities()
|
||||
|
||||
|
||||
@then(
|
||||
'the capabilities list should contain "tool-execution", "tool-calculator", and "tool-web_search"'
|
||||
)
|
||||
def step_impl(context):
|
||||
assert "tool-execution" in context.capabilities
|
||||
assert "tool-calculator" in context.capabilities
|
||||
assert "tool-web_search" in context.capabilities
|
||||
assert len(context.capabilities) == 3
|
||||
@@ -0,0 +1,95 @@
|
||||
Feature: Detailed Tool Agent and Tool Functionality
|
||||
This feature covers detailed test cases for Tool, WebSearchTool, and ToolAgent,
|
||||
focusing on error handling, configuration issues, and edge cases to improve
|
||||
test coverage.
|
||||
|
||||
Scenario: Calling execute on the base Tool class raises an error
|
||||
Given I have a base Tool instance
|
||||
When I try to execute the tool with input "test"
|
||||
Then an NotImplementedError should be raised
|
||||
|
||||
Scenario: WebSearchTool raises error if Google API key is missing
|
||||
Given no "GOOGLE_SEARCH_API_KEY" environment variable is set
|
||||
And a WebSearchTool configuration without an API key but with a CX
|
||||
When I try to create a WebSearchTool with this configuration
|
||||
Then a ConfigurationError should be raised with a message containing "API key for search engine 'google' not found"
|
||||
|
||||
Scenario: WebSearchTool raises error if Google CX is missing
|
||||
Given a "GOOGLE_SEARCH_API_KEY" environment variable is set to "fake_key"
|
||||
And no "GOOGLE_CX" environment variable is set
|
||||
And a WebSearchTool configuration without a CX
|
||||
When I try to create a WebSearchTool with this configuration
|
||||
Then an AgentCreationError should be raised with a message containing "Google Custom Search Engine ID (cx) not found"
|
||||
|
||||
Scenario: WebSearchTool handles API errors gracefully
|
||||
Given a configured WebSearchTool
|
||||
And the Google Search API will return a 403 error
|
||||
When I execute the web_search tool with query "test"
|
||||
Then an ExecutionError should be raised with a message containing "Google Search API error: Received status 403"
|
||||
|
||||
Scenario: WebSearchTool handles connection errors
|
||||
Given a configured WebSearchTool
|
||||
And the Google Search API will raise a connection error
|
||||
When I execute the web_search tool with query "test"
|
||||
Then an ExecutionError should be raised with a message containing "Failed to connect to Google Search API"
|
||||
|
||||
Scenario: WebSearchTool handles unexpected errors during search
|
||||
Given a configured WebSearchTool
|
||||
And the Google Search API will raise an unexpected error
|
||||
When I execute the web_search tool with query "test"
|
||||
Then an ExecutionError should be raised with a message containing "An unexpected error occurred during web search"
|
||||
|
||||
Scenario: WebSearchTool handles no search results
|
||||
Given a configured WebSearchTool
|
||||
And the Google Search API will return no results
|
||||
When I execute the web_search tool with query "test"
|
||||
Then the result must be "No web search results found."
|
||||
|
||||
Scenario Outline: ToolAgent initialization fails with invalid tool configurations
|
||||
Given a ToolAgent configuration with an invalid tool entry: <invalid_config>
|
||||
When I try to create a ToolAgent with this configuration
|
||||
Then an AgentCreationError should be raised with a message containing "<error_message>"
|
||||
|
||||
Examples:
|
||||
| invalid_config | error_message |
|
||||
| 'not_a_dictionary' | Each tool configuration must be a dictionary |
|
||||
| '{"name": ""}' | Tool 'name' must be a non-empty string |
|
||||
| '{"name": 123}' | Tool 'name' must be a non-empty string |
|
||||
| '{"name": "custom_tool"}' | No module specified for custom tool 'custom_tool' |
|
||||
| '{"name": "non_existent", "module": "non.existent"}' | Failed to load tool 'non_existent' from module 'non.existent' |
|
||||
| '{"name": "bad_class", "module": "tests.custom_module"}' | has no attribute 'BadClassTool' |
|
||||
|
||||
Scenario: ToolAgent with no tools cannot process messages
|
||||
Given a ToolAgent is created with no tools
|
||||
When I try to process a message "hello" with the agent
|
||||
Then an ExecutionError should be raised with a message containing "ToolAgent has no tools configured"
|
||||
|
||||
Scenario: ToolAgent handles requests for non-existent tools
|
||||
Given a ToolAgent with a "calculator" tool
|
||||
When I process the following message "unknown_tool(1+1)" with the agent
|
||||
Then the response must be "Tool 'unknown_tool' not found. Available tools: calculator"
|
||||
|
||||
Scenario: ToolAgent handles exceptions during tool execution
|
||||
Given a ToolAgent with a "calculator" tool
|
||||
When I process the following message "calculator(1/0)" with the agent
|
||||
Then an ExecutionError should be raised with a message containing "Failed to process message: Failed to evaluate expression: division by zero"
|
||||
|
||||
Scenario: ToolAgent handles exceptions during message parsing
|
||||
Given a ToolAgent is created with no tools
|
||||
When I try to parse a tool request with the agent
|
||||
Then an ExecutionError should be raised with a message containing "Failed to parse tool request"
|
||||
|
||||
Scenario: ToolAgent parses message with invalid JSON gracefully
|
||||
Given a ToolAgent with a "calculator" tool
|
||||
When I process the following message "calculator: 1+1" with the agent
|
||||
Then the response must be "2"
|
||||
|
||||
Scenario: ToolAgent infers tool from message content
|
||||
Given a ToolAgent with "calculator" and "web_search" tools
|
||||
When I process the following message "what is the result of calculator 2+2" with the agent
|
||||
Then the response must be "4"
|
||||
|
||||
Scenario: ToolAgent get_capabilities returns correct list
|
||||
Given a ToolAgent with "calculator" and "web_search" tools
|
||||
When I obtain the agent's capabilities
|
||||
Then the capabilities list should contain "tool-execution", "tool-calculator", and "tool-web_search"
|
||||
Reference in New Issue
Block a user