diff --git a/tests/features/steps/tool_agent_detailed_steps.py b/tests/features/steps/tool_agent_detailed_steps.py new file mode 100644 index 000000000..e34b864f4 --- /dev/null +++ b/tests/features/steps/tool_agent_detailed_steps.py @@ -0,0 +1,309 @@ +import ast +import asyncio +import json +import os +from unittest.mock import AsyncMock +from unittest.mock import MagicMock +from unittest.mock import patch + +import aiohttp +from behave import given +from behave import then +from behave import when + +from cleveragents.agents.tool import AgentCreationError +from cleveragents.agents.tool import ConfigurationError +from cleveragents.agents.tool import ExecutionError +from cleveragents.agents.tool import Tool +from cleveragents.agents.tool import ToolAgent +from cleveragents.agents.tool import WebSearchTool +from cleveragents.templates.renderer import TemplateRenderer + + +# Helper to run async functions in behave steps +def run_async(func): + return asyncio.run(func()) + + +@given("I have a base Tool instance") +def step_impl(context): + context.tool = Tool(name="base_tool", description="A base tool.") + + +@when('I try to execute the tool with input "{input_data}"') +def step_impl(context, input_data): + async def execute_tool(): + context.error = None + try: + await context.tool.execute(input_data) + except Exception as e: + context.error = e + + run_async(execute_tool) + + +@then("an NotImplementedError should be raised") +def step_impl(context): + assert isinstance(context.error, NotImplementedError) + + +@given('no "GOOGLE_SEARCH_API_KEY" environment variable is set') +def step_impl(context): + if "GOOGLE_SEARCH_API_KEY" in os.environ: + context.original_api_key = os.environ.pop("GOOGLE_SEARCH_API_KEY") + else: + context.original_api_key = None + + +@given("a WebSearchTool configuration without an API key but with a CX") +def step_impl(context): + context.tool_config = {"cx": "test_cx"} + + +@when("I try to create a WebSearchTool with this configuration") +def step_impl(context): + context.error = None + try: + WebSearchTool(config=context.tool_config) + except Exception as e: + context.error = e + + if hasattr(context, "original_api_key") and context.original_api_key is not None: + os.environ["GOOGLE_SEARCH_API_KEY"] = context.original_api_key + + +@then('a ConfigurationError should be raised with a message containing "{message}"') +def step_impl(context, message): + assert isinstance(context.error, ConfigurationError) + assert message in str(context.error) + + +@given('a "GOOGLE_SEARCH_API_KEY" environment variable is set to "{value}"') +def step_impl(context, value): + context.original_api_key = os.environ.get("GOOGLE_SEARCH_API_KEY") + os.environ["GOOGLE_SEARCH_API_KEY"] = value + + +@given('no "GOOGLE_CX" environment variable is set') +def step_impl(context): + if "GOOGLE_CX" in os.environ: + context.original_cx = os.environ.pop("GOOGLE_CX") + else: + context.original_cx = None + + +@given("a WebSearchTool configuration without a CX") +def step_impl(context): + context.tool_config = {} + + +@then('an AgentCreationError should be raised with a message containing "{message}"') +def step_impl(context, message): + assert isinstance(context.error, AgentCreationError) + assert message in str(context.error) + + if hasattr(context, "original_api_key") and context.original_api_key is not None: + os.environ["GOOGLE_SEARCH_API_KEY"] = context.original_api_key + elif "GOOGLE_SEARCH_API_KEY" in os.environ: + del os.environ["GOOGLE_SEARCH_API_KEY"] + + if hasattr(context, "original_cx") and context.original_cx is not None: + os.environ["GOOGLE_CX"] = context.original_cx + elif "GOOGLE_CX" in os.environ: + del os.environ["GOOGLE_CX"] + + +@given("a configured WebSearchTool") +def step_impl(context): + context.tool = WebSearchTool(config={"api_key": "test_key", "cx": "test_cx"}) + + +def patch_aiohttp_session(context, status=200, json_data=None, error=None, text=""): + mock_response = AsyncMock() + mock_response.status = status + mock_response.json = AsyncMock(return_value=json_data) + mock_response.text = AsyncMock(return_value=text) + + get_cm = AsyncMock() + get_cm.__aenter__.return_value = mock_response + session_instance = AsyncMock() + if error: + session_instance.get = MagicMock(side_effect=error) + else: + session_instance.get = MagicMock(return_value=get_cm) + mock_session_cm = AsyncMock() + mock_session_cm.__aenter__.return_value = session_instance + + context.aiohttp_patch = patch("aiohttp.ClientSession", return_value=mock_session_cm) + context.aiohttp_patch.start() + + +@given("the Google Search API will return a 403 error") +def step_impl(context): + patch_aiohttp_session(context, status=403, text="Forbidden") + + +@when('I execute the web_search tool with query "{query}"') +def step_impl(context, query): + async def execute_tool(): + context.error = None + context.result = None + try: + context.result = await context.tool.execute(query) + except Exception as e: + context.error = e + finally: + if hasattr(context, "aiohttp_patch"): + context.aiohttp_patch.stop() + + run_async(execute_tool) + + +@then('an ExecutionError should be raised with a message containing "{message}"') +def step_impl(context, message): + assert isinstance( + context.error, ExecutionError + ), f"Expected ExecutionError, but got {type(context.error)}" + assert message in str(context.error) + + +@given("the Google Search API will raise a connection error") +def step_impl(context): + patch_aiohttp_session(context, error=aiohttp.ClientError("Connection failed")) + + +@given("the Google Search API will raise an unexpected error") +def step_impl(context): + patch_aiohttp_session(context, error=ValueError("Unexpected error")) + + +@given("the Google Search API will return no results") +def step_impl(context): + patch_aiohttp_session(context, status=200, json_data={"items": []}) + + +@then('the result must be "{expected_result}"') +def step_impl(context, expected_result): + assert context.result == expected_result + + +@given("a ToolAgent configuration with an invalid tool entry: {invalid_config}") +def step_impl(context, invalid_config): + try: + # The feature file has strings like "'{\"name\": \"\"}'" or "'not_a_dictionary'". + # We need to evaluate the outer quotes to get the inner string. + config_str = ast.literal_eval(invalid_config) + try: + # Try to parse the result as JSON + config_data = json.loads(config_str) + except (json.JSONDecodeError, TypeError): + # If it's not JSON (e.g., 'not_a_dictionary'), use the string itself + config_data = config_str + except (ValueError, SyntaxError): + config_data = invalid_config + + context.agent_config = {"config": {"tools": [config_data]}} + context.template_renderer = TemplateRenderer() + + +@when("I try to create a ToolAgent with this configuration") +def step_impl(context): + context.error = None + try: + ToolAgent( + name="test_agent", + config=context.agent_config, + template_renderer=context.template_renderer, + ) + except Exception as e: + context.error = e + + +@given("a ToolAgent is created with no tools") +def step_impl(context): + config = {"config": {"tools": []}} + context.agent = ToolAgent( + name="no_tool_agent", config=config, template_renderer=TemplateRenderer() + ) + + +@when('I try to process a message "{message}" with the agent') +def step_impl(context, message): + async def process_message(): + context.error = None + try: + await context.agent.process(message) + except Exception as e: + context.error = e + + run_async(process_message) + + +@given('a ToolAgent with a "{tool_name}" tool') +def step_impl(context, tool_name): + config = {"config": {"tools": [{"name": tool_name}]}} + context.agent = ToolAgent( + name="single_tool_agent", config=config, template_renderer=TemplateRenderer() + ) + + +@when('I process the following message "{message}" with the agent') +def step_impl(context, message): + async def process_message(): + context.error = None + context.response = None + try: + context.response = await context.agent.process(message) + except Exception as e: + context.error = e + + run_async(process_message) + + +@then('the response must be "{expected_response}"') +def step_impl(context, expected_response): + assert context.response == expected_response + + +@when("I try to parse a tool request with the agent") +def step_impl(context): + context.error = None + try: + # This will fail because there are no tools, causing next(iter) to raise StopIteration + context.agent._parse_tool_request("any message") + except Exception as e: + context.error = e + + +@given('a ToolAgent with "calculator" and "web_search" tools') +def step_impl(context): + config = { + "config": { + "tools": [ + {"name": "calculator"}, + {"name": "web_search", "config": {"api_key": "fake", "cx": "fake"}}, + ] + } + } + context.agent = ToolAgent( + name="multi_tool_agent", config=config, template_renderer=TemplateRenderer() + ) + # Mock the web_search tool's execute method to avoid real calls + context.agent.tools["web_search"].execute = AsyncMock( + return_value="web search result" + ) + + +@when("I obtain the agent's capabilities") +def step_impl(context): + context.capabilities = context.agent.get_capabilities() + + +@then( + 'the capabilities list should contain "tool-execution", "tool-calculator", and "tool-web_search"' +) +def step_impl(context): + assert "tool-execution" in context.capabilities + assert "tool-calculator" in context.capabilities + assert "tool-web_search" in context.capabilities + assert len(context.capabilities) == 3 diff --git a/tests/features/tool_agent_detailed.feature b/tests/features/tool_agent_detailed.feature new file mode 100644 index 000000000..a52f92d4a --- /dev/null +++ b/tests/features/tool_agent_detailed.feature @@ -0,0 +1,95 @@ +Feature: Detailed Tool Agent and Tool Functionality + This feature covers detailed test cases for Tool, WebSearchTool, and ToolAgent, + focusing on error handling, configuration issues, and edge cases to improve + test coverage. + + Scenario: Calling execute on the base Tool class raises an error + Given I have a base Tool instance + When I try to execute the tool with input "test" + Then an NotImplementedError should be raised + + Scenario: WebSearchTool raises error if Google API key is missing + Given no "GOOGLE_SEARCH_API_KEY" environment variable is set + And a WebSearchTool configuration without an API key but with a CX + When I try to create a WebSearchTool with this configuration + Then a ConfigurationError should be raised with a message containing "API key for search engine 'google' not found" + + Scenario: WebSearchTool raises error if Google CX is missing + Given a "GOOGLE_SEARCH_API_KEY" environment variable is set to "fake_key" + And no "GOOGLE_CX" environment variable is set + And a WebSearchTool configuration without a CX + When I try to create a WebSearchTool with this configuration + Then an AgentCreationError should be raised with a message containing "Google Custom Search Engine ID (cx) not found" + + Scenario: WebSearchTool handles API errors gracefully + Given a configured WebSearchTool + And the Google Search API will return a 403 error + When I execute the web_search tool with query "test" + Then an ExecutionError should be raised with a message containing "Google Search API error: Received status 403" + + Scenario: WebSearchTool handles connection errors + Given a configured WebSearchTool + And the Google Search API will raise a connection error + When I execute the web_search tool with query "test" + Then an ExecutionError should be raised with a message containing "Failed to connect to Google Search API" + + Scenario: WebSearchTool handles unexpected errors during search + Given a configured WebSearchTool + And the Google Search API will raise an unexpected error + When I execute the web_search tool with query "test" + Then an ExecutionError should be raised with a message containing "An unexpected error occurred during web search" + + Scenario: WebSearchTool handles no search results + Given a configured WebSearchTool + And the Google Search API will return no results + When I execute the web_search tool with query "test" + Then the result must be "No web search results found." + + Scenario Outline: ToolAgent initialization fails with invalid tool configurations + Given a ToolAgent configuration with an invalid tool entry: + When I try to create a ToolAgent with this configuration + Then an AgentCreationError should be raised with a message containing "" + + Examples: + | invalid_config | error_message | + | 'not_a_dictionary' | Each tool configuration must be a dictionary | + | '{"name": ""}' | Tool 'name' must be a non-empty string | + | '{"name": 123}' | Tool 'name' must be a non-empty string | + | '{"name": "custom_tool"}' | No module specified for custom tool 'custom_tool' | + | '{"name": "non_existent", "module": "non.existent"}' | Failed to load tool 'non_existent' from module 'non.existent' | + | '{"name": "bad_class", "module": "tests.custom_module"}' | has no attribute 'BadClassTool' | + + Scenario: ToolAgent with no tools cannot process messages + Given a ToolAgent is created with no tools + When I try to process a message "hello" with the agent + Then an ExecutionError should be raised with a message containing "ToolAgent has no tools configured" + + Scenario: ToolAgent handles requests for non-existent tools + Given a ToolAgent with a "calculator" tool + When I process the following message "unknown_tool(1+1)" with the agent + Then the response must be "Tool 'unknown_tool' not found. Available tools: calculator" + + Scenario: ToolAgent handles exceptions during tool execution + Given a ToolAgent with a "calculator" tool + When I process the following message "calculator(1/0)" with the agent + Then an ExecutionError should be raised with a message containing "Failed to process message: Failed to evaluate expression: division by zero" + + Scenario: ToolAgent handles exceptions during message parsing + Given a ToolAgent is created with no tools + When I try to parse a tool request with the agent + Then an ExecutionError should be raised with a message containing "Failed to parse tool request" + + Scenario: ToolAgent parses message with invalid JSON gracefully + Given a ToolAgent with a "calculator" tool + When I process the following message "calculator: 1+1" with the agent + Then the response must be "2" + + Scenario: ToolAgent infers tool from message content + Given a ToolAgent with "calculator" and "web_search" tools + When I process the following message "what is the result of calculator 2+2" with the agent + Then the response must be "4" + + Scenario: ToolAgent get_capabilities returns correct list + Given a ToolAgent with "calculator" and "web_search" tools + When I obtain the agent's capabilities + Then the capabilities list should contain "tool-execution", "tool-calculator", and "tool-web_search"