Files
CoreRasurae 52451b2550
CI / lint (pull_request) Successful in 36s
CI / typecheck (pull_request) Successful in 52s
CI / security (pull_request) Successful in 50s
CI / quality (pull_request) Successful in 34s
CI / unit_tests (pull_request) Successful in 3m9s
CI / integration_tests (pull_request) Successful in 1m3s
CI / build (pull_request) Successful in 35s
CI / coverage (pull_request) Successful in 3m9s
CI / status-check (pull_request) Successful in 4s
CI / lint (push) Successful in 46s
CI / typecheck (push) Successful in 51s
CI / security (push) Successful in 50s
CI / quality (push) Successful in 34s
CI / unit_tests (push) Successful in 3m8s
CI / build (push) Successful in 37s
CI / coverage (push) Successful in 3m8s
CI / integration_tests (push) Successful in 1m12s
CI / status-check (push) Successful in 2s
test(agents): disable retry in error-path tests to avoid unnecessary delays
Pre-existing tests that inject error-raising mock chat models (raising
LangChainException or RuntimeError) were triggering the full
exponential-backoff retry loop (~63.5s per test). With 6 such tests,
this added minutes to the suite.

Set max_retries=0 in each error-path step definition so the retry loop
fails fast on the first attempt, preserving the test contract while
avoiding unnecessary sleep delays.
2026-07-03 16:00:20 +01:00

1394 lines
51 KiB
Python

"""Step definitions for LLM Agent Initialization and Message Processing BDD tests."""
import asyncio
import os
from unittest.mock import AsyncMock, Mock, patch
from behave import given, then, when
from behave.api.async_step import async_run_until_complete
from langchain_core.messages import AIMessage
from cleveractors.agents.llm import LLMAgent
from cleveractors.core.exceptions import ConfigurationError, ExecutionError
from cleveractors.templates.renderer import TemplateRenderer
def create_mock_chat_model(response_text=None, error_message=None):
"""Create a mock chat model for testing."""
mock_model = Mock()
if error_message:
from langchain_core.exceptions import LangChainException
mock_model.ainvoke = AsyncMock(side_effect=LangChainException(error_message))
else:
mock_response = AIMessage(content=response_text or "Mock response")
mock_model.ainvoke = AsyncMock(return_value=mock_response)
return mock_model
@given("the LLM agent system is initialized")
def step_system_initialized(context):
"""Initialize the system for testing."""
context.system_initialized = True
@given("I have a basic LLM agent configuration")
def step_basic_llm_config(context):
"""Set up basic LLM agent configuration."""
context.llm_config = {"name": "test_llm_agent", "api_key": "test_api_key"}
@given("I have a custom LLM agent configuration")
def step_custom_llm_config(context):
"""Set up custom LLM agent configuration."""
context.llm_config = {
"name": "custom_llm_agent",
"provider": "anthropic",
"model": "claude-3-sonnet",
"temperature": 0.5,
"max_tokens": 2000,
"system_prompt": "You are a helpful AI assistant specialized in testing.",
"api_key": "test_anthropic_key",
"memory_enabled": True,
"max_history": 20,
}
@given("I have an LLM agent configuration without API key")
def step_config_no_api_key(context):
"""Set up configuration without API key."""
context.llm_config = {"name": "no_key_agent", "provider": "openai"}
@given("I have an LLM agent configuration with unsupported provider")
def step_config_unsupported_provider(context):
"""Set up configuration with unsupported provider."""
context.llm_config = {
"name": "unsupported_agent",
"provider": "unsupported_provider",
"api_key": "test_key",
}
@given("I have an LLM agent configuration with API key in config")
def step_config_with_api_key(context):
"""Set up configuration with API key in config."""
context.llm_config = {
"name": "config_key_agent",
"provider": "openai",
"api_key": "config_api_key",
}
@given("I have an environment variable set for OpenAI API key")
def step_env_openai_key(context):
"""Set up OpenAI environment variable."""
context.env_patch = patch.dict(os.environ, {"OPENAI_API_KEY": "env_openai_key"})
context.env_patch.start()
@given("I have an LLM agent configuration for environment test")
def step_config_no_key_for_env(context):
"""Set up configuration without API key for environment test."""
context.llm_config = {"name": "env_key_agent", "provider": "openai"}
@given("I have an environment variable set for Anthropic API key")
def step_env_anthropic_key(context):
"""Set up Anthropic environment variable."""
context.env_patch = patch.dict(
os.environ, {"ANTHROPIC_API_KEY": "env_anthropic_key"}
)
context.env_patch.start()
@given("I have an LLM agent configuration for Anthropic without API key")
def step_config_anthropic_no_key(context):
"""Set up Anthropic configuration without API key."""
context.llm_config = {"name": "anthropic_env_agent", "provider": "anthropic"}
@given("I have an environment variable set for Google API key")
def step_env_google_key(context):
"""Set up Google environment variable."""
context.env_patch = patch.dict(os.environ, {"GOOGLE_API_KEY": "env_google_key"})
context.env_patch.start()
@given("I have an LLM agent configuration for Google without API key")
def step_config_google_no_key(context):
"""Set up Google configuration without API key."""
context.llm_config = {
"name": "google_env_agent",
"provider": "google",
"model": "gemini-pro",
}
@given("I have an LLM agent configuration for OpenAI")
def step_config_openai(context):
"""Set up OpenAI configuration."""
context.llm_config = {
"name": "openai_agent",
"provider": "openai",
"api_key": "openai_test_key",
}
@given("I have an LLM agent configuration for Anthropic")
def step_config_anthropic(context):
"""Set up Anthropic configuration."""
context.llm_config = {
"name": "anthropic_agent",
"provider": "anthropic",
"api_key": "anthropic_test_key",
}
@given("I have an LLM agent configuration for Google")
def step_config_google(context):
"""Set up Google configuration."""
context.llm_config = {
"name": "google_agent",
"provider": "google",
"model": "gemini-pro",
"api_key": "google_test_key",
}
@given("I have an initialized LLM agent")
def step_initialized_llm_agent(context):
"""Initialize an LLM agent for testing."""
config = {"name": "test_agent", "provider": "openai", "api_key": "test_key"}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("test_agent", config, context.template_renderer)
@given("I have an initialized LLM agent with template configuration")
def step_initialized_llm_agent_with_template(context):
"""Initialize an LLM agent with template configuration."""
config = {
"name": "template_agent",
"provider": "openai",
"api_key": "test_key",
"template": "test_template",
"template_vars": {"var1": "value1"},
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("template_agent", config, context.template_renderer)
@given("I have a template renderer with test template")
def step_template_renderer_setup(context):
"""Set up template renderer mock."""
context.template_renderer.render.return_value = "rendered message with variables"
@given("I have an initialized OpenAI LLM agent")
def step_initialized_openai_agent(context):
"""Initialize OpenAI LLM agent."""
config = {"name": "openai_agent", "provider": "openai", "api_key": "openai_key"}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("openai_agent", config, context.template_renderer)
@given("I have an initialized Anthropic LLM agent")
def step_initialized_anthropic_agent(context):
"""Initialize Anthropic LLM agent."""
config = {
"name": "anthropic_agent",
"provider": "anthropic",
"api_key": "anthropic_key",
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("anthropic_agent", config, context.template_renderer)
@given("I have an initialized Google LLM agent")
def step_initialized_google_agent(context):
"""Initialize Google LLM agent.
Injects a mock model via the chat_model setter to avoid constructing a real
``ChatGoogleGenerativeAI`` (which may not be installed in the test
environment). With lazy initialisation, setting ``chat_model`` before first
access prevents the actual lazy-init path from running.
"""
config = {
"name": "google_agent",
"provider": "google",
"model": "gemini-pro",
"api_key": "google_key",
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("google_agent", config, context.template_renderer)
# Pre-set mock via setter — bypasses lazy init (ChatGoogleGenerativeAI
# may not be installed in the CI environment).
context.llm_agent.chat_model = create_mock_chat_model("Mock Google response")
@given("I have an initialized LLM agent with memory enabled")
def step_initialized_llm_agent_memory_enabled(context):
"""Initialize LLM agent with memory enabled."""
config = {
"name": "memory_agent",
"provider": "openai",
"api_key": "test_key",
"memory_enabled": True,
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("memory_agent", config, context.template_renderer)
context.llm_agent.update_memory = AsyncMock()
context.llm_agent.get_memory = AsyncMock(return_value=[])
@given("I have an initialized OpenAI LLM agent with memory enabled")
def step_initialized_openai_llm_agent_memory_enabled(context):
"""Initialize OpenAI LLM agent with memory enabled."""
config = {
"name": "openai_memory_agent",
"provider": "openai",
"api_key": "test_key",
"memory_enabled": True,
"max_history": 10,
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
"openai_memory_agent", config, context.template_renderer
)
context.llm_agent.update_memory = AsyncMock()
context.llm_agent.get_memory = AsyncMock(return_value=[])
@given("I have an initialized OpenAI LLM agent with memory disabled")
def step_initialized_openai_llm_agent_memory_disabled(context):
"""Initialize OpenAI LLM agent with memory disabled."""
config = {
"name": "openai_no_memory_agent",
"provider": "openai",
"api_key": "test_key",
"memory_enabled": False,
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
"openai_no_memory_agent", config, context.template_renderer
)
@given("I have an initialized LLM agent with memory disabled")
def step_initialized_llm_agent_memory_disabled(context):
"""Initialize LLM agent with memory disabled."""
config = {
"name": "no_memory_agent",
"provider": "openai",
"api_key": "test_key",
"memory_enabled": False,
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("no_memory_agent", config, context.template_renderer)
context.llm_agent.update_memory = AsyncMock()
@given("I have existing conversation history in memory")
def step_existing_conversation_history(context):
"""Set up existing conversation history."""
context.existing_history = [
{"role": "user", "content": "Previous user message"},
{"role": "assistant", "content": "Previous assistant response"},
]
context.llm_agent.get_memory = AsyncMock(return_value=context.existing_history)
@given("I have an initialized LLM agent with custom configuration")
def step_initialized_custom_llm_agent(context):
"""Initialize LLM agent with custom configuration."""
config = {
"name": "custom_agent",
"provider": "anthropic",
"model": "claude-3-sonnet",
"temperature": 0.3,
"max_tokens": 1500,
"api_key": "test_key",
"memory_enabled": True,
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent("custom_agent", config, context.template_renderer)
@given("I have an initialized LLM agent with template")
def step_initialized_llm_agent_template(context):
"""Initialize LLM agent with template configuration."""
config = {
"name": "template_agent",
"provider": "openai",
"api_key": "test_key",
"template": "test_template",
"template_vars": {"global_var": "global_value"},
}
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.template_renderer.render.return_value = (
"rendered test message with global_value and local_value"
)
context.llm_agent = LLMAgent("template_agent", config, context.template_renderer)
@given("I have a conversation history at maximum length")
def step_max_length_history(context):
"""Set up conversation history at maximum length."""
context.llm_agent.config["max_history"] = 4
context.max_history = [
{"role": "user", "content": "Message 1"},
{"role": "assistant", "content": "Response 1"},
{"role": "user", "content": "Message 2"},
{"role": "assistant", "content": "Response 2"},
]
context.llm_agent.get_memory = AsyncMock(return_value=context.max_history)
@when("I create an LLM agent with default settings")
def step_create_llm_agent_default(context):
"""Create LLM agent with default settings."""
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
context.llm_config["name"], context.llm_config, context.template_renderer
)
@when("I create an LLM agent with custom settings")
def step_create_llm_agent_custom(context):
"""Create LLM agent with custom settings."""
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
context.llm_config["name"], context.llm_config, context.template_renderer
)
@when("I try to create an LLM agent")
def step_try_create_llm_agent(context):
"""Try to create LLM agent and catch exceptions.
With lazy initialisation, ConfigurationErrors (missing API key, unsupported
provider) are raised on first ``chat_model`` access rather than during
``__init__``. This step therefore also triggers lazy initialisation so
the exception is captured here, keeping the test scenarios unchanged.
"""
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
try:
# Clear any environment variables that might interfere
with patch.dict(os.environ, {}, clear=True):
context.llm_agent = LLMAgent(
context.llm_config["name"],
context.llm_config,
context.template_renderer,
)
# Trigger lazy init — ConfigurationError is raised here (deferred
# from __init__ by the lazy-init refactor).
_ = context.llm_agent.chat_model
context.exception = None
except Exception as e:
context.exception = e
@when("I create an LLM agent")
def step_create_llm_agent(context):
"""Create LLM agent."""
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
context.llm_config["name"], context.llm_config, context.template_renderer
)
@when("I create an LLM agent with OpenAI provider")
def step_create_openai_agent(context):
"""Create LLM agent with OpenAI provider."""
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
context.llm_config["name"], context.llm_config, context.template_renderer
)
@when("I create an LLM agent with Anthropic provider")
def step_create_anthropic_agent(context):
"""Create LLM agent with Anthropic provider."""
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
context.llm_config["name"], context.llm_config, context.template_renderer
)
@when("I create an LLM agent with Google provider")
def step_create_google_agent(context):
"""Create LLM agent with Google provider.
With lazy initialisation the LangChain client is not created during
``__init__``. We patch ``ChatGoogleGenerativeAI`` and then immediately
trigger the lazy init (``_ensure_chat_model``) while the patch is active,
so the agent holds a mock model for subsequent assertions.
"""
context.template_renderer = Mock(spec=TemplateRenderer)
context.template_renderer.render_string.return_value = "mocked system prompt"
context.llm_agent = LLMAgent(
context.llm_config["name"], context.llm_config, context.template_renderer
)
# Trigger lazy init within the patch scope so the Google class is available
mock_google = Mock()
mock_google.return_value = create_mock_chat_model("Mock Google response")
with patch("cleveractors.agents.llm.ChatGoogleGenerativeAI", mock_google):
context.llm_agent._ensure_chat_model()
@when("I process a simple message without template")
def step_process_simple_message(context):
"""Process a simple message without template."""
context.test_message = "Hello, world!"
expected_response = "Hello! How can I help you?"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
@when("I process a message with template variables")
def step_process_message_with_template(context):
"""Process a message with template variables."""
context.test_message = "Process this: {variable}"
context.test_context = {"variable": "test_value"}
expected_response = "Processed response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message, context.test_context)
)
@when("I process a message and OpenAI API returns success")
def step_process_message_openai_success(context):
"""Process message with successful OpenAI API response."""
context.test_message = "Test message"
context.expected_response = "OpenAI response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(context.expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
@when("I process a message and OpenAI API returns error")
def step_process_message_openai_error(context):
"""Process message with OpenAI API error."""
context.test_message = "Test message"
# Disable retry — test expects fast failure on first ainvoke error
context.llm_agent._max_retries = 0
# Replace the chat model with a mock that raises an error
context.llm_agent.chat_model = create_mock_chat_model(error_message="API Error")
try:
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
context.exception = None
except Exception as e:
context.exception = e
@when("I process a message and Anthropic API returns success")
def step_process_message_anthropic_success(context):
"""Process message with successful Anthropic API response."""
context.test_message = "Test message"
context.expected_response = "Anthropic response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(context.expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
@when("I process a message and Anthropic API returns error")
def step_process_message_anthropic_error(context):
"""Process message with Anthropic API error."""
context.test_message = "Test message"
# Disable retry — test expects fast failure on first ainvoke error
context.llm_agent._max_retries = 0
# Replace the chat model with a mock that raises an error
context.llm_agent.chat_model = create_mock_chat_model(error_message="Unauthorized")
try:
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
context.exception = None
except Exception as e:
context.exception = e
@when("I process a message and Google API returns success")
def step_process_message_google_success(context):
"""Process message with successful Google API response."""
context.test_message = "Test message"
context.expected_response = "Google response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(context.expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
@when("I process a message and Google API returns error")
def step_process_message_google_error(context):
"""Process message with Google API error."""
context.test_message = "Test message"
# Disable retry — test expects fast failure on first ainvoke error
context.llm_agent._max_retries = 0
# Replace the chat model with a mock that raises an error
context.llm_agent.chat_model = create_mock_chat_model(error_message="Forbidden")
try:
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
context.exception = None
except Exception as e:
context.exception = e
@when("I process a message successfully")
def step_process_message_successfully(context):
"""Process a message successfully."""
context.test_message = "Test message"
context.expected_response = "Test response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(context.expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
@when("I process a message")
def step_process_message(context):
"""Process a message."""
context.test_message = "Test message"
expected_response = "Response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
@when("I request the agent capabilities")
def step_request_capabilities(context):
"""Request agent capabilities."""
context.capabilities = context.llm_agent.get_capabilities()
@when("I request the agent metadata")
def step_request_metadata(context):
"""Request agent metadata."""
context.metadata = context.llm_agent.get_metadata()
@when("an exception occurs during message processing")
def step_exception_during_processing(context):
"""Simulate exception during message processing."""
context.test_message = "Test message"
# Disable retry — test expects fast failure on first ainvoke error
context.llm_agent._max_retries = 0
# Replace the chat model with a mock that raises an error
context.llm_agent.chat_model = create_mock_chat_model(
error_message="Test exception"
)
try:
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
context.exception = None
except Exception as e:
context.exception = e
@when("I process a message with context parameter")
def step_process_message_with_context(context):
"""Process message with context parameter."""
context.test_message = "Test message"
context.test_context = {"key": "value"}
expected_response = "Response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message, context.test_context)
)
@when("I process a message with template_vars in config")
def step_process_message_with_template_vars(context):
"""Process message with template_vars in config."""
context.test_message = "Test message with {global_var}"
context.test_context = {"local_var": "local_value"}
expected_response = "Response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message, context.test_context)
)
@when("I process a new message")
def step_process_new_message(context):
"""Process a new message."""
context.test_message = "New message"
context.expected_response = "New response"
# Replace the chat model with a mock
context.llm_agent.chat_model = create_mock_chat_model(context.expected_response)
context.result = asyncio.run(
context.llm_agent.process_message(context.test_message)
)
@then("the LLM agent should be initialized successfully")
def step_llm_agent_initialized(context):
"""Verify LLM agent is initialized successfully."""
assert context.llm_agent is not None
assert context.llm_agent.name == context.llm_config["name"]
@then('the provider should be "{expected_provider}"')
def step_provider_should_be(context, expected_provider):
"""Verify provider setting."""
assert context.llm_agent.provider == expected_provider
@then('the model should be "{expected_model}"')
def step_model_should_be(context, expected_model):
"""Verify model setting."""
assert context.llm_agent.model == expected_model
@then("the temperature should be {expected_temp:f}")
def step_temperature_should_be(context, expected_temp):
"""Verify temperature setting."""
assert context.llm_agent.temperature == expected_temp
@then("the max_tokens should be {expected_tokens:d}")
def step_max_tokens_should_be(context, expected_tokens):
"""Verify max_tokens setting."""
assert context.llm_agent.max_tokens == expected_tokens
@then('the system message should be "{expected_message}"')
def step_system_message_should_be(context, expected_message):
"""Verify system message setting."""
assert context.llm_agent.system_message == expected_message
@then("the custom configuration should be applied")
def step_custom_config_applied(context):
"""Verify custom configuration is applied."""
assert context.llm_agent.provider == context.llm_config["provider"]
assert context.llm_agent.model == context.llm_config["model"]
assert context.llm_agent.temperature == context.llm_config["temperature"]
assert context.llm_agent.max_tokens == context.llm_config["max_tokens"]
assert context.llm_agent.system_message == context.llm_config["system_prompt"]
@then("an LLM ConfigurationError should be raised")
def step_configuration_error_raised(context):
"""Verify ConfigurationError is raised."""
assert context.exception is not None
assert isinstance(context.exception, ConfigurationError)
@then("the error should mention missing API key")
def step_error_mentions_missing_key(context):
"""Verify error mentions missing API key."""
# LangChain models handle API keys internally, so we just check for any configuration error
assert context.exception is not None
@then("the error should mention unsupported provider")
def step_error_mentions_unsupported_provider(context):
"""Verify error mentions unsupported provider."""
assert "Unsupported provider" in str(context.exception)
@then("the API key should be resolved from configuration")
def step_api_key_from_config(context):
"""Verify API key is resolved from configuration."""
# With LangChain, API keys are handled internally
assert context.llm_agent.chat_model is not None
@then("the API key should be resolved from environment")
def step_api_key_from_environment(context):
"""Verify API key is resolved from environment."""
if hasattr(context, "env_patch"):
context.env_patch.stop()
# With LangChain, API keys are handled internally
assert context.llm_agent.chat_model is not None
@then("the OpenAI configuration should be set up correctly")
def step_openai_config_setup(context):
"""Verify OpenAI configuration setup."""
assert context.llm_agent.chat_model is not None
assert context.llm_agent.provider == "openai"
@then('the base URL should be "{expected_url}"')
def step_base_url_should_be(context, expected_url):
"""Verify base URL."""
# LangChain handles URLs internally
assert context.llm_agent.chat_model is not None
@then("the headers should contain authorization bearer token")
def step_headers_contain_bearer_token(context):
"""Verify headers contain bearer token."""
# LangChain handles authorization internally
assert context.llm_agent.chat_model is not None
@then("the Anthropic configuration should be set up correctly")
def step_anthropic_config_setup(context):
"""Verify Anthropic configuration setup."""
assert context.llm_agent.chat_model is not None
assert context.llm_agent.provider == "anthropic"
@then("the headers should contain x-api-key")
def step_headers_contain_x_api_key(context):
"""Verify headers contain x-api-key."""
# LangChain handles API keys internally
assert context.llm_agent.chat_model is not None
@then("the Google configuration should be set up correctly")
def step_google_config_setup(context):
"""Verify Google configuration setup."""
assert context.llm_agent.chat_model is not None
assert context.llm_agent.provider == "google"
@then("the base URL should contain the model name")
def step_base_url_contains_model(context):
"""Verify base URL contains model name."""
# LangChain handles URLs internally
assert context.llm_agent.model is not None
@then("the API key should be in the URL as query parameter")
def step_api_key_in_url(context):
"""Verify API key is in URL as query parameter."""
# LangChain handles API keys internally
assert context.llm_agent.chat_model is not None
@then("the message should be processed successfully")
def step_message_processed_successfully(context):
"""Verify message was processed successfully."""
assert context.result is not None
assert isinstance(context.result, str)
@then("the response should be returned")
def step_response_returned(context):
"""Verify response is returned."""
assert context.result is not None
@then("the template should be rendered with variables")
def step_template_rendered_with_variables(context):
"""Verify template is rendered with variables."""
context.template_renderer.render.assert_called_once()
call_args = context.template_renderer.render.call_args
assert call_args[0][0] == "test_template" # template name
template_vars = call_args[0][1] # variables
assert "message" in template_vars
assert "context" in template_vars
@then("the processed message should be used for LLM call")
def step_processed_message_used(context):
"""Verify processed message is used for LLM call."""
# The rendered message should be used in the API call
assert context.result is not None
@then("the OpenAI API should be called with correct payload")
def step_openai_api_called_correctly(context):
"""Verify OpenAI API is called with correct payload."""
# LangChain handles the API call internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
assert len(call_args) > 0 # Should have messages
@then("the response should be extracted from choices")
def step_response_extracted_from_choices(context):
"""Verify response is extracted from choices."""
assert context.result == context.expected_response
@then("the result should be returned")
def step_result_returned(context):
"""Verify result is returned."""
assert context.result is not None
@then("an LLM ExecutionError should be raised")
def step_execution_error_raised(context):
"""Verify ExecutionError is raised."""
assert context.exception is not None
assert isinstance(context.exception, ExecutionError)
@then("the error should contain API error details")
def step_error_contains_api_details(context):
"""Verify error contains API error details."""
error_message = str(context.exception)
# LangChain errors may come in different formats, just check that we have an error
assert context.exception is not None
@then("the Anthropic API should be called with correct payload")
def step_anthropic_api_called_correctly(context):
"""Verify Anthropic API is called with correct payload."""
# LangChain handles the API call internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
assert len(call_args) > 0 # Should have messages
@then("the response should be extracted from content")
def step_response_extracted_from_content(context):
"""Verify response is extracted from content."""
assert context.result == context.expected_response
@then("the Google API should be called with correct payload")
def step_google_api_called_correctly(context):
"""Verify Google API is called with correct payload."""
# LangChain handles the API call internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
assert len(call_args) > 0 # Should have messages
@then("the response should be extracted from candidates")
def step_response_extracted_from_candidates(context):
"""Verify response is extracted from candidates."""
assert context.result == context.expected_response
@then("the last message should be stored in memory")
def step_last_message_stored(context):
"""Verify last message is stored in memory."""
context.llm_agent.update_memory.assert_any_call(
"last_message", context.test_message
)
@then("the last response should be stored in memory")
def step_last_response_stored(context):
"""Verify last response is stored in memory."""
context.llm_agent.update_memory.assert_any_call(
"last_response", context.expected_response
)
@then("no memory updates should occur")
def step_no_memory_updates(context):
"""Verify no memory updates occur."""
context.llm_agent.update_memory.assert_not_called()
@then("the conversation history should be included in API call")
def step_conversation_history_included(context):
"""Verify conversation history is included in API call."""
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
# Should have system + history + current message
assert len(call_args) > 2
@then("the new message should be added to history")
def step_new_message_added_to_history(context):
"""Verify new message is added to history."""
# Check if update_memory was called with conversation_history
update_calls = [
call
for call in context.llm_agent.update_memory.call_args_list
if call[0][0] == "conversation_history"
]
assert len(update_calls) > 0
@then("the response should be added to history")
def step_response_added_to_history(context):
"""Verify response is added to history."""
# This is checked as part of the conversation_history update
update_calls = [
call
for call in context.llm_agent.update_memory.call_args_list
if call[0][0] == "conversation_history"
]
assert len(update_calls) > 0
@then("history should be limited to max_history setting")
def step_history_limited(context):
"""Verify history is limited to max_history setting."""
# This is handled internally in the agent
assert True # The agent handles this internally
@then("only system message and current user message should be sent")
def step_only_system_and_current_message(context):
"""Verify only system and current user message are sent."""
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
assert len(call_args) == 2 # system + user message
@then("no history should be included")
def step_no_history_included(context):
"""Verify no history is included."""
# Already verified by checking message count
assert True
@then("the capabilities should include text-generation")
def step_capabilities_include_text_generation(context):
"""Verify capabilities include text-generation."""
assert "text-generation" in context.capabilities
@then("the capabilities should include conversation")
def step_capabilities_include_conversation(context):
"""Verify capabilities include conversation."""
assert "conversation" in context.capabilities
@then("the capabilities should include reasoning")
def step_capabilities_include_reasoning(context):
"""Verify capabilities include reasoning."""
assert "reasoning" in context.capabilities
@then("the capabilities should include analysis")
def step_capabilities_include_analysis(context):
"""Verify capabilities include analysis."""
assert "analysis" in context.capabilities
@then("the capabilities should include creative-writing")
def step_capabilities_include_creative_writing(context):
"""Verify capabilities include creative-writing."""
assert "creative-writing" in context.capabilities
@then("the metadata should include base agent metadata")
def step_metadata_includes_base(context):
"""Verify metadata includes base agent metadata."""
# Base metadata is merged in get_metadata
assert isinstance(context.metadata, dict)
@then("the metadata should include provider information")
def step_metadata_includes_provider(context):
"""Verify metadata includes provider information."""
assert "provider" in context.metadata
assert context.metadata["provider"] == context.llm_agent.provider
@then("the metadata should include model information")
def step_metadata_includes_model(context):
"""Verify metadata includes model information."""
assert "model" in context.metadata
assert context.metadata["model"] == context.llm_agent.model
@then("the metadata should include temperature setting")
def step_metadata_includes_temperature(context):
"""Verify metadata includes temperature setting."""
assert "temperature" in context.metadata
assert context.metadata["temperature"] == context.llm_agent.temperature
@then("the metadata should include max_tokens setting")
def step_metadata_includes_max_tokens(context):
"""Verify metadata includes max_tokens setting."""
assert "max_tokens" in context.metadata
assert context.metadata["max_tokens"] == context.llm_agent.max_tokens
@then("the metadata should include memory_enabled setting")
def step_metadata_includes_memory_enabled(context):
"""Verify metadata includes memory_enabled setting."""
assert "memory_enabled" in context.metadata
@then("the LLM error should be logged")
def step_error_logged(context):
"""Verify error is logged."""
# Error logging is handled internally
assert True
@then("the original error should be wrapped")
def step_original_error_wrapped(context):
"""Verify original error is wrapped."""
assert isinstance(context.exception, ExecutionError)
@then("the context should be available for template rendering")
def step_context_available_for_template(context):
"""Verify context is available for template rendering."""
# Context is passed to template rendering
assert True
@then("the context should be passed to API calls")
def step_context_passed_to_api(context):
"""Verify context is passed to API calls."""
# Context is used in API calls
assert True
@then("the template_vars should be merged with message and context")
def step_template_vars_merged(context):
"""Verify template_vars are merged."""
# Template vars are merged in template rendering
assert True
@then("all variables should be available for template rendering")
def step_all_variables_available(context):
"""Verify all variables are available for template rendering."""
# All variables are passed to template rendering
assert True
@then("the messages array should have system message first")
def step_messages_have_system_first(context):
"""Verify messages array has system message first."""
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
# Check first message is SystemMessage
from langchain_core.messages import SystemMessage
assert isinstance(call_args[0], SystemMessage)
@then("the messages array should have user message last")
def step_messages_have_user_last(context):
"""Verify messages array has user message last."""
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
# Check last message is HumanMessage
from langchain_core.messages import HumanMessage
assert isinstance(call_args[-1], HumanMessage)
@then("the payload should have required OpenAI fields")
def step_payload_has_openai_fields(context):
"""Verify payload has required OpenAI fields."""
# LangChain handles payload construction internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
assert context.llm_agent.model is not None
assert context.llm_agent.temperature is not None
@then("the payload should have system field separate")
def step_payload_has_system_separate(context):
"""Verify payload has system field separate."""
# LangChain handles system messages internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
assert context.llm_agent.system_message is not None
@then("the messages array should only contain user message")
def step_messages_only_user(context):
"""Verify messages array only contains user message."""
# For Anthropic, system is separate, so messages contain only user message
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
# Count non-system messages
from langchain_core.messages import SystemMessage
non_system_messages = [
msg for msg in call_args if not isinstance(msg, SystemMessage)
]
assert len(non_system_messages) >= 1
@then("the payload should have required Anthropic fields")
def step_payload_has_anthropic_fields(context):
"""Verify payload has required Anthropic fields."""
# LangChain handles payload construction internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
assert context.llm_agent.model is not None
assert context.llm_agent.temperature is not None
assert context.llm_agent.system_message is not None
@then("the contents should have parts with combined system and user text")
def step_contents_have_combined_text(context):
"""Verify contents have parts with combined system and user text."""
# LangChain handles Google's content structure internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
call_args = context.llm_agent.chat_model.ainvoke.call_args[0][0]
assert len(call_args) > 0
@then("the generationConfig should have temperature and maxOutputTokens")
def step_generation_config_has_temp_tokens(context):
"""Verify generationConfig has temperature and maxOutputTokens."""
# LangChain handles generation config internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
assert context.llm_agent.temperature is not None
assert context.llm_agent.max_tokens is not None
@then("the payload should have required Google fields")
def step_payload_has_google_fields(context):
"""Verify payload has required Google fields."""
# LangChain handles payload construction internally
if hasattr(context.llm_agent.chat_model, "ainvoke"):
context.llm_agent.chat_model.ainvoke.assert_called_once()
assert context.llm_agent.model is not None
assert context.llm_agent.temperature is not None
@then("the oldest messages should be removed")
def step_oldest_messages_removed(context):
"""Verify oldest messages are removed."""
# This is handled internally by the agent
assert True
@then("the history length should not exceed max_history")
def step_history_length_not_exceed_max(context):
"""Verify history length doesn't exceed max_history."""
# This is handled internally by the agent
assert True
@then("the newest messages should be preserved")
def step_newest_messages_preserved(context):
"""Verify newest messages are preserved."""
# This is handled internally by the agent
assert True
# Enhanced Conversation History Steps
@given("I have conversation history in context:")
def step_conversation_history_in_context(context):
"""Set up conversation history in context."""
import json
context.context_history = json.loads(context.text.strip())
@given("I have different conversation history in agent memory:")
def step_different_conversation_history_in_memory(context):
"""Set up different conversation history in agent memory."""
import json
context.memory_history = json.loads(context.text.strip())
@given("I have conversation history in agent memory:")
def step_conversation_history_in_memory(context):
"""Set up conversation history in agent memory."""
import json
context.memory_history = json.loads(context.text.strip())
@given("I have no conversation history in context")
def step_no_conversation_history_in_context(context):
"""Set up no conversation history in context."""
context.context_history = None
@given("I have empty conversation history in context")
def step_empty_conversation_history_in_context(context):
"""Set up empty conversation history in context."""
context.context_history = []
@when("I process a message with context history")
@async_run_until_complete
async def step_process_message_with_context_history(context):
"""Process a message with context history."""
from unittest.mock import AsyncMock, patch
# Mock the chat model
with patch("cleveractors.agents.llm.build_chat_model") as mock_create_model:
mock_model = AsyncMock()
mock_model.ainvoke.return_value = AsyncMock(content="Test response")
mock_create_model.return_value = mock_model
# Create agent with memory enabled
context.agent = context.llm_agent_class(
name="test_agent",
config={"memory_enabled": True},
template_renderer=context.template_renderer,
)
# Set up memory with different history
if hasattr(context, "memory_history"):
context.agent.memory["conversation_history"] = context.memory_history
# Process message with context history
context.context = {"conversation_history": context.context_history}
context.result = await context.agent.process_message(
"Test message", context.context
)
@when("I process a message without context history")
@async_run_until_complete
async def step_process_message_without_context_history(context):
"""Process a message without context history."""
from unittest.mock import AsyncMock, patch
# Mock the chat model
with patch("cleveractors.agents.llm.build_chat_model") as mock_create_model:
mock_model = AsyncMock()
mock_model.ainvoke.return_value = AsyncMock(content="Test response")
mock_create_model.return_value = mock_model
# Create agent with memory enabled
context.agent = context.llm_agent_class(
name="test_agent",
config={"memory_enabled": True},
template_renderer=context.template_renderer,
)
# Set up memory with history
if hasattr(context, "memory_history"):
context.agent.memory["conversation_history"] = context.memory_history
# Process message without context history
context.context = {}
context.result = await context.agent.process_message(
"Test message", context.context
)
@when("I process a message with empty context history")
@async_run_until_complete
async def step_process_message_with_empty_context_history(context):
"""Process a message with empty context history."""
from unittest.mock import AsyncMock, patch
# Mock the chat model
with patch("cleveractors.agents.llm.build_chat_model") as mock_create_model:
mock_model = AsyncMock()
mock_model.ainvoke.return_value = AsyncMock(content="Test response")
mock_create_model.return_value = mock_model
# Create agent with memory enabled
context.agent = context.llm_agent_class(
name="test_agent",
config={"memory_enabled": True},
template_renderer=context.template_renderer,
)
# Process message with empty context history
context.context = {"conversation_history": context.context_history}
context.result = await context.agent.process_message(
"Test message", context.context
)
@then("the agent should use context history instead of memory history")
def step_agent_uses_context_history(context):
"""Verify agent uses context history instead of memory history."""
# This is verified by checking that the context history was used
# The actual verification would be in the mock call to ainvoke
assert context.context_history is not None
@then("the context history should be included in the request")
def step_context_history_included_in_request(context):
"""Verify context history is included in the request."""
# This would be verified by checking the mock call
assert context.context_history is not None
@then("the agent should use memory history")
def step_agent_uses_memory_history(context):
"""Verify agent uses memory history."""
# This is verified by checking that memory history was used
assert hasattr(context, "memory_history") and context.memory_history is not None
@then("the memory history should be included in the request")
def step_memory_history_included_in_request(context):
"""Verify memory history is included in the request."""
# This would be verified by checking the mock call
assert hasattr(context, "memory_history") and context.memory_history is not None
@then("the agent should handle empty history gracefully")
def step_agent_handles_empty_history_gracefully(context):
"""Verify agent handles empty history gracefully."""
# The agent should not crash with empty history
assert context.result is not None
@then("no conversation history should be included in the request")
def step_no_conversation_history_in_request(context):
"""Verify no conversation history is included in the request."""
# This would be verified by checking the mock call
assert context.context_history == []