forked from cleveragents/cleveragents-core
165 lines
8.0 KiB
Plaintext
165 lines
8.0 KiB
Plaintext
*** Settings ***
|
|
Resource common.resource
|
|
Library Process
|
|
Library OperatingSystem
|
|
Library Collections
|
|
Suite Setup Setup Test Environment
|
|
Suite Teardown Cleanup Test Environment
|
|
|
|
*** Variables ***
|
|
${PYTHON} python
|
|
${SRC_DIR} ${CURDIR}/..
|
|
|
|
*** Test Cases ***
|
|
OpenAI Provider Generates Changes
|
|
${script}= Catenate SEPARATOR=\n
|
|
... import sys
|
|
... from unittest.mock import MagicMock, patch
|
|
... sys.path.insert(0, '${SRC_DIR}/src')
|
|
... from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
|
|
... project = MagicMock(name='Project')
|
|
... plan = MagicMock(name='Plan')
|
|
... plan.prompt = 'Add logging'
|
|
... contexts = [MagicMock(name='Context')]
|
|
... contexts[0].content = 'ctx'
|
|
... chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
|
|
... graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
|
|
... chat_cls = chat_patcher.start()
|
|
... graph_cls = graph_patcher.start()
|
|
... chat_instance = MagicMock(name='ChatOpenAIInstance')
|
|
... chat_instance.get_num_tokens.return_value = 42
|
|
... chat_cls.return_value = chat_instance
|
|
... change = {'plan_id': 1, 'file_path': 'robot.py', 'operation': 'modify', 'new_content': '# code'}
|
|
... graph_instance = MagicMock(name='GraphInstance')
|
|
... graph_instance.invoke.return_value = {'generated_changes': [change], 'validation_result': {'status': 'PASS'}, 'error': None}
|
|
... graph_cls.return_value = graph_instance
|
|
... provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
|
|
... response = provider.generate_changes(project, plan, contexts)
|
|
... chat_patcher.stop()
|
|
... graph_patcher.stop()
|
|
... assert len(response.changes) == 1
|
|
... assert response.changes[0].file_path == 'robot.py'
|
|
... assert response.token_count == 42
|
|
... print('provider-success')
|
|
${result}= Run Process ${PYTHON} -c ${script}
|
|
Log Process Failure ${result}
|
|
|
|
Should Be Equal As Integers ${result.rc} 0
|
|
Should Contain ${result.stdout} provider-success
|
|
|
|
OpenAI Provider Streams Events
|
|
${script}= Catenate SEPARATOR=\n
|
|
... import sys
|
|
... from unittest.mock import MagicMock, patch
|
|
... sys.path.insert(0, '${SRC_DIR}/src')
|
|
... from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
|
|
... project = MagicMock(name='Project')
|
|
... plan = MagicMock(name='Plan')
|
|
... plan.prompt = 'Stream plan'
|
|
... contexts = [MagicMock(name='Context')]
|
|
... contexts[0].content = 'ctx'
|
|
... chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
|
|
... graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
|
|
... chat_cls = chat_patcher.start()
|
|
... graph_cls = graph_patcher.start()
|
|
... chat_instance = MagicMock(name='ChatOpenAIInstance')
|
|
... chat_instance.get_num_tokens.return_value = 21
|
|
... chat_cls.return_value = chat_instance
|
|
... graph_instance = MagicMock(name='GraphInstance')
|
|
... events = [
|
|
... {'load_context': {'status': 'ok'}},
|
|
... {'generate_plan': {'generated_changes': [{'plan_id': 1, 'file_path': 'stream.py', 'operation': 'modify', 'new_content': '# stream'}]}},
|
|
... {'validate': {'validation_result': {'status': 'PASS'}}},
|
|
... ]
|
|
... graph_instance.stream.side_effect = lambda *_args, **_kwargs: iter(events)
|
|
... graph_instance.invoke.return_value = {'generated_changes': [], 'validation_result': {'status': 'PASS'}, 'error': None}
|
|
... graph_cls.return_value = graph_instance
|
|
... provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
|
|
... streamed = list(provider.stream_changes(project, plan, contexts))
|
|
... chat_patcher.stop()
|
|
... graph_patcher.stop()
|
|
... assert streamed[-1]['__end__']['response'].model_used == 'gpt-4o-mini'
|
|
... assert len(streamed[-1]['__end__']['response'].changes) == 1
|
|
... print('stream-success')
|
|
${result}= Run Process ${PYTHON} -c ${script}
|
|
Log Process Failure ${result}
|
|
|
|
Should Be Equal As Integers ${result.rc} 0
|
|
Should Contain ${result.stdout} stream-success
|
|
|
|
OpenAI Provider Surfaces Graph Errors
|
|
${script}= Catenate SEPARATOR=\n
|
|
... import sys
|
|
... from unittest.mock import MagicMock, patch
|
|
... sys.path.insert(0, '${SRC_DIR}/src')
|
|
... from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
|
|
... project = MagicMock(name='Project')
|
|
... plan = MagicMock(name='Plan')
|
|
... plan.prompt = 'Error plan'
|
|
... contexts = [MagicMock(name='Context')]
|
|
... contexts[0].content = 'ctx'
|
|
... chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
|
|
... graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
|
|
... chat_cls = chat_patcher.start()
|
|
... graph_cls = graph_patcher.start()
|
|
... chat_instance = MagicMock(name='ChatOpenAIInstance')
|
|
... chat_instance.get_num_tokens.return_value = 0
|
|
... chat_cls.return_value = chat_instance
|
|
... graph_instance = MagicMock(name='GraphInstance')
|
|
... graph_instance.invoke.side_effect = ValueError('graph exploded')
|
|
... graph_cls.return_value = graph_instance
|
|
... provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
|
|
... response = provider.generate_changes(project, plan, contexts)
|
|
... chat_patcher.stop()
|
|
... graph_patcher.stop()
|
|
... assert response.error_message == 'graph exploded'
|
|
... assert response.changes == []
|
|
... print('provider-error')
|
|
${result}= Run Process ${PYTHON} -c ${script}
|
|
Log Process Failure ${result}
|
|
|
|
Should Be Equal As Integers ${result.rc} 0
|
|
Should Contain ${result.stdout} provider-error
|
|
|
|
OpenAI Provider Reports Runtime Error
|
|
${script}= Catenate SEPARATOR=\n
|
|
... import sys
|
|
... from unittest.mock import MagicMock, patch
|
|
... sys.path.insert(0, '${SRC_DIR}/src')
|
|
... from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
|
|
... project = MagicMock(name='Project')
|
|
... plan = MagicMock(name='Plan')
|
|
... plan.prompt = 'Retry plan'
|
|
... contexts = [MagicMock(name='Context')]
|
|
... contexts[0].content = 'ctx'
|
|
... chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
|
|
... graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
|
|
... chat_cls = chat_patcher.start()
|
|
... graph_cls = graph_patcher.start()
|
|
... chat_instance = MagicMock(name='ChatOpenAIInstance')
|
|
... chat_instance.get_num_tokens.return_value = 0
|
|
... chat_cls.return_value = chat_instance
|
|
... graph_instance = MagicMock(name='GraphInstance')
|
|
... graph_instance.invoke.side_effect = RuntimeError('deadline reached')
|
|
... graph_cls.return_value = graph_instance
|
|
... provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
|
|
... response = provider.generate_changes(project, plan, contexts)
|
|
... chat_patcher.stop()
|
|
... graph_patcher.stop()
|
|
... assert response.error_message == 'deadline reached'
|
|
... assert response.changes == []
|
|
... print('provider-runtime-error')
|
|
${result}= Run Process ${PYTHON} -c ${script}
|
|
Log Process Failure ${result}
|
|
|
|
Should Be Equal As Integers ${result.rc} 0
|
|
Should Contain ${result.stdout} provider-runtime-error
|
|
|
|
*** Keywords ***
|
|
Log Process Failure
|
|
[Arguments] ${result}
|
|
Run Keyword If ${result.rc} == 0 Return From Keyword
|
|
Log To Console Process failed with rc=${result.rc}
|
|
Log To Console STDOUT:${\n}${result.stdout}
|
|
Log To Console STDERR:${\n}${result.stderr}
|