cleveragents-core/robot/openai_provider.robot

*** Settings ***
Resource          ${CURDIR}/common.resource
Library           Process
Library           OperatingSystem
Library           Collections
Suite Setup       Setup Test Environment
Suite Teardown    Cleanup Test Environment

*** Variables ***
${SRC_DIR}        ${CURDIR}/..

*** Test Cases ***
OpenAI Provider Generates Changes
    ${script}=    Catenate    SEPARATOR=\n
    ...    import sys
    ...    from unittest.mock import MagicMock, patch
    ...    sys.path.insert(0, '${SRC_DIR}/src')
    ...    from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
    ...    project = MagicMock(name='Project')
    ...    plan = MagicMock(name='Plan')
    ...    plan.prompt = 'Add logging'
    ...    contexts = [MagicMock(name='Context')]
    ...    contexts[0].content = 'ctx'
    ...    chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
    ...    graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
    ...    chat_cls = chat_patcher.start()
    ...    graph_cls = graph_patcher.start()
    ...    chat_instance = MagicMock(name='ChatOpenAIInstance')
    ...    chat_instance.get_num_tokens.return_value = 42
    ...    chat_cls.return_value = chat_instance
    ...    change = {'plan_id': 1, 'file_path': 'robot.py', 'operation': 'modify', 'new_content': '# code'}
    ...    graph_instance = MagicMock(name='GraphInstance')
    ...    graph_instance.invoke.return_value = {'generated_changes': [change], 'validation_result': {'status': 'PASS'}, 'error': None}
    ...    graph_cls.return_value = graph_instance
    ...    provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
    ...    response = provider.generate_changes(project, plan, contexts)
    ...    chat_patcher.stop()
    ...    graph_patcher.stop()
    ...    assert len(response.changes) == 1
    ...    assert response.changes[0].file_path == 'robot.py'
    ...    assert response.token_count == 42
    ...    print('provider-success')
     ${result}=    Run Process    ${PYTHON}    -c    ${script}
     Log Process Failure    ${result}

    Should Be Equal As Integers    ${result.rc}    0
    Should Contain    ${result.stdout}    provider-success

OpenAI Provider Streams Events
    ${script}=    Catenate    SEPARATOR=\n
    ...    import sys
    ...    from unittest.mock import MagicMock, patch
    ...    sys.path.insert(0, '${SRC_DIR}/src')
    ...    from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
    ...    project = MagicMock(name='Project')
    ...    plan = MagicMock(name='Plan')
    ...    plan.prompt = 'Stream plan'
    ...    contexts = [MagicMock(name='Context')]
    ...    contexts[0].content = 'ctx'
    ...    chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
    ...    graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
    ...    chat_cls = chat_patcher.start()
    ...    graph_cls = graph_patcher.start()
    ...    chat_instance = MagicMock(name='ChatOpenAIInstance')
    ...    chat_instance.get_num_tokens.return_value = 21
    ...    chat_cls.return_value = chat_instance
    ...    graph_instance = MagicMock(name='GraphInstance')
    ...    events = [
    ...        {'load_context': {'status': 'ok'}},
    ...        {'generate_plan': {'generated_changes': [{'plan_id': 1, 'file_path': 'stream.py', 'operation': 'modify', 'new_content': '# stream'}]}},
    ...        {'validate': {'validation_result': {'status': 'PASS'}}},
    ...    ]
    ...    graph_instance.stream.side_effect = lambda *_args, **_kwargs: iter(events)
    ...    graph_instance.invoke.return_value = {'generated_changes': [], 'validation_result': {'status': 'PASS'}, 'error': None}
    ...    graph_cls.return_value = graph_instance
    ...    provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
    ...    streamed = list(provider.stream_changes(project, plan, contexts))
    ...    chat_patcher.stop()
    ...    graph_patcher.stop()
    ...    assert streamed[-1]['__end__']['response'].model_used == 'gpt-4o-mini'
    ...    assert len(streamed[-1]['__end__']['response'].changes) == 1
    ...    print('stream-success')
     ${result}=    Run Process    ${PYTHON}    -c    ${script}
     Log Process Failure    ${result}

    Should Be Equal As Integers    ${result.rc}    0
    Should Contain    ${result.stdout}    stream-success

OpenAI Provider Surfaces Graph Errors
    ${script}=    Catenate    SEPARATOR=\n
    ...    import sys
    ...    from unittest.mock import MagicMock, patch
    ...    sys.path.insert(0, '${SRC_DIR}/src')
    ...    from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
    ...    project = MagicMock(name='Project')
    ...    plan = MagicMock(name='Plan')
    ...    plan.prompt = 'Error plan'
    ...    contexts = [MagicMock(name='Context')]
    ...    contexts[0].content = 'ctx'
    ...    chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
    ...    graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
    ...    chat_cls = chat_patcher.start()
    ...    graph_cls = graph_patcher.start()
    ...    chat_instance = MagicMock(name='ChatOpenAIInstance')
    ...    chat_instance.get_num_tokens.return_value = 0
    ...    chat_cls.return_value = chat_instance
    ...    graph_instance = MagicMock(name='GraphInstance')
    ...    graph_instance.invoke.side_effect = ValueError('graph exploded')
    ...    graph_cls.return_value = graph_instance
    ...    provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
    ...    response = provider.generate_changes(project, plan, contexts)
    ...    chat_patcher.stop()
    ...    graph_patcher.stop()
    ...    assert response.error_message == 'graph exploded'
    ...    assert response.changes == []
    ...    print('provider-error')
     ${result}=    Run Process    ${PYTHON}    -c    ${script}
     Log Process Failure    ${result}

    Should Be Equal As Integers    ${result.rc}    0
    Should Contain    ${result.stdout}    provider-error

OpenAI Provider Reports Runtime Error
    ${script}=    Catenate    SEPARATOR=\n
    ...    import sys
    ...    from unittest.mock import MagicMock, patch
    ...    sys.path.insert(0, '${SRC_DIR}/src')
    ...    from cleveragents.providers.llm.openai_provider import OpenAIChatProvider
    ...    project = MagicMock(name='Project')
    ...    plan = MagicMock(name='Plan')
    ...    plan.prompt = 'Retry plan'
    ...    contexts = [MagicMock(name='Context')]
    ...    contexts[0].content = 'ctx'
    ...    chat_patcher = patch('cleveragents.providers.llm.openai_provider.ChatOpenAI')
    ...    graph_patcher = patch('cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph')
    ...    chat_cls = chat_patcher.start()
    ...    graph_cls = graph_patcher.start()
    ...    chat_instance = MagicMock(name='ChatOpenAIInstance')
    ...    chat_instance.get_num_tokens.return_value = 0
    ...    chat_cls.return_value = chat_instance
    ...    graph_instance = MagicMock(name='GraphInstance')
    ...    graph_instance.invoke.side_effect = RuntimeError('deadline reached')
    ...    graph_cls.return_value = graph_instance
    ...    provider = OpenAIChatProvider(api_key='sk-robot', model='gpt-4o-mini')
    ...    response = provider.generate_changes(project, plan, contexts)
    ...    chat_patcher.stop()
    ...    graph_patcher.stop()
    ...    assert response.error_message == 'deadline reached'
    ...    assert response.changes == []
    ...    print('provider-runtime-error')
     ${result}=    Run Process    ${PYTHON}    -c    ${script}
     Log Process Failure    ${result}

    Should Be Equal As Integers    ${result.rc}    0
     Should Contain    ${result.stdout}    provider-runtime-error

*** Keywords ***
Log Process Failure
    [Arguments]    ${result}
    Run Keyword If    ${result.rc} == 0    Return From Keyword
    Log To Console    Process failed with rc=${result.rc}
    Log To Console    STDOUT:${\n}${result.stdout}
    Log To Console    STDERR:${\n}${result.stderr}