cleveragents-core/robot/e2e/common_e2e.resource

*** Settings ***
Documentation     Common resources and keywords for E2E Robot Framework tests.
...
...               E2E tests use **zero mocking** — they exercise the real
...               CleverAgents CLI with real LLM API keys.  This resource
...               provides shared setup/teardown, API key detection with
...               graceful skip, and flexible assertion helpers.
Library           OperatingSystem
Library           String
Library           Process

*** Variables ***
${WORKSPACE}          ${CURDIR}/../..
${SRC_DIR}            ${WORKSPACE}/src/cleveragents
${E2E_TEMP_ROOT}      ${TEMPDIR}${/}cleveragents_e2e

*** Keywords ***
E2E Suite Setup
    [Documentation]    Set up the E2E test environment with per-suite isolation.
    ...
    ...                Creates a unique CLEVERAGENTS_HOME directory.
    ...                Does NOT enable mock AI — E2E tests use real providers.
    ...                Propagates LLM API keys from the environment.
    Log    Setting up E2E test environment
    Set Environment Variable    NO_COLOR    1
    ${safe_suite}=    Replace String    ${SUITE NAME}    ${SPACE}    _
    ${safe_suite}=    Replace String    ${safe_suite}    .    _
    ${safe_suite}=    Replace String Using Regexp    ${safe_suite}    [^A-Za-z0-9_-]    _
    ${home}=    Set Variable    ${E2E_TEMP_ROOT}${/}${safe_suite}
    ${rm_status}    ${rm_msg}=    Run Keyword And Ignore Error    Remove Directory    ${home}    recursive=True
    IF    '${rm_status}' == 'FAIL'
        Log    Could not remove previous suite home ${home}: ${rm_msg}    WARN
    END
    Create Directory    ${home}
    Set Environment Variable    CLEVERAGENTS_HOME    ${home}
    Set Suite Variable    ${SUITE_HOME}    ${home}
    Set Environment Variable    CLEVERAGENTS_AUTO_APPLY_MIGRATIONS    true
    # Explicitly disable mock AI — E2E tests use real providers
    Remove Environment Variable    CLEVERAGENTS_TESTING_USE_MOCK_AI
    # Get the actual Python executable being used
    ${python_exec}=    Evaluate    sys.executable    sys
    Set Suite Variable    ${PYTHON}    ${python_exec}
    # Explicitly initialize the workspace once per suite so DB-dependent
    # commands do not need per-test init workarounds.
    ${init}=    Run CleverAgents Command    init    e2e_test    --yes    --force    --path    ${SUITE_HOME}
    Should Be Equal As Integers    ${init.rc}    0
    # Post-init sanity: ensure init produced the expected workspace state.
    Directory Should Exist    ${SUITE_HOME}${/}.cleveragents
    File Should Exist    ${SUITE_HOME}${/}.cleveragents${/}db.sqlite

E2E Suite Teardown
    [Documentation]    Clean up the E2E test environment.
    Log    Cleaning up E2E test environment
    Run Keyword And Ignore Error    Remove Directory    ${SUITE_HOME}    recursive=True
    Remove Environment Variable    CLEVERAGENTS_HOME
    Remove Environment Variable    CLEVERAGENTS_AUTO_APPLY_MIGRATIONS
    Remove Environment Variable    NO_COLOR

Skip If No LLM Keys
    [Documentation]    Skip the current test if no LLM API keys are available.
    ...
    ...                Checks for ANTHROPIC_API_KEY and OPENAI_API_KEY.
    ...                If neither is set, the test is skipped gracefully.
    ...                Keys are evaluated inline to avoid storing raw secrets
    ...                in Robot Framework variables (which may be logged at DEBUG level).
    ${openai_key}=    Evaluate    os.environ.get('OPENAI_API_KEY', '')    modules=os
    ${anthropic_key}=    Evaluate    os.environ.get('ANTHROPIC_API_KEY', '')    modules=os
    ${prev_level}=    Set Log Level    NONE
    ${openai_valid}=    Run Keyword And Return Status    Should Match Regexp    ${openai_key}    ^(sk|rk)-[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)?$
    ${anthropic_valid}=    Run Keyword And Return Status    Should Match Regexp    ${anthropic_key}    ^sk-(ant|live)-[A-Za-z0-9_-]+$
    Set Log Level    ${prev_level}
    ${has_keys}=    Set Variable    ${openai_valid}
    Run Keyword If    not ${has_keys}    ${has_keys}=    Set Variable    ${anthropic_valid}
    Run Keyword If    not ${has_keys}    Skip    No usable LLM API keys available (ANTHROPIC_API_KEY / OPENAI_API_KEY). Skipping E2E test.

Resolve LLM Actor
    [Documentation]    Probe the OpenAI API with a minimal request to verify the key
    ...                is actually functional (not quota-exhausted), then return the
    ...                appropriate actor name.
    ...
    ...                Unlike a naive key-presence check, this keyword sends a real
    ...                HTTP request ("Hi", max_tokens=1, gpt-4o-mini) so that a
    ...                quota-exhausted key is detected *before* any test runs and the
    ...                suite can fall back to Anthropic automatically.
    ...
    ...                Arguments:
    ...                  openai_model    — actor name returned when the probe succeeds
    ...                                   (default: openai/gpt-4o).
    ...                  anthropic_model — actor name returned when the probe fails or
    ...                                   the key is absent
    ...                                   (default: anthropic/claude-sonnet-4-20250514).
    [Arguments]    ${openai_model}=openai/gpt-4o    ${anthropic_model}=anthropic/claude-sonnet-4-20250514
    # Short-circuit: if the key is not present at all, skip the network probe.
    ${has_openai}=    Evaluate    bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
    IF    not ${has_openai}
        Log    OPENAI_API_KEY not set — using ${anthropic_model}    WARN
        RETURN    ${anthropic_model}
    END
    # Probe the key with a minimal API call to detect quota exhaustion.
    ${check_script}=    Set Variable    ${CURDIR}${/}check_openai_key.py
    ${result}=    Run Process    ${PYTHON}    ${check_script}    timeout=30s    on_timeout=kill
    IF    ${result.rc} == 0
        Log    OpenAI key probe succeeded (${result.stdout.strip()}) — using ${openai_model}
        RETURN    ${openai_model}
    END
    Log    OpenAI key probe failed: ${result.stdout.strip()} — falling back to ${anthropic_model}    WARN
    RETURN    ${anthropic_model}

Run CleverAgents Command
    [Documentation]    Run a CleverAgents CLI command and return the result.
    ...
    ...                Executes ``python -m cleveragents <args>`` using the
    ...                venv Python.  Runs in ``SUITE_HOME`` so the CLI creates
    ...                its ``.cleveragents`` workspace inside the per-suite
    ...                temp directory, avoiding cross-run database pollution.
    ...                Returns the Process result object.
    ...
    ...                Accepts both multi-token form (one word per arg, separated
    ...                by 2+ spaces) and legacy single-string form (all words in
    ...                one arg separated by single spaces). Tokens of the form
    ...                ``expected_rc=X`` or ``timeout=Ys`` are extracted from any
    ...                position and used as keyword options.
    [Arguments]    @{args}    ${expected_rc}=${0}    ${timeout}=120s    ${cwd}=${SUITE_HOME}
    # Normalise: split any space-containing token using shlex rules, and
    # extract embedded expected_rc=X / timeout=Ys overrides.
    ${_cli}=    Evaluate    []
    FOR    ${_arg}    IN    @{args}
        ${_parts}=    Evaluate    __import__('shlex').split($_arg) if ' ' in $_arg else [$_arg]
        ${_new}=    Evaluate    [p for p in $_parts if not p.startswith('expected_rc=') and not p.startswith('timeout=')]
        ${_rc}=    Evaluate    [p.split('=',1)[1] for p in $_parts if p.startswith('expected_rc=')]
        ${_to}=    Evaluate    [p.split('=',1)[1] for p in $_parts if p.startswith('timeout=')]
        ${_cli}=    Evaluate    $_cli + $_new
        IF    len($_rc) > 0
            ${expected_rc}=    Evaluate    $_rc[-1]
        END
        IF    len($_to) > 0
            ${timeout}=    Evaluate    $_to[-1]
        END
    END
    ${result}=    Run Process    ${PYTHON}    -m    cleveragents    @{_cli}
    ...    cwd=${cwd}
    ...    timeout=${timeout}
    ...    on_timeout=kill
    ...    env:CLEVERAGENTS_HOME=${SUITE_HOME}
    ...    env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true
    ...    env:NO_COLOR=1
    Log    STDOUT: ${result.stdout}    level=DEBUG
    Log    STDERR: ${result.stderr}    level=DEBUG
    IF    '${expected_rc}' != 'None'
        Should Be Equal As Integers    ${result.rc}    ${expected_rc}
        ...    CleverAgents command failed with rc=${result.rc}. Check DEBUG-level log entries above.
    END
    RETURN    ${result}

Output Should Contain
    [Documentation]    Assert that command output contains expected text (flexible).
    ...
    ...                Checks stdout first, then stderr.  Case-insensitive by default.
    [Arguments]    ${result}    ${expected}    ${case_insensitive}=${TRUE}
    ${combined}=    Set Variable    ${result.stdout}\n${result.stderr}
    IF    ${case_insensitive}
        Should Contain    ${combined.lower()}    ${expected.lower()}
    ELSE
        Should Contain    ${combined}    ${expected}
    END

Safe Parse Json Field
    [Documentation]    Safely extract a JSON field from CLI stdout.
    ...
    ...                Tries two strategies to locate the JSON object:
    ...                1. Outer-bracket: first ``{`` to last ``}`` (works for
    ...                   single-object output with optional non-JSON preamble).
    ...                2. Last-line fallback: iterates lines from the end looking
    ...                   for a parseable JSON line (handles multi-object output
    ...                   where each object is on its own line).
    ...                Returns empty string if no JSON is found, the field is
    ...                absent, or all parse attempts fail.
    [Arguments]    ${stdout}    ${field_name}
    # Strategy 1: outer-bracket extraction (first '{' to last '}')
    ${pos}=    Evaluate    $stdout.find('{')
    IF    ${pos} == -1
        RETURN    ${EMPTY}
    END
    ${end}=    Evaluate    $stdout.rfind('}')
    IF    ${end} == -1 or ${end} < ${pos}
        RETURN    ${EMPTY}
    END
    ${json_str}=    Evaluate    $stdout[${pos}:${end}+1]
    ${status}    ${value}=    Run Keyword And Ignore Error
    ...    Evaluate    (lambda d,k: d[k] if k in d else next((v[k] for v in d.values() if isinstance(v,dict) and k in v),''))(__import__('json').loads($json_str),$field_name)
    IF    '${status}' == 'PASS'
        RETURN    ${value}
    END
    # Strategy 2: last-line fallback — scan lines in reverse for a parseable
    # JSON object that contains the requested field.
    ${strategy1_err}=    Set Variable    ${value}
    ${strategy2_err}=    Set Variable    no JSON lines found
    ${lines}=    Evaluate    list(reversed([l.strip() for l in $stdout.splitlines() if l.strip().startswith('{') and l.strip().endswith('}')]))
    FOR    ${line}    IN    @{lines}
        ${ls}    ${lv}=    Run Keyword And Ignore Error
        ...    Evaluate    __import__('json').loads($line).get($field_name, '')
        IF    '${ls}' == 'PASS' and '${lv}' != ''
            RETURN    ${lv}
        END
        ${strategy2_err}=    Set Variable    ${lv}
    END
    Log    Safe Parse Json Field: JSON parse failed for field '${field_name}'. Strategy 1: ${strategy1_err}; Strategy 2: ${strategy2_err}    WARN
    RETURN    ${EMPTY}

Run CLI
    [Documentation]    Run ``python -m cleveragents`` inside a workspace directory
    ...                with the correct environment variables.
    ...
    ...                Requires the calling suite to set ``${WS}`` (workspace
    ...                directory) and ``${SUITE_HOME}`` (CLEVERAGENTS_HOME) as
    ...                suite variables before calling this keyword.
    ...
    ...                Do not embed raw stdout/stderr in assertion failure messages —
    ...                they may contain API key material.  DEBUG-level logs have the
    ...                details.
    [Arguments]    @{args}    ${expected_rc}=${0}    ${timeout}=120s
    ${result}=    Run Process    ${PYTHON}    -m    cleveragents    @{args}
    ...    cwd=${WS}
    ...    env:CLEVERAGENTS_HOME=${SUITE_HOME}
    ...    env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true
    ...    env:NO_COLOR=1
    ...    timeout=${timeout}
    ...    on_timeout=kill
    Log    STDOUT: ${result.stdout}    level=DEBUG
    Log    STDERR: ${result.stderr}    level=DEBUG
    Should Be Equal As Integers    ${result.rc}    ${expected_rc}
    ...    CLI failed (rc=${result.rc}). Check DEBUG-level log entries above for stdout/stderr.
    RETURN    ${result}

Extract JSON From Stdout
    [Documentation]    Extract the first JSON object from stdout that may contain
    ...                leading non-JSON text (e.g. structlog debug messages).
    ...                Uses ``raw_decode`` with ``strict=False`` to tolerate
    ...                literal control characters that Rich's ``console.print``
    ...                may inject when it wraps long lines inside JSON string
    ...                values.  Wrapped in ``TRY/EXCEPT`` for clear failure messages.
    ...
    ...                **Note:** This picks the *first* JSON object (by scanning for
    ...                the first ``{``).  Non-JSON preamble (e.g. structlog lines)
    ...                is mitigated by ``NO_COLOR=1`` (disables Rich formatting) and
    ...                structlog routing to stderr, so stdout typically starts with
    ...                the JSON payload.  See ``Safe Parse Json Field`` in
    ...                ``common_e2e.resource`` for a last-line fallback strategy
    ...                that scans in reverse.
    [Arguments]    ${text}
    TRY
        ${start}=    Evaluate    $text.index('{')
        ${json_obj}=    Evaluate    json.JSONDecoder(strict=False).raw_decode($text, $start)[0]    modules=json
    EXCEPT    AS    ${err}
        Fail    Failed to extract JSON object from stdout: ${err}
    END
    RETURN    ${json_obj}

Link Resource To Project
    [Documentation]    Link the suite-level workspace resource to a project.
    ...
    ...                Requires the calling suite to set ``${WS_RESOURCE}`` as a
    ...                suite variable before calling this keyword.
    [Arguments]    ${project_name}
    ${result}=    Run CLI    project    link-resource    ${project_name}    ${WS_RESOURCE}
    Should Be Equal As Integers    ${result.rc}    0
    ...    msg=project link-resource failed (rc=${result.rc}). Check DEBUG logs above.

Create Synthetic Codebase
    [Documentation]    Populate a directory with small Python files and one
    ...                deliberately large file for budget-enforcement testing.
    ...
    ...                ``${project_label}`` is embedded in docstrings and print
    ...                statements to distinguish output from different suites.
    [Arguments]    ${base_dir}    ${project_label}=E2E test project
    ${main_py}=    Catenate    SEPARATOR=\n
    ...    """Main entry point for ${project_label}."""
    ...    ${EMPTY}
    ...    def main() -> None:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}"""Run the application."""
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}print("Hello from ${project_label}")
    ...    ${EMPTY}
    ...    if __name__ == "__main__":
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}main()
    Create File    ${base_dir}${/}main.py    ${main_py}
    ${utils_py}=    Catenate    SEPARATOR=\n
    ...    """Utility helpers."""
    ...    ${EMPTY}
    ...    def add(a: int, b: int) -> int:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return a + b
    ...    ${EMPTY}
    ...    def multiply(a: int, b: int) -> int:
    ...    ${SPACE}${SPACE}${SPACE}${SPACE}return a * b
    Create File    ${base_dir}${/}utils.py    ${utils_py}
    Create File    ${base_dir}${/}config.py    TIMEOUT = 30\nMAX_RETRIES = 3\nDEBUG = False
    # Large file (>1 KiB) for budget testing
    ${large_content}=    Evaluate    "# auto-generated large file\\n" + ("x = 1\\n" * 250)
    Create File    ${base_dir}${/}large_file.py    ${large_content}

Create Temp Git Repo
    [Documentation]    Create a temporary git repository for E2E testing.
    ...
    ...                Returns the path to the created repository.
    [Arguments]    ${name}=test-repo
    ${repo_dir}=    Set Variable    ${SUITE_HOME}${/}${name}
    Create Directory    ${repo_dir}
    ${r1}=    Run Process    git    init    cwd=${repo_dir}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${r1.rc}    0    msg=git init failed (rc=${r1.rc}). Check DEBUG logs above.
    ${r2}=    Run Process    git    config    user.name    E2E Test    cwd=${repo_dir}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${r2.rc}    0    msg=git config user.name failed
    ${r3}=    Run Process    git    config    user.email    e2e@test.local    cwd=${repo_dir}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${r3.rc}    0    msg=git config user.email failed
    ${r3b}=    Run Process    git    config    commit.gpgsign    false    cwd=${repo_dir}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${r3b.rc}    0    msg=git config commit.gpgsign failed
    # Create an initial commit so the repo has a HEAD
    Create File    ${repo_dir}${/}README.md    # Test Repository\n
    ${r4}=    Run Process    git    add    .    cwd=${repo_dir}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${r4.rc}    0    msg=git add failed (rc=${r4.rc}). Check DEBUG logs above.
    ${r5}=    Run Process    git    commit    -m    Initial commit    cwd=${repo_dir}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${r5.rc}    0    msg=git commit failed (rc=${r5.rc}). Check DEBUG logs above.
    RETURN    ${repo_dir}