From d4a1a0d87b201def91e02e38382ccc99e4c89323 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Thu, 12 Mar 2026 21:11:37 +0000 Subject: [PATCH] =?UTF-8?q?test(e2e):=20set=20up=20E2E=20test=20infrastruc?= =?UTF-8?q?ture=20=E2=80=94=20nox=20session,=20CI=20job,=20Robot=20Framewo?= =?UTF-8?q?rk=20@E2E=20tag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added dedicated E2E test infrastructure completely separate from the existing integration test suite. E2E tests use zero mocking — they exercise the real CleverAgents CLI with real LLM API keys. Key changes: - New e2e_tests nox session running Robot Framework with --include E2E tag filter against robot/e2e/ directory. Uses sequential robot (not pabot) since E2E tests hit real API endpoints with rate limits. Propagates ANTHROPIC_API_KEY, OPENAI_API_KEY, and GOOGLE_API_KEY from environment. Output goes to build/reports/robot-e2e/ to avoid artifact collisions. - Existing integration_tests session now excludes E2E-tagged tests via --exclude E2E on the pabot invocation. - New robot/e2e/common_e2e.resource provides shared E2E keywords: suite setup/teardown with per-suite isolation (no mock AI), graceful skip when LLM API keys are absent, CLI runner keyword, flexible output assertions, and temporary git repo fixture creation. - Minimal smoke test (robot/e2e/smoke_test.robot) validates the harness by running agents --version and agents --help. Does not require LLM keys. - Dedicated e2e_tests CI job in .forgejo/workflows/ci.yml injects LLM API keys from Forgejo secrets. Runs independently (no needs dependencies) and does not block regular CI. - e2e_tests is deliberately NOT in the default nox sessions list since it requires real API keys not present in all environments. ISSUES CLOSED: #740 --- .forgejo/workflows/ci.yml | 24 +++++++++ CHANGELOG.md | 6 +++ noxfile.py | 67 ++++++++++++++++++++++++ robot/e2e/common_e2e.resource | 99 +++++++++++++++++++++++++++++++++++ robot/e2e/smoke_test.robot | 25 +++++++++ 5 files changed, 221 insertions(+) create mode 100644 robot/e2e/common_e2e.resource create mode 100644 robot/e2e/smoke_test.robot diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 4ed41a549..d5ccd03a5 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -150,6 +150,30 @@ jobs: env: NOX_DEFAULT_VENV_BACKEND: uv + e2e_tests: + runs-on: docker + container: + image: python:3.13-slim + steps: + - name: Install system dependencies (nodejs for checkout, git for E2E tests) + run: | + apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + + - name: Install uv and nox + run: | + pip install -q uv==${{ env.UV_VERSION }} nox + + - name: Run E2E tests via nox + run: | + nox -s e2e_tests + env: + NOX_DEFAULT_VENV_BACKEND: uv + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + coverage: runs-on: docker container: diff --git a/CHANGELOG.md b/CHANGELOG.md index 735677199..4001686e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- Added dedicated E2E test infrastructure: new `nox -s e2e_tests` session + running Robot Framework with `--include E2E` tag filter against `robot/e2e/` + directory, dedicated CI job with real LLM API key secrets, graceful skip + when API keys are absent, and `--exclude E2E` on the standard integration + test session. Includes a minimal smoke test exercising `agents --version` + and `agents --help`. (#740) - Added TDD-style failing Behave BDD tests for the session list DI container missing `db` provider bug. Three scenarios exercise `session list`, `_get_session_service()`, and `session list --format json` through the real diff --git a/noxfile.py b/noxfile.py index 72e47daf4..9de22a385 100644 --- a/noxfile.py +++ b/noxfile.py @@ -604,6 +604,8 @@ def integration_tests(session: nox.Session): "code_blocks", "--exclude", "wip", + "--exclude", + "E2E", "--listener", "robot/tdd_expected_fail_listener.py", *robot_args, @@ -635,6 +637,71 @@ def slow_integration_tests(session: nox.Session): ) +@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv") +def e2e_tests(session: nox.Session): + """Run end-to-end Robot Framework tests with real LLM API keys. + + E2E tests use zero mocking — they exercise the real CleverAgents CLI + against real LLM API keys (Anthropic/OpenAI). Tests are tagged with + ``E2E`` and live in the ``robot/e2e/`` directory. + + This session is NOT included in the default ``nox`` run because it + requires real API keys. Run explicitly via ``nox -s e2e_tests``. + + Tests that require LLM API keys will skip gracefully when the keys + are not present in the environment. + """ + session.install("-e", ".[tests]") + session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true" + session.env["NO_COLOR"] = "1" + session.env["PYTHONPATH"] = "src" + + # Propagate venv bin to PATH so Run Process in robot files finds + # the venv's python/robot rather than the system copies. + venv_bin = os.path.join(session.virtualenv.location, "bin") + session.env["PATH"] = venv_bin + os.pathsep + os.environ.get("PATH", "") + venv_python = os.path.join(venv_bin, "python") + + # Ensure output directory exists + os.makedirs("build/reports/robot-e2e", exist_ok=True) + + # Pre-compile bytecode to avoid cold-compilation overhead. + session.run("python", "-m", "compileall", "-q", "src/") + + # Propagate LLM API keys from the environment into the session + # so that real E2E tests can authenticate with providers. + for key in ( + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "GOOGLE_API_KEY", + ): + value = os.environ.get(key) + if value: + session.env[key] = value + + session.run( + "robot", + "--outputdir", + "build/reports/robot-e2e", + "--loglevel", + "INFO", + "--report", + "report.html", + "--log", + "log.html", + "--xunit", + "xunit.xml", + "--variable", + f"PYTHON:{venv_python}", + "--include", + "E2E", + "--listener", + "robot/tdd_expected_fail_listener.py", + *session.posargs, + "robot/e2e/", + ) + + COVERAGE_THRESHOLD = 97 diff --git a/robot/e2e/common_e2e.resource b/robot/e2e/common_e2e.resource new file mode 100644 index 000000000..c78f0ec0a --- /dev/null +++ b/robot/e2e/common_e2e.resource @@ -0,0 +1,99 @@ +*** Settings *** +Documentation Common resources and keywords for E2E Robot Framework tests. +... +... E2E tests use **zero mocking** — they exercise the real +... CleverAgents CLI with real LLM API keys. This resource +... provides shared setup/teardown, API key detection with +... graceful skip, and flexible assertion helpers. +Library OperatingSystem +Library String +Library Collections +Library Process + +*** Variables *** +${WORKSPACE} ${CURDIR}/../.. +${SRC_DIR} ${WORKSPACE}/src/cleveragents +${E2E_TEMP_ROOT} ${TEMPDIR}${/}cleveragents_e2e + +*** Keywords *** +E2E Suite Setup + [Documentation] Set up the E2E test environment with per-suite isolation. + ... + ... Creates a unique CLEVERAGENTS_HOME directory. + ... Does NOT enable mock AI — E2E tests use real providers. + ... Propagates LLM API keys from the environment. + Log Setting up E2E test environment + ${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _ + ${home}= Set Variable ${E2E_TEMP_ROOT}${/}${safe_suite} + Run Keyword And Ignore Error Remove Directory ${home} recursive=True + Create Directory ${home} + Set Environment Variable CLEVERAGENTS_HOME ${home} + Set Suite Variable ${SUITE_HOME} ${home} + Set Environment Variable CLEVERAGENTS_AUTO_APPLY_MIGRATIONS true + # Explicitly disable mock AI — E2E tests use real providers + Remove Environment Variable CLEVERAGENTS_TESTING_USE_MOCK_AI + # Get the actual Python executable being used + ${python_exec}= Evaluate sys.executable sys + Set Suite Variable ${PYTHON} ${python_exec} + +E2E Suite Teardown + [Documentation] Clean up the E2E test environment. + Log Cleaning up E2E test environment + Run Keyword And Ignore Error Remove Directory ${SUITE_HOME} recursive=True + Remove Environment Variable CLEVERAGENTS_HOME + Remove Environment Variable CLEVERAGENTS_AUTO_APPLY_MIGRATIONS + +Skip If No LLM Keys + [Documentation] Skip the current test if no LLM API keys are available. + ... + ... Checks for ANTHROPIC_API_KEY and OPENAI_API_KEY. + ... If neither is set, the test is skipped gracefully. + ${anthropic}= Get Environment Variable ANTHROPIC_API_KEY default=${EMPTY} + ${openai}= Get Environment Variable OPENAI_API_KEY default=${EMPTY} + Skip If '${anthropic}' == '' and '${openai}' == '' + ... No LLM API keys available (ANTHROPIC_API_KEY / OPENAI_API_KEY). Skipping E2E test. + +Run CleverAgents Command + [Documentation] Run a CleverAgents CLI command and return the result. + ... + ... Executes ``python -m cleveragents `` using the + ... venv Python. Returns the Process result object. + [Arguments] @{args} ${expected_rc}=${0} ${timeout}=120s + ${result}= Run Process ${PYTHON} -m cleveragents @{args} + ... timeout=${timeout} + ... env:CLEVERAGENTS_HOME=${SUITE_HOME} + ... env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true + ... env:NO_COLOR=1 + Log STDOUT: ${result.stdout} + Log STDERR: ${result.stderr} + Run Keyword If '${expected_rc}' != 'None' + ... Should Be Equal As Integers ${result.rc} ${expected_rc} + ... CleverAgents command failed with rc=${result.rc}.\nSTDOUT: ${result.stdout}\nSTDERR: ${result.stderr} + RETURN ${result} + +Output Should Contain + [Documentation] Assert that command output contains expected text (flexible). + ... + ... Checks stdout first, then stderr. Case-insensitive by default. + [Arguments] ${result} ${expected} ${case_insensitive}=${TRUE} + ${combined}= Set Variable ${result.stdout}\n${result.stderr} + Run Keyword If ${case_insensitive} + ... Should Contain ${combined.lower()} ${expected.lower()} + ... ELSE + ... Should Contain ${combined} ${expected} + +Create Temp Git Repo + [Documentation] Create a temporary git repository for E2E testing. + ... + ... Returns the path to the created repository. + [Arguments] ${name}=test-repo + ${repo_dir}= Set Variable ${SUITE_HOME}${/}${name} + Create Directory ${repo_dir} + Run Process git init cwd=${repo_dir} + Run Process git config user.name E2E Test cwd=${repo_dir} + Run Process git config user.email e2e@test.local cwd=${repo_dir} + # Create an initial commit so the repo has a HEAD + Create File ${repo_dir}${/}README.md # Test Repository\n + Run Process git add . cwd=${repo_dir} + Run Process git commit -m Initial commit cwd=${repo_dir} + RETURN ${repo_dir} diff --git a/robot/e2e/smoke_test.robot b/robot/e2e/smoke_test.robot new file mode 100644 index 000000000..6d1847008 --- /dev/null +++ b/robot/e2e/smoke_test.robot @@ -0,0 +1,25 @@ +*** Settings *** +Documentation Minimal E2E smoke test to validate the E2E test infrastructure. +... +... This test exercises ``agents --version`` to verify that the +... E2E harness (nox session, Robot Framework tag filtering, +... and common resource keywords) works correctly. It does +... NOT require LLM API keys. +Resource common_e2e.resource +Suite Setup E2E Suite Setup +Suite Teardown E2E Suite Teardown + +*** Test Cases *** +CleverAgents Version Smoke Test + [Documentation] Verify the CleverAgents CLI responds to --version. + [Tags] E2E + ${result}= Run CleverAgents Command --version + Should Not Be Empty ${result.stdout} + Output Should Contain ${result} cleveragents + +CleverAgents Help Smoke Test + [Documentation] Verify the CleverAgents CLI responds to --help. + [Tags] E2E + ${result}= Run CleverAgents Command --help + Should Not Be Empty ${result.stdout} + Output Should Contain ${result} usage -- 2.52.0