From 94d133c5bc7aae4423b7194524b106d9da4e754b Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Sun, 5 Apr 2026 04:13:34 +0000 Subject: [PATCH] perf(ci): optimize e2e_tests push job to reduce execution time - Replaced the e2e_tests nox session runner from sequential robot to the parallel pabot runner to shrink CI push times. - Added _split_pabot_args and _pabot_parallel_args support to the e2e_tests session, mirroring the pattern used by integration_tests for consistent parallel execution control. - CI workflow: added TEST_PROCESSES: "4" environment variable to the e2e_tests job to run four parallel workers. Rationale: E2E suites are IO-bound (LLM API calls) rather than CPU-bound, so increasing parallelism reduces wall-clock time without CPU contention. - Updated the e2e_tests docstring to document parallelism control via the TEST_PROCESSES environment variable or the --processes positional argument. - Pre-compiled bytecode comment updated to explain thundering-herd prevention for parallel workers. - Template DB comment updated to explain the criticality of proper initialization for parallel pabot execution. Key design decisions: - Used pabot (robotframework-pabot>=4.0.0), a project-provided dependency, instead of introducing new tooling. - Set TEST_PROCESSES=4 in CI (instead of the default min(cpu,2)) because E2E tasks are IO-bound and benefit from higher concurrency without CPU contention. - Suite-level isolation via E2E Suite Setup (separate CLEVERAGENTS_HOME per suite) ensures workers do not share database state, enabling safe parallelism. - Followed the exact same pattern as integration_tests for consistency and predictability. Modules/components affected: - e2e_tests nox session (parallelization logic and arg parsing) - CI workflow (TEST_PROCESSES environment variable) - E2E session docstring and related comments (documentation of parallelism and initialization) - Inline comments for pre-compiled bytecode and template database initialization to reflect parallel execution considerations ISSUES CLOSED: #1860 --- .forgejo/workflows/ci.yml | 4 ++++ noxfile.py | 36 ++++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 5e2e88d88..1d4b02471 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -315,6 +315,10 @@ jobs: nox -s e2e_tests 2>&1 | tee build/nox-e2e-tests-output.log env: NOX_DEFAULT_VENV_BACKEND: uv + # Run E2E suites in parallel via pabot. 4 workers keeps + # wall-clock time well under the 45-minute timeout while + # staying within the memory budget of the docker runner. + TEST_PROCESSES: "4" ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} diff --git a/noxfile.py b/noxfile.py index 01ae1f963..7394f7b17 100644 --- a/noxfile.py +++ b/noxfile.py @@ -413,6 +413,10 @@ def e2e_tests(session: nox.Session): Tests that require LLM API keys will skip gracefully when the keys are not present in the environment. + + Parallelism is controlled by the ``TEST_PROCESSES`` environment variable + (default: min(cpu_count, 2)). Override via ``TEST_PROCESSES=N nox -s + e2e_tests`` or by passing ``--processes N`` as a session argument. """ session.install("-e", ".[tests]") session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true" @@ -428,12 +432,18 @@ def e2e_tests(session: nox.Session): # Ensure output directory exists os.makedirs("build/reports/robot-e2e", exist_ok=True) - # Pre-compile bytecode to avoid cold-compilation overhead. + # Pre-compile bytecode so that parallel pabot workers (and the + # Python sub-processes they spawn via ``Run Process``) can read + # cached .pyc files instead of each cold-compiling every module + # from source simultaneously — avoids a thundering-herd race on + # CI runners with high core counts. session.run("python", "-m", "compileall", "-q", "src/") # Build a pre-migrated template DB so helper scripts that call # setup_workspace() can copy it instead of running 25+ Alembic # migrations per test — reduces per-test setup from ~1-3 s to ~1 ms. + # Critical for parallel pabot execution where many suites start + # simultaneously and would otherwise all race to run migrations. template_path = _create_template_db(session) session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path @@ -449,8 +459,26 @@ def e2e_tests(session: nox.Session): if value: session.env[key] = value + # Split posargs into pabot-specific args (--processes) and robot args. + pabot_args, robot_args = _split_pabot_args(session.posargs) + parallel_args = _pabot_parallel_args(pabot_args) + + # TDD expected-fail listener — inverts results for @tdd_expected_fail + # tagged tests and validates TDD tag combinations. + # See CONTRIBUTING.md > TDD Issue Test Tags. + tdd_listener = str( + Path(__file__).parent / "robot" / "tdd_expected_fail_listener.py" + ) + + # Use pabot (parallel Robot Framework runner) instead of the sequential + # ``robot`` runner. Each E2E suite file runs in its own pabot worker + # process, so independent suites execute concurrently rather than + # serially. Suite-level isolation (separate CLEVERAGENTS_HOME per + # suite) ensures workers do not share database state. session.run( - "robot", + "pabot", + *parallel_args, + *pabot_args, "--outputdir", "build/reports/robot-e2e", "--loglevel", @@ -466,8 +494,8 @@ def e2e_tests(session: nox.Session): "--include", "E2E", "--listener", - str(Path(__file__).parent / "robot" / "tdd_expected_fail_listener.py"), - *session.posargs, + tdd_listener, + *robot_args, "robot/e2e/", ) -- 2.52.0