From 162b1e35d0042aa8f07ca8bdc445fcf8845548c1 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Sat, 20 Dec 2025 10:12:32 -0500 Subject: [PATCH] Build: Made Robot integration tests run in parallel --- .gitignore | 1 + noxfile.py | 74 +++++++++++++++------------ pyproject.toml | 3 +- robot/cli.robot | 3 ++ robot/cli_plan_context_commands.robot | 4 +- robot/common.resource | 5 ++ robot/context_analysis_agent.robot | 3 ++ robot/core_cli_commands.robot | 8 +-- robot/google_provider.robot | 2 + robot/openai_provider.robot | 2 + robot/plan_generation_graph.robot | 3 ++ 11 files changed, 68 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index faa8a52a1..54ee302a2 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,4 @@ claude-flow.ps1 hive-mind-prompt-*.txt .cleveragents/ +.pabotsuitenames diff --git a/noxfile.py b/noxfile.py index 3e422e52b..30e33b179 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,6 +20,13 @@ BEHAVE_PARALLEL_URL = ( ) +def _default_processes() -> int: + try: + return len(os.sched_getaffinity(0)) or 1 + except AttributeError: + return os.cpu_count() or 1 + + def _behave_parallel_args(posargs: list[str]) -> list[str]: has_custom_processes = any( arg in {"--processes", "-j"} or arg.startswith(("--processes=", "-j")) @@ -27,11 +34,22 @@ def _behave_parallel_args(posargs: list[str]) -> list[str]: ) if has_custom_processes: return [] - return ["--processes", str(os.cpu_count() or 1)] + return ["--processes", str(_default_processes())] + + +def _pabot_parallel_args(posargs: list[str]) -> list[str]: + has_custom_processes = any( + arg in {"--processes", "-p"} or arg.startswith(("--processes=", "-p")) + for arg in posargs + ) + if has_custom_processes: + return [] + return ["--processes", str(_default_processes())] def _install_behave_parallel(session: nox.Session) -> None: """Install behave-parallel, repackaging the sdist if needed.""" + try: session.install(f"behave-parallel=={BEHAVE_PARALLEL_VERSION}") return @@ -251,24 +269,29 @@ def integration_tests(session: nox.Session): session.install("-e", ".[tests]") session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true" - # If posargs provided, run only those specific files/suites + parallel_args = _pabot_parallel_args(session.posargs) + common_args = [ + "pabot", + "--outputdir", + "build", + "--loglevel", + "INFO", + "--report", + "reports/robot/report.html", + "--log", + "reports/robot/log.html", + "--xunit", + "reports/robot/xunit.xml", + "--exclude", + "slow", + "--exclude", + "discovery", + *parallel_args, + ] + if session.posargs: session.run( - "robot", - "--outputdir", - "build/reports/robot", - "--loglevel", - "INFO", - "--report", - "report.html", - "--log", - "log.html", - "--xunit", - "xunit.xml", - "--exclude", - "slow", - "--exclude", - "discovery", # Exclude discovery tests + *common_args, "--exitonfailure", # Exit immediately on test failure *session.posargs, external=False, # Ensure exit code is checked @@ -277,22 +300,7 @@ def integration_tests(session: nox.Session): else: # Default: run all tests in robot/ directory session.run( - "robot", - "--outputdir", - "build/reports/robot", - "--loglevel", - "INFO", - "--report", - "report.html", - "--log", - "log.html", - "--xunit", - "xunit.xml", - "--exclude", - "slow", - "--exclude", - "discovery", # Exclude discovery tests - # "--exitonfailure", # Exit immediately on test failure + *common_args, "robot/", external=False, # Ensure exit code is checked success_codes=[0], # Only code 0 is success diff --git a/pyproject.toml b/pyproject.toml index a7d68d371..6e517b2d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,8 @@ dev = [ tests = [ "coverage>=7.11.0", "asv>=0.6.5", - "robotframework>=7.3.2" + "robotframework>=7.3.2", + "robotframework-pabot>=4.0.0", ] docs = [ "mkdocs>=1.6.1", diff --git a/robot/cli.robot b/robot/cli.robot index a1bfec3bf..9841879b7 100644 --- a/robot/cli.robot +++ b/robot/cli.robot @@ -1,9 +1,12 @@ *** Settings *** Documentation CLI Benchmark and Integration Tests for CleverAgents +Resource common.resource Library Process Library OperatingSystem Library String Library Collections +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment *** Variables *** # PYTHON variable is passed from nox via --variable PYTHON:/path/to/python diff --git a/robot/cli_plan_context_commands.robot b/robot/cli_plan_context_commands.robot index 1af07c824..e2d6bd583 100644 --- a/robot/cli_plan_context_commands.robot +++ b/robot/cli_plan_context_commands.robot @@ -5,7 +5,7 @@ Resource common.resource Library OperatingSystem Library Process Library String -Suite Setup Set Environment Variable CLEVERAGENTS_AUTO_APPLY_MIGRATIONS true +Suite Setup Run Keywords Setup Test Environment AND Set Environment Variable CLEVERAGENTS_AUTO_APPLY_MIGRATIONS true *** Variables *** ${PYTHON} python @@ -252,8 +252,6 @@ Setup Test Directory Cleanup Test Directory [Documentation] Remove test directory and cleanup processes - # Kill any hanging cleveragents processes - Run Keyword And Ignore Error Run Process pkill -f cleveragents # Clean up test directory Run Keyword And Ignore Error Remove Directory ${TEST_DIR} recursive=True diff --git a/robot/common.resource b/robot/common.resource index 665ce9426..48489f3c2 100644 --- a/robot/common.resource +++ b/robot/common.resource @@ -16,6 +16,11 @@ ${PYTHON} python Setup Test Environment [Documentation] Setup common test environment Log Setting up test environment + ${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _ + ${home}= Set Variable ${TEMPDIR}${/}.cleveragents_${safe_suite} + Run Keyword And Ignore Error Remove Directory ${home} recursive=True + Create Directory ${home} + Set Environment Variable CLEVERAGENTS_HOME ${home} Run Keyword And Ignore Error Remove Directory ${TEMPDIR}${/}.cleveragents recursive=True Cleanup Test Environment diff --git a/robot/context_analysis_agent.robot b/robot/context_analysis_agent.robot index b5a1019f5..1b4393b45 100644 --- a/robot/context_analysis_agent.robot +++ b/robot/context_analysis_agent.robot @@ -1,9 +1,12 @@ *** Settings *** Documentation Integration tests for ContextAnalysisAgent workflow +Resource common.resource Library Process Library OperatingSystem Library Collections Library String +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment *** Variables *** ${PYTHON} python diff --git a/robot/core_cli_commands.robot b/robot/core_cli_commands.robot index ef948761b..79a5b62f8 100644 --- a/robot/core_cli_commands.robot +++ b/robot/core_cli_commands.robot @@ -206,6 +206,10 @@ Test Project Status With Project Setup Test Environment [Documentation] Setup the test environment Create Directory ${TEST_DIR} + ${home}= Set Variable ${TEST_DIR}${/}.cleveragents_home + Run Keyword And Ignore Error Remove Directory ${home} recursive=True + Create Directory ${home} + Set Environment Variable CLEVERAGENTS_HOME ${home} Clean Core CLI Temp Dir Run Keyword And Ignore Error Remove Directory ${TEST_DIR} recursive=True @@ -213,7 +217,5 @@ Clean Core CLI Temp Dir Cleanup Test Environment [Documentation] Clean up after tests - # Kill any hanging cleveragents processes - Run Keyword And Ignore Error Run Process pkill -f cleveragents # Clean up test directory - Run Keyword And Ignore Error Remove Directory ${TEST_DIR} recursive=true \ No newline at end of file + Run Keyword And Ignore Error Remove Directory ${TEST_DIR} recursive=true diff --git a/robot/google_provider.robot b/robot/google_provider.robot index 49a016519..7fd7d9b88 100644 --- a/robot/google_provider.robot +++ b/robot/google_provider.robot @@ -3,6 +3,8 @@ Resource common.resource Library Process Library OperatingSystem Library Collections +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment *** Variables *** ${PYTHON} python diff --git a/robot/openai_provider.robot b/robot/openai_provider.robot index de251546b..8f1c20230 100644 --- a/robot/openai_provider.robot +++ b/robot/openai_provider.robot @@ -3,6 +3,8 @@ Resource common.resource Library Process Library OperatingSystem Library Collections +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment *** Variables *** ${PYTHON} python diff --git a/robot/plan_generation_graph.robot b/robot/plan_generation_graph.robot index 45e2f27d5..82d31502a 100644 --- a/robot/plan_generation_graph.robot +++ b/robot/plan_generation_graph.robot @@ -1,9 +1,12 @@ *** Settings *** Documentation Integration tests for PlanGenerationGraph workflow +Resource common.resource Library Process Library OperatingSystem Library Collections Library String +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment *** Variables *** ${PYTHON} ${CURDIR}/../.nox/integration_tests-3-13/bin/python