From 81d72cc0479998c8a493bc7acbc28b87d6dd1515 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Thu, 12 Mar 2026 04:50:29 +0000 Subject: [PATCH] fix(test): env-gate Robot LLM tests to skip when API keys unavailable Add [Tags] llm-required to 28 Robot Framework test cases that instantiate ChatAnthropic (requiring ANTHROPIC_API_KEY): - 7 tests in context_analysis_agent.robot - 2 tests in database_integration.robot - 19 tests in plan_generation_graph.robot Conditionally pass --exclude llm-required to pabot in the integration_tests nox session when ANTHROPIC_API_KEY is not set, so CI environments without the secret skip those tests instead of failing. --- noxfile.py | 8 ++++++++ robot/context_analysis_agent.robot | 7 +++++++ robot/database_integration.robot | 2 ++ robot/plan_generation_graph.robot | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/noxfile.py b/noxfile.py index 72e47daf..fae14f69 100644 --- a/noxfile.py +++ b/noxfile.py @@ -580,6 +580,13 @@ def integration_tests(session: nox.Session): pabot_args, robot_args = _split_pabot_args(session.posargs) parallel_args = _pabot_parallel_args(pabot_args) + # Skip tests that require real LLM API keys when the keys are not + # available (e.g. CI environments without secrets configured). + llm_exclude_args: list[str] = [] + if not os.environ.get("ANTHROPIC_API_KEY"): + session.log("ANTHROPIC_API_KEY not set – excluding llm-required tests") + llm_exclude_args.extend(["--exclude", "llm-required"]) + session.run( "pabot", *parallel_args, @@ -604,6 +611,7 @@ def integration_tests(session: nox.Session): "code_blocks", "--exclude", "wip", + *llm_exclude_args, "--listener", "robot/tdd_expected_fail_listener.py", *robot_args, diff --git a/robot/context_analysis_agent.robot b/robot/context_analysis_agent.robot index e9fb7c83..ac43fef2 100644 --- a/robot/context_analysis_agent.robot +++ b/robot/context_analysis_agent.robot @@ -25,6 +25,7 @@ Context Analysis Agent Module Can Be Imported Context Analysis Agent Can Be Instantiated With Default Parameters [Documentation] Create ContextAnalysisAgent with defaults + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -42,6 +43,7 @@ Context Analysis Agent Can Be Instantiated With Default Parameters Context Analysis Agent Can Be Instantiated With Custom Chunk Settings [Documentation] Create ContextAnalysisAgent with custom chunk_size and overlap + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -58,6 +60,7 @@ Context Analysis Agent Can Be Instantiated With Custom Chunk Settings Context Analysis Agent Workflow Contains Expected Nodes [Documentation] Verify all workflow nodes are present using helper script + [Tags] llm-required ${result}= Run Process ${PYTHON} ${HELPER} nodes Log ${result.stdout} Log ${result.stderr} @@ -81,6 +84,7 @@ LangGraph Graphs Package Exports Context Analysis Agent Context Analysis Agent Can Load Files [Documentation] Test file loading functionality using helper script + [Tags] llm-required ${result}= Run Process ${PYTHON} ${HELPER} load_files Log ${result.stdout} Log ${result.stderr} @@ -90,6 +94,7 @@ Context Analysis Agent Can Load Files Context Analysis Agent Handles Missing Files [Documentation] Test error handling for missing files using helper script + [Tags] llm-required ${result}= Run Process ${PYTHON} ${HELPER} missing_file Log ${result.stdout} Log ${result.stderr} @@ -99,6 +104,7 @@ Context Analysis Agent Handles Missing Files Context Analysis Agent Invoke Returns Complete Result [Documentation] Test complete workflow execution via invoke using helper script + [Tags] llm-required ${result}= Run Process ${PYTHON} ${HELPER} invoke Log ${result.stdout} Log ${result.stderr} @@ -108,6 +114,7 @@ Context Analysis Agent Invoke Returns Complete Result Context Analysis Agent Streaming Produces Updates [Documentation] Test streaming workflow execution using helper script + [Tags] llm-required ${result}= Run Process ${PYTHON} ${HELPER} streaming Log ${result.stdout} Log ${result.stderr} diff --git a/robot/database_integration.robot b/robot/database_integration.robot index f7f3ec32..0f39a11d 100644 --- a/robot/database_integration.robot +++ b/robot/database_integration.robot @@ -112,6 +112,7 @@ Unit Of Work Transaction Rollback Service Layer Uses Repositories [Documentation] Test that services properly use repositories + [Tags] llm-required Create Temporary Project Directory Initialize Project With Service service-project ${project}= Get Current Project From Service @@ -127,6 +128,7 @@ Service Layer Uses Repositories End To End Database Workflow [Documentation] Test complete workflow using database + [Tags] llm-required Create Temporary Project Directory # Run the complete workflow in one script diff --git a/robot/plan_generation_graph.robot b/robot/plan_generation_graph.robot index 94444970..548e0fe1 100644 --- a/robot/plan_generation_graph.robot +++ b/robot/plan_generation_graph.robot @@ -25,6 +25,7 @@ Plan Generation Graph Module Can Be Imported Plan Generation Graph Can Be Instantiated With Default Parameters [Documentation] Create PlanGenerationGraph with defaults + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -41,6 +42,7 @@ Plan Generation Graph Can Be Instantiated With Default Parameters Plan Generation Graph Can Be Instantiated With Custom Max Retries [Documentation] Create PlanGenerationGraph with custom max_retries + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -55,6 +57,7 @@ Plan Generation Graph Can Be Instantiated With Custom Max Retries Plan Generation Graph Creates Prompt Templates [Documentation] Verify prompt templates are created + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -73,6 +76,7 @@ Plan Generation Graph Creates Prompt Templates Plan Generation Graph Builds Workflow With Correct Nodes [Documentation] Verify workflow graph has correct nodes + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -109,6 +113,7 @@ LangGraph Graphs Package Exports Workflow Classes Format Context Summary With No Files Returns Appropriate Message [Documentation] Test _format_context_summary with empty list + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -124,6 +129,7 @@ Format Context Summary With No Files Returns Appropriate Message Format Context Summary With Multiple Files [Documentation] Test _format_context_summary with multiple Context objects + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -146,6 +152,7 @@ Format Context Summary With Multiple Files Format Context Summary Limits To Five Files [Documentation] Test that _format_context_summary limits to first 5 files + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -165,6 +172,7 @@ Format Context Summary Limits To Five Files Load Context Node Initializes State [Documentation] Test _load_context node execution + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -187,6 +195,7 @@ Load Context Node Initializes State Load Context Node Generates Summary With Sample Contexts [Documentation] Ensure context analysis metadata is populated + [Tags] llm-required ${result}= Run Process ${PYTHON} ${CURDIR}/helper_plan_generation.py Should Contain ${result.stdout} Context analysis summary ready Should Be Equal As Integers ${result.rc} 0 @@ -194,6 +203,7 @@ Load Context Node Generates Summary With Sample Contexts Should Retry Returns Retry When Validation Fails And Retries Available [Documentation] Test _should_retry returns "retry" appropriately + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -214,6 +224,7 @@ Should Retry Returns Retry When Validation Fails And Retries Available Should Retry Returns End When Validation Passes [Documentation] Test _should_retry returns "end" when validation succeeds + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -233,6 +244,7 @@ Should Retry Returns End When Validation Passes Should Retry Returns End When Max Retries Reached [Documentation] Test _should_retry returns "end" when max retries reached + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -277,6 +289,7 @@ Plan Generation State TypedDict Has Correct Structure Validate Node Fails When No Changes Provided [Documentation] Test _validate with no changes + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -294,6 +307,7 @@ Validate Node Fails When No Changes Provided Generate Plan Handles Missing Requirements [Documentation] Test _generate_plan with no requirements + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') @@ -311,6 +325,7 @@ Generate Plan Handles Missing Requirements Generate Plan Infers Test File Name From Prompt [Documentation] Test file name inference for test-related prompts + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path @@ -340,6 +355,7 @@ Generate Plan Infers Test File Name From Prompt Generate Plan Infers Error Handler File Name From Prompt [Documentation] Test file name inference for error/exception prompts + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path @@ -368,6 +384,7 @@ Generate Plan Infers Error Handler File Name From Prompt Workflow Invoke Method Returns Complete State [Documentation] Test that invoke() returns complete workflow state + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path @@ -398,6 +415,7 @@ Workflow Invoke Method Returns Complete State Workflow Stream Method Yields Events [Documentation] Test that stream() yields workflow events + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path @@ -420,6 +438,7 @@ Workflow Stream Method Yields Events Graph Has Checkpointer For State Persistence [Documentation] Verify checkpointer is configured + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}')