diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ce2744f7..567d698ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,14 +97,6 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Priority-based filtering (Critical/High/Medium/Low) reduces noise. Backlog-groomer performs intelligent cleanup with age thresholds by priority. -- **OpenAI Quota Fallback to Anthropic Haiku** (#10042): Implemented graceful degradation - for E2E robot integration tests when OpenAI API hits quota limit errors (429, insufficient_quota, - rate_limit). The `StrategyActor` now detects quota-specific errors and automatically falls back - to Anthropic Haiku for strategy decisions, ensuring CI/CD pipelines complete E2E tests even - when the primary provider hits capacity limits. Improved pipeline reliability and reduced false - negatives caused by provider-specific quota issues. Added comprehensive logging for quota error - detection and provider fallback, plus E2E test scenarios for fallback verification. - - **PR Agent Reorganization**: All PR-related agents renamed and reorganized to follow the `*-pool-supervisor` naming pattern. New agents added: `pr-editor` (safe PR editing with description preservation), `pr-manager` (unified PR interface), and diff --git a/features/steps/strategy_actor_llm_steps.py b/features/steps/strategy_actor_llm_steps.py index a5dc7ea67..cb5e02855 100644 --- a/features/steps/strategy_actor_llm_steps.py +++ b/features/steps/strategy_actor_llm_steps.py @@ -2082,114 +2082,3 @@ def step_build_decisions_non_ulid_plan_id(context, plan_id): context.caught_error = None except ValidationError as exc: context.caught_error = exc - - -# --------------------------------------------------------------- -# Quota error fallback to Anthropic Haiku (Issue #10042) -# --------------------------------------------------------------- - - -def _make_fallback_mock_registry(both_fail=False): - """Create a mock registry that supports quota error fallback testing.""" - # Create primary LLM that raises a quota error - mock_primary = MagicMock() - mock_primary.invoke.side_effect = Exception("Error code: 429 - insufficient_quota") - - # Create fallback LLM - if both_fail: - mock_fallback = MagicMock() - mock_fallback.invoke.side_effect = Exception("Error code: 429 - fallback_quota") - else: - mock_fallback = MagicMock() - mock_fallback.invoke.return_value = MagicMock(content=STRATEGY_JSON_RESPONSE) - - # Create a registry with side_effect to return different LLMs on consecutive calls - registry = SimpleNamespace() - registry.create_llm = MagicMock(side_effect=[mock_primary, mock_fallback]) - return registry - - -@given("a StrategyActor with OpenAI returning a quota error") -def step_strategy_actor_quota_error(context): - """Create a StrategyActor with OpenAI LLM that raises a quota error.""" - registry = _make_fallback_mock_registry(both_fail=False) - lifecycle = make_mock_lifecycle() - context.strategy_actor = StrategyActor( - provider_registry=registry, - lifecycle_service=lifecycle, - ) - context.quota_error_expected = True - - -@given("a fallback provider (Anthropic) is available") -def step_fallback_provider_available(context): - """Mark that fallback provider is available (already set up above).""" - pass - - -@given("a StrategyActor with OpenAI returning a 429 rate limit error") -def step_strategy_actor_429_error(context): - """Create a StrategyActor with OpenAI LLM that raises a 429 error.""" - registry = _make_fallback_mock_registry(both_fail=False) - lifecycle = make_mock_lifecycle() - context.strategy_actor = StrategyActor( - provider_registry=registry, - lifecycle_service=lifecycle, - ) - context.quota_error_expected = True - - -@given("the fallback provider (Anthropic) also fails") -def step_fallback_provider_also_fails(context): - """Configure both providers to fail.""" - registry = _make_fallback_mock_registry(both_fail=True) - lifecycle = make_mock_lifecycle() - context.strategy_actor = StrategyActor( - provider_registry=registry, - lifecycle_service=lifecycle, - ) - context.both_fail = True - - -@when('I execute strategy for plan "{plan_id}" expecting quota failure') -def step_execute_expecting_quota_failure(context, plan_id): - """Execute strategy, catching quota errors.""" - context.quota_plan_id = plan_id - try: - context.strategy_result = context.strategy_actor.execute( - plan_id=plan_id, - definition_of_done="Build a module", - ) - context.caught_error = None - except Exception as exc: - context.caught_error = exc - - -@then("a quota error should be raised") -def step_quota_error_raised(context): - """Verify that a quota error was raised.""" - assert context.caught_error is not None, "Expected a quota error to be raised" - assert ( - "429" in str(context.caught_error) - or "quota" in str(context.caught_error).lower() - ), f"Expected quota error, got: {context.caught_error}" - - -@then("the fallback provider should have been used after quota error") -def step_fallback_provider_used(context): - """Verify that the fallback provider was used.""" - assert context.strategy_result is not None, ( - "Expected strategy result when fallback succeeds" - ) - assert hasattr(context.strategy_result, "decisions"), ( - "Expected decisions in strategy result" - ) - - -@then("the fallback provider should have been attempted") -def step_fallback_provider_attempted(context): - """Verify fallback was attempted (implicit from reaching this step).""" - # The fact that we got here without a quota error means fallback was used - assert context.caught_error is not None, ( - "Expected quota error when both providers fail" - ) diff --git a/features/strategy_actor_llm.feature b/features/strategy_actor_llm.feature index 32dfe38e8..a36c2b9f6 100644 --- a/features/strategy_actor_llm.feature +++ b/features/strategy_actor_llm.feature @@ -743,33 +743,8 @@ Feature: LLM-powered Strategy Actor When I call strategy actor execute with non-ULID plan_id "not-a-valid-ulid" Then sa828 a ValidationError should be raised with message "plan_id must be a valid ULID" - # CR7-M2b: build_decisions rejects non-ULID plan_id - Scenario: build_decisions rejects non-ULID plan_id format - Given a StrategyActor in stub mode - When I call build_decisions with non-ULID plan_id "not-a-valid-ulid" - Then sa828 a ValidationError should be raised with message "plan_id must be a valid ULID" - - # --------------------------------------------------------------- - # Quota error fallback to Anthropic Haiku (Issue #10042) - # --------------------------------------------------------------- - - Scenario: StrategyActor detects OpenAI quota error and falls back to Anthropic - Given a StrategyActor with OpenAI returning a quota error - And a fallback provider (Anthropic) is available - When I execute strategy for plan "01HX0000000000QQMRNE00QT01" with definition "Build a module" - Then the strategy result should contain decisions - And the fallback provider should have been used after quota error - - Scenario: StrategyActor detects 429 error and falls back to Anthropic - Given a StrategyActor with OpenAI returning a 429 rate limit error - And a fallback provider (Anthropic) is available - When I execute strategy for plan "01HX0000000000QQMRNE00QT02" with definition "Build a feature" - Then the strategy result should contain decisions - - @skip - Scenario: StrategyActor recovery flow with quota error and successful fallback - Given a StrategyActor with OpenAI returning a quota error - And a fallback provider (Anthropic) is available - When I execute strategy for plan "01HX0000000000QQMRNE00QT03" with definition "Build another module" - Then the strategy result should contain decisions - And the fallback provider should have been used after quota error + # CR7-M2b: build_decisions rejects non-ULID plan_id + Scenario: build_decisions rejects non-ULID plan_id format + Given a StrategyActor in stub mode + When I call build_decisions with non-ULID plan_id "not-a-valid-ulid" + Then sa828 a ValidationError should be raised with message "plan_id must be a valid ULID" diff --git a/robot/e2e/check_openai_key.py b/robot/e2e/check_openai_key.py new file mode 100644 index 000000000..c0e8ffb5f --- /dev/null +++ b/robot/e2e/check_openai_key.py @@ -0,0 +1,96 @@ +"""Probe the OpenAI API to verify that the key is functional. + +This script is used by the E2E test suite setup (via ``Resolve LLM Actor`` +in ``common_e2e.resource``) to decide which LLM actor to use before any test +runs. It replaces the naive key-presence check that would select the OpenAI +actor even when the key exists but is quota-exhausted. + +Usage:: + + python check_openai_key.py + +Exit codes: + 0 — OpenAI API returned HTTP 200; key is functional. + 1 — Key is missing, quota-exhausted (HTTP 429), unauthorised (HTTP 401), + or any other error occurred; caller should fall back to Anthropic. + +The script uses only Python standard-library modules (``urllib.request``, +``json``, ``os``) — no third-party dependencies are required. + +The probe sends the cheapest possible request: + model: gpt-4o-mini + messages: [{"role": "user", "content": "Hi"}] + max_tokens: 1 +This costs a fraction of a cent and adds < 5 s to suite setup time. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import urllib.error +import urllib.request + +_OPENAI_URL = "https://api.openai.com/v1/chat/completions" +_PROBE_MODEL = "gpt-4o-mini" +_TIMEOUT_SECONDS = 15 + + +def _probe(api_key: str) -> tuple[bool, str]: + """Send a minimal chat-completion request to the OpenAI API. + + Returns ``(True, "ok")`` when the API responds with HTTP 200. + Returns ``(False, reason)`` for any other outcome. + """ + payload = json.dumps( + { + "model": _PROBE_MODEL, + "messages": [{"role": "user", "content": "Hi"}], + "max_tokens": 1, + } + ).encode() + + req = urllib.request.Request( + _OPENAI_URL, + data=payload, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=_TIMEOUT_SECONDS) as resp: + if resp.status == 200: + return True, "ok" + # Unexpected non-200 success-range status + return False, f"unexpected HTTP {resp.status}" + except urllib.error.HTTPError as exc: + body = "" + with contextlib.suppress(Exception): + body = exc.read().decode(errors="replace") + return False, f"HTTP {exc.code}: {body[:200]}" + except urllib.error.URLError as exc: + return False, f"network error: {exc.reason}" + except TimeoutError: + return False, f"timed out after {_TIMEOUT_SECONDS}s" + except Exception as exc: + return False, f"unexpected error: {exc}" + + +def main() -> int: + """Entrypoint. Returns the process exit code.""" + api_key = os.environ.get("OPENAI_API_KEY", "") + if not api_key: + print("OPENAI_API_KEY is not set") + return 1 + + ok, reason = _probe(api_key) + print(reason) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/robot/e2e/common_e2e.resource b/robot/e2e/common_e2e.resource index d35bc4321..0de7cef3b 100644 --- a/robot/e2e/common_e2e.resource +++ b/robot/e2e/common_e2e.resource @@ -66,15 +66,38 @@ Skip If No LLM Keys Skip No LLM API keys available (ANTHROPIC_API_KEY / OPENAI_API_KEY). Skipping E2E test. END -Skip If No Fallback LLM Key - [Documentation] Skip the current test if no fallback LLM (Anthropic) key is available. +Resolve LLM Actor + [Documentation] Probe the OpenAI API with a minimal request to verify the key + ... is actually functional (not quota-exhausted), then return the + ... appropriate actor name. ... - ... For quota fallback testing, we specifically need Anthropic API key - ... to test the fallback mechanism. - ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) - IF not ${has_anthropic} - Skip No Anthropic API key available (ANTHROPIC_API_KEY). Skipping quota fallback test. + ... Unlike a naive key-presence check, this keyword sends a real + ... HTTP request ("Hi", max_tokens=1, gpt-4o-mini) so that a + ... quota-exhausted key is detected *before* any test runs and the + ... suite can fall back to Anthropic automatically. + ... + ... Arguments: + ... openai_model — actor name returned when the probe succeeds + ... (default: openai/gpt-4o). + ... anthropic_model — actor name returned when the probe fails or + ... the key is absent + ... (default: anthropic/claude-sonnet-4-20250514). + [Arguments] ${openai_model}=openai/gpt-4o ${anthropic_model}=anthropic/claude-sonnet-4-20250514 + # Short-circuit: if the key is not present at all, skip the network probe. + ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) + IF not ${has_openai} + Log OPENAI_API_KEY not set — using ${anthropic_model} WARN + RETURN ${anthropic_model} END + # Probe the key with a minimal API call to detect quota exhaustion. + ${check_script}= Set Variable ${CURDIR}${/}check_openai_key.py + ${result}= Run Process ${PYTHON} ${check_script} timeout=30s on_timeout=kill + IF ${result.rc} == 0 + Log OpenAI key probe succeeded (${result.stdout.strip()}) — using ${openai_model} + RETURN ${openai_model} + END + Log OpenAI key probe failed: ${result.stdout.strip()} — falling back to ${anthropic_model} WARN + RETURN ${anthropic_model} Run CleverAgents Command [Documentation] Run a CleverAgents CLI command and return the result. diff --git a/robot/e2e/m6_acceptance.robot b/robot/e2e/m6_acceptance.robot index 9163960ca..e1f4224a4 100644 --- a/robot/e2e/m6_acceptance.robot +++ b/robot/e2e/m6_acceptance.robot @@ -21,13 +21,9 @@ M6 Suite Setup ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] Set Suite Variable ${RUN_SUFFIX} ${suffix} # Register the local/code-review action needed by plan lifecycle tests. - # Pick an actor that matches the available API key. - ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) - IF ${has_openai} - ${actor}= Set Variable openai/gpt-4o - ELSE - ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 - END + # Probe the OpenAI API to confirm the key is usable before selecting actor. + # Falls back to Anthropic when the key is absent or quota-exhausted. + ${actor}= Resolve LLM Actor ${action_yaml}= Catenate SEPARATOR=\n ... name: local/code-review ... description: "Perform a code review on project sources" @@ -339,35 +335,28 @@ M6 E2E Event Queue Via Plan Lifecycle Transitions ${initial_phase}= Safe Parse Json Field ${status1.stdout} phase ${initial_state}= Safe Parse Json Field ${status1.stdout} processing_state Log Initial phase=${initial_phase} processing_state=${initial_state} - # 4. Execute plan — triggers PHASE_TRANSITION and execution events - ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s - # P0-3: Hard assertions on SUCCESS path; LLM execution may legitimately fail - # (e.g. strategize phase not yet complete — no background worker in E2E). - IF ${execute.rc} == 0 - Output Should Contain ${execute} ${plan_id} - # 5. Re-check status — should reflect post-execute state - ${status2}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s - Should Be Equal As Integers ${status2.rc} 0 msg=plan status failed after execute (rc=${status2.rc}): ${status2.stderr} - ${post_phase}= Safe Parse Json Field ${status2.stdout} phase - ${post_state}= Safe Parse Json Field ${status2.stdout} processing_state - Log Post-execute phase=${post_phase} processing_state=${post_state} - # P0-3: Hard assertion — at least one state field must be populated, - # proving the event bus delivered and processed lifecycle events. - ${state_populated}= Evaluate '${post_phase}' != '' or '${post_state}' != '' - Should Be True ${state_populated} - ... Event queue should process lifecycle events — expected non-empty phase or processing_state after execute - ELSE - # Check if failure is due to LLM quota exhaustion (expected in parallel E2E runs) - ${combined}= Set Variable ${execute.stdout} ${execute.stderr} - ${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower() - IF ${is_quota_error} - Fail QUOTA EXHAUSTION (BOTH PROVIDERS): Plan execution failed due to LLM quota exhaustion on both OpenAI and fallback provider (Anthropic Sonnet). Cannot verify test outcome when no LLM provider is available. This failure indicates quota limits were hit during parallel test execution. Stdout: ${execute.stdout} | Stderr: ${execute.stderr} - ELSE - Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} - END - END - # 6. Final: plan should still appear in list - Verify Plan In List ${plan_id} + # 4. Execute plan — triggers PHASE_TRANSITION and execution events + ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s + # P0-3: Hard assertions on SUCCESS path; LLM execution may legitimately fail + # (e.g. strategize phase not yet complete — no background worker in E2E). + IF ${execute.rc} == 0 + Output Should Contain ${execute} ${plan_id} + # 5. Re-check status — should reflect post-execute state + ${status2}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${status2.rc} 0 msg=plan status failed after execute (rc=${status2.rc}): ${status2.stderr} + ${post_phase}= Safe Parse Json Field ${status2.stdout} phase + ${post_state}= Safe Parse Json Field ${status2.stdout} processing_state + Log Post-execute phase=${post_phase} processing_state=${post_state} + # P0-3: Hard assertion — at least one state field must be populated, + # proving the event bus delivered and processed lifecycle events. + ${state_populated}= Evaluate '${post_phase}' != '' or '${post_state}' != '' + Should Be True ${state_populated} + ... Event queue should process lifecycle events — expected non-empty phase or processing_state after execute + ELSE + Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} + END + # 6. Final: plan should still appear in list + Verify Plan In List ${plan_id} M6 E2E Hierarchical Decomposition Via Plan Tree [Documentation] Verify hierarchical plan decomposition via the plan tree command. @@ -387,19 +376,12 @@ M6 E2E Hierarchical Decomposition Via Plan Tree Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output - # Execute plan to populate the decision tree with strategy/execution decisions - ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s - IF ${execute.rc} != 0 - # Check if failure is due to LLM quota exhaustion (expected in parallel E2E runs) - ${combined}= Set Variable ${execute.stdout} ${execute.stderr} - ${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower() - IF ${is_quota_error} - Fail QUOTA EXHAUSTION (BOTH PROVIDERS): Plan execution failed due to LLM quota exhaustion on both OpenAI and fallback provider (Anthropic Sonnet). Cannot verify test outcome when no LLM provider is available. This failure indicates quota limits were hit during parallel test execution. Stdout: ${execute.stdout} | Stderr: ${execute.stderr} - ELSE - Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} - END - END - # View decision tree — must contain at least root-level decisions + # Execute plan to populate the decision tree with strategy/execution decisions + ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s + IF ${execute.rc} != 0 + Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} + END + # View decision tree — must contain at least root-level decisions ${tree}= Run CleverAgents Command plan tree ${plan_id} --format json expected_rc=None timeout=120s # P0-4: Hard assertion — tree command must succeed. Should Be Equal As Integers ${tree.rc} 0 msg=plan tree failed (rc=${tree.rc}): ${tree.stderr} @@ -416,70 +398,33 @@ M6 E2E Hierarchical Decomposition Via Plan Tree END M6 E2E Full Autonomy Acceptance Flow - [Documentation] End-to-end: init, resource, project, action, plan use, execute, status, apply. - [Tags] tdd_issue tdd_issue_4189 - [Teardown] Run CleverAgents Command config set core.automation-profile manual expected_rc=None - Skip If No LLM Keys - ${proj_name}= Setup Plan Test Resources flow - # Use an action to create a plan - ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj_name} --automation-profile full-auto --format json expected_rc=None timeout=180s - # P0-2: Fail (not Skip) when API keys are present but plan use fails. - Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} - # Parse plan_id from JSON output - ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id - Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output - # Check plan status - ${status}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s - Should Be Equal As Integers ${status.rc} 0 msg=plan status failed (rc=${status.rc}): ${status.stderr} - Output Should Contain ${status} ${plan_id} - # Execute - ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s - IF ${execute.rc} == 0 - Output Should Contain ${execute} ${plan_id} - # Verify execute output contains phase-transition indicators - ${exec_phase}= Safe Parse Json Field ${execute.stdout} phase - Log Execute phase: ${exec_phase} - # Apply - Full Flow Apply Step ${plan_id} - ELSE - # Check if failure is due to LLM quota exhaustion (expected in parallel E2E runs) - ${combined}= Set Variable ${execute.stdout} ${execute.stderr} - ${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower() - IF ${is_quota_error} - Fail QUOTA EXHAUSTION (BOTH PROVIDERS): Plan execution failed due to LLM quota exhaustion on both OpenAI and fallback provider (Anthropic Sonnet). Cannot verify test outcome when no LLM provider is available. This failure indicates quota limits were hit during parallel test execution. Stdout: ${execute.stdout} | Stderr: ${execute.stderr} - ELSE - Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} - END - END - # Final verification: list should show the plan - Verify Plan In List ${plan_id} - -M6 E2E LLM Quota Fallback to Anthropic - [Documentation] Verify that OpenAI quota errors trigger fallback to Anthropic Haiku. - ... This test is marked as a known scenario where OpenAI may hit quota limits - ... and the system gracefully falls back to the Anthropic provider. - [Tags] e2e_quota_fallback - [Teardown] Run CleverAgents Command config set core.automation-profile manual expected_rc=None - Skip If No LLM Keys - Skip If No Fallback LLM Key - ${proj_name}= Setup Plan Test Resources quota-fallback - # Create a plan that would trigger strategy generation - ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj_name} --automation-profile ci --format json expected_rc=None timeout=180s - # The plan should succeed even if OpenAI quota is hit, due to fallback to Anthropic - IF ${plan_use.rc} == 0 - Output Should Contain ${plan_use} plan_id - ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id - Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output - # Log contains the fallback decision (if quota error was encountered) - ${combined}= Set Variable ${plan_use.stdout} ${plan_use.stderr} - Log Plan creation completed (may have used fallback if quota error occurred) - ELSE - # If plan use fails, check if it's specifically a quota error - ${combined}= Set Variable ${plan_use.stdout} ${plan_use.stderr} - ${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower() - IF ${is_quota_error} - Skip Plan creation failed with quota error (expected scenario for this test). - ELSE - Fail plan use failed (rc=${plan_use.rc}) with unexpected error: ${plan_use.stderr} - END - END + [Documentation] End-to-end: init, resource, project, action, plan use, execute, status, apply. + [Tags] tdd_issue tdd_issue_4189 + [Teardown] Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj_name}= Setup Plan Test Resources flow + # Use an action to create a plan + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj_name} --automation-profile full-auto --format json expected_rc=None timeout=180s + # P0-2: Fail (not Skip) when API keys are present but plan use fails. + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + # Parse plan_id from JSON output + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + # Check plan status + ${status}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${status.rc} 0 msg=plan status failed (rc=${status.rc}): ${status.stderr} + Output Should Contain ${status} ${plan_id} + # Execute + ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s + IF ${execute.rc} == 0 + Output Should Contain ${execute} ${plan_id} + # Verify execute output contains phase-transition indicators + ${exec_phase}= Safe Parse Json Field ${execute.stdout} phase + Log Execute phase: ${exec_phase} + # Apply + Full Flow Apply Step ${plan_id} + ELSE + Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} + END + # Final verification: list should show the plan + Verify Plan In List ${plan_id} diff --git a/robot/e2e/wf04_multi_project.robot b/robot/e2e/wf04_multi_project.robot index 862a12c05..cc7e3fafc 100644 --- a/robot/e2e/wf04_multi_project.robot +++ b/robot/e2e/wf04_multi_project.robot @@ -45,16 +45,9 @@ WF04 Suite Setup Set Suite Variable ${SVC1_PROJECT} ${SVC1_BASE}-proj-${suffix} Set Suite Variable ${SVC2_PROJECT} ${SVC2_BASE}-proj-${suffix} Set Suite Variable ${SVC3_PROJECT} ${SVC3_BASE}-proj-${suffix} - # Pick an actor that matches the available API key. - ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) - ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) - IF ${has_openai} - ${actor}= Set Variable openai/gpt-4o - ELSE IF ${has_anthropic} - ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 - ELSE - ${actor}= Set Variable openai/gpt-4o - END + # Probe the OpenAI API to confirm the key is usable before selecting actor. + # Falls back to Anthropic when the key is absent or quota-exhausted. + ${actor}= Resolve LLM Actor Set Suite Variable ${LLM_ACTOR} ${actor} Create Library Repo diff --git a/robot/e2e/wf05_db_migration.robot b/robot/e2e/wf05_db_migration.robot index 9a7c2ec34..6960f6ebe 100644 --- a/robot/e2e/wf05_db_migration.robot +++ b/robot/e2e/wf05_db_migration.robot @@ -27,17 +27,9 @@ WF05 Suite Setup # constraint collisions on repeated E2E runs against the same database. ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] Set Suite Variable ${RUN_SUFFIX} ${suffix} - # Pick an actor that matches available API keys. - # Prefer OpenAI first to reduce Anthropic credit-quota flakiness. - ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) - ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) - IF ${has_openai} - ${actor}= Set Variable openai/gpt-4o - ELSE IF ${has_anthropic} - ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 - ELSE - ${actor}= Set Variable openai/gpt-4o - END + # Probe the OpenAI API to confirm the key is usable before selecting actor. + # Falls back to Anthropic when the key is absent or quota-exhausted. + ${actor}= Resolve LLM Actor Set Suite Variable ${WF05_ACTOR} ${actor} Create DB App Repo diff --git a/robot/e2e/wf07_cicd.robot b/robot/e2e/wf07_cicd.robot index c1e11d883..a74c173f9 100644 --- a/robot/e2e/wf07_cicd.robot +++ b/robot/e2e/wf07_cicd.robot @@ -155,17 +155,9 @@ WF07 E2E CI Plan Launch [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail [Teardown] Log CI Plan Launch teardown complete Skip If No LLM Keys - # Dynamically select actor based on available API keys. - # Prefer OpenAI first to reduce Anthropic credit-quota flakiness. - ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) - ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) - IF ${has_openai} - ${actor}= Set Variable openai/gpt-4o - ELSE IF ${has_anthropic} - ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 - ELSE - ${actor}= Set Variable openai/gpt-4o - END + # Probe the OpenAI API to confirm the key is usable before selecting actor. + # Falls back to Anthropic when the key is absent or quota-exhausted. + ${actor}= Resolve LLM Actor # Create action with spec-aligned fields and dynamic actor ${action_yaml}= Catenate SEPARATOR=\n ... name: local/review-pr diff --git a/robot/e2e/wf16_devcontainer.robot b/robot/e2e/wf16_devcontainer.robot index 286fb045c..2de85ce32 100644 --- a/robot/e2e/wf16_devcontainer.robot +++ b/robot/e2e/wf16_devcontainer.robot @@ -47,19 +47,11 @@ WF16 Suite Setup Set Suite Variable ${ACTION_NAME} ${ACTION_PREFIX}-${suffix} Set Suite Variable ${RESOURCE_NAME} ${RESOURCE_PREFIX}-${suffix} Set Suite Variable ${PROJECT_NAME} ${PROJECT_PREFIX}-${suffix} - # Pick an actor that matches available API keys. - # Prefer OpenAI first to reduce Anthropic credit-quota flakiness. - ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) - ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) - IF ${has_openai} - # Use gpt-4o-mini for cost optimization — WF16 exercises plan lifecycle + # Probe the OpenAI API to confirm the key is usable before selecting actor. + # Falls back to Anthropic when the key is absent or quota-exhausted. + # Uses gpt-4o-mini for cost optimization — WF16 exercises plan lifecycle # mechanics (not LLM quality), so a smaller model suffices. - ${actor}= Set Variable openai/gpt-4o-mini - ELSE IF ${has_anthropic} - ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 - ELSE - ${actor}= Set Variable openai/gpt-4o-mini - END + ${actor}= Resolve LLM Actor openai_model=openai/gpt-4o-mini Set Suite Variable ${SELECTED_ACTOR} ${actor} Create Devcontainer Repo diff --git a/src/cleveragents/application/services/strategy_actor.py b/src/cleveragents/application/services/strategy_actor.py index 5ed594b7f..d46882178 100644 --- a/src/cleveragents/application/services/strategy_actor.py +++ b/src/cleveragents/application/services/strategy_actor.py @@ -103,32 +103,6 @@ _LLM_MAX_RETRIES = 2 _LLM_RETRY_BASE_DELAY = 1.0 _ULID_ALPHABET = frozenset("0123456789ABCDEFGHJKMNPQRSTVWXYZ") _ULID_LEN = 26 -_FALLBACK_PROVIDER = "anthropic" -_FALLBACK_MODEL = "claude-sonnet-4-20250514" -_QUOTA_RECOVERY_INTERVAL = 300 # 5 minutes in seconds - - -def _is_quota_error(exc: Exception) -> bool: - """Detect if an exception represents an API quota exhaustion error. - - Catches quota-specific errors from OpenAI, Anthropic, and other providers: - - "insufficient_quota" (OpenAI) - - "429" (HTTP status code) - - "quota" (case-insensitive substring match) - - Args: - exc: The exception to check. - - Returns: - True if the exception appears to be a quota error, False otherwise. - """ - exc_str = str(exc).lower() - return ( - "insufficient_quota" in exc_str - or "429" in exc_str - or "quota" in exc_str - or "rate_limit" in exc_str - ) # --------------------------------------------------------------------------- @@ -185,12 +159,6 @@ class StrategyActor: self._lifecycle = lifecycle_service self._acms_pipeline = acms_pipeline self._logger = logger.bind(actor="strategy_actor") - # Quota fallback state tracking - self._fallback_llm: Any = None # Cached fallback LLM instance - self._last_quota_error_time: float | None = ( - None # Timestamp of last quota error - ) - self._using_fallback: bool = False # Whether currently in fallback mode @property def has_llm(self) -> bool: @@ -528,108 +496,8 @@ class StrategyActor: HumanMessage(content=prompt), ] - # Quota recovery logic: check if we should attempt recovery - current_time = time.time() - should_try_primary = True - if self._using_fallback and self._last_quota_error_time is not None: - time_since_error = current_time - self._last_quota_error_time - if time_since_error < _QUOTA_RECOVERY_INTERVAL: - # Still in fallback mode, don't retry primary yet - should_try_primary = False - self._logger.debug( - "Still in quota fallback mode (%.0fs since error, " - "recovery check in %.0fs)", - time_since_error, - _QUOTA_RECOVERY_INTERVAL - time_since_error, - plan_id=plan_id, - ) - else: - # Recovery interval passed, try primary provider again - self._logger.info( - "Quota recovery interval elapsed, attempting primary provider", - plan_id=plan_id, - time_since_error=time_since_error, - ) - self._using_fallback = False - llm = self._registry.create_llm( - provider_type=provider_type, model_id=model_id - ) - should_try_primary = True - # Retry loop for transient LLM failures - try: - if should_try_primary: - content = self._invoke_llm_with_retry(llm, messages, plan_id) - else: - # Use cached fallback LLM - if self._fallback_llm is None: - self._fallback_llm = self._registry.create_llm( - provider_type=_FALLBACK_PROVIDER, - model_id=_FALLBACK_MODEL, - ) - content = self._invoke_llm_with_retry( - self._fallback_llm, messages, plan_id - ) - except Exception as exc: - # Detect quota errors and attempt fallback to Anthropic Sonnet - if _is_quota_error(exc) and should_try_primary: - self._logger.warning( - "Quota error on primary provider, switching to fallback %s/%s", - _FALLBACK_PROVIDER, - _FALLBACK_MODEL, - plan_id=plan_id, - original_error=str(exc), - ) - # Update state: we're now in fallback mode - self._last_quota_error_time = current_time - self._using_fallback = True - - try: - # Create or reuse cached fallback LLM - if self._fallback_llm is None: - self._logger.warning( - "Creating fallback LLM instance: %s/%s", - _FALLBACK_PROVIDER, - _FALLBACK_MODEL, - plan_id=plan_id, - ) - self._fallback_llm = self._registry.create_llm( - provider_type=_FALLBACK_PROVIDER, - model_id=_FALLBACK_MODEL, - ) - self._logger.warning( - "Fallback LLM created, attempting invocation", - plan_id=plan_id, - ) - else: - self._logger.warning( - "Using cached fallback LLM, attempting invocation", - plan_id=plan_id, - ) - content = self._invoke_llm_with_retry( - self._fallback_llm, messages, plan_id - ) - self._logger.warning( - "Quota error recovery successful with fallback provider", - plan_id=plan_id, - fallback_provider=_FALLBACK_PROVIDER, - fallback_model=_FALLBACK_MODEL, - ) - except Exception as fallback_exc: - # Log the actual fallback error with full context - self._logger.error( - "FALLBACK PROVIDER FAILED: %s/%s returned error: %s [%s]", - _FALLBACK_PROVIDER, - _FALLBACK_MODEL, - str(fallback_exc), - type(fallback_exc).__name__, - plan_id=plan_id, - exc_info=True, - ) - # Re-raise the original exception to let the caller handle it - raise exc from fallback_exc - else: - raise + content = self._invoke_llm_with_retry(llm, messages, plan_id) self._logger.debug( "LLM strategy response",