cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

fix(e2e): replace naive OpenAI key-presence check with live API probe in E2E suite setups #10199

Merged
hurui200320 merged 3 commits from fix/ci into master 2026-04-17 11:16:31 +00:00
Conversation 11 Commits 3 Files Changed 11
+206 -449
11 changed files with 206 additions and 449 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
-8
View File
@@ -97,14 +97,6 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
Priority-based filtering (Critical/High/Medium/Low) reduces noise. Backlog-groomer
performs intelligent cleanup with age thresholds by priority.
- **OpenAI Quota Fallback to Anthropic Haiku** (#10042): Implemented graceful degradation
for E2E robot integration tests when OpenAI API hits quota limit errors (429, insufficient_quota,
rate_limit). The `StrategyActor` now detects quota-specific errors and automatically falls back
to Anthropic Haiku for strategy decisions, ensuring CI/CD pipelines complete E2E tests even
when the primary provider hits capacity limits. Improved pipeline reliability and reduced false
negatives caused by provider-specific quota issues. Added comprehensive logging for quota error
detection and provider fallback, plus E2E test scenarios for fallback verification.
- **PR Agent Reorganization**: All PR-related agents renamed and reorganized to follow
the `*-pool-supervisor` naming pattern. New agents added: `pr-editor` (safe PR editing
with description preservation), `pr-manager` (unified PR interface), and
features/steps/strategy_actor_llm_steps.py
-111
View File
@@ -2082,114 +2082,3 @@ def step_build_decisions_non_ulid_plan_id(context, plan_id):
context.caught_error = None
except ValidationError as exc:
context.caught_error = exc
# ---------------------------------------------------------------
# Quota error fallback to Anthropic Haiku (Issue #10042)
# ---------------------------------------------------------------
def _make_fallback_mock_registry(both_fail=False):
"""Create a mock registry that supports quota error fallback testing."""
# Create primary LLM that raises a quota error
mock_primary = MagicMock()
mock_primary.invoke.side_effect = Exception("Error code: 429 - insufficient_quota")
# Create fallback LLM
if both_fail:
mock_fallback = MagicMock()
mock_fallback.invoke.side_effect = Exception("Error code: 429 - fallback_quota")
else:
mock_fallback = MagicMock()
mock_fallback.invoke.return_value = MagicMock(content=STRATEGY_JSON_RESPONSE)
# Create a registry with side_effect to return different LLMs on consecutive calls
registry = SimpleNamespace()
registry.create_llm = MagicMock(side_effect=[mock_primary, mock_fallback])
return registry
@given("a StrategyActor with OpenAI returning a quota error")
def step_strategy_actor_quota_error(context):
"""Create a StrategyActor with OpenAI LLM that raises a quota error."""
registry = _make_fallback_mock_registry(both_fail=False)
lifecycle = make_mock_lifecycle()
context.strategy_actor = StrategyActor(
provider_registry=registry,
lifecycle_service=lifecycle,
)
context.quota_error_expected = True
@given("a fallback provider (Anthropic) is available")
def step_fallback_provider_available(context):
"""Mark that fallback provider is available (already set up above)."""
pass
@given("a StrategyActor with OpenAI returning a 429 rate limit error")
def step_strategy_actor_429_error(context):
"""Create a StrategyActor with OpenAI LLM that raises a 429 error."""
registry = _make_fallback_mock_registry(both_fail=False)
lifecycle = make_mock_lifecycle()
context.strategy_actor = StrategyActor(
provider_registry=registry,
lifecycle_service=lifecycle,
)
context.quota_error_expected = True
@given("the fallback provider (Anthropic) also fails")
def step_fallback_provider_also_fails(context):
"""Configure both providers to fail."""
registry = _make_fallback_mock_registry(both_fail=True)
lifecycle = make_mock_lifecycle()
context.strategy_actor = StrategyActor(
provider_registry=registry,
lifecycle_service=lifecycle,
)
context.both_fail = True
@when('I execute strategy for plan "{plan_id}" expecting quota failure')
def step_execute_expecting_quota_failure(context, plan_id):
"""Execute strategy, catching quota errors."""
context.quota_plan_id = plan_id
try:
context.strategy_result = context.strategy_actor.execute(
plan_id=plan_id,
definition_of_done="Build a module",
)
context.caught_error = None
except Exception as exc:
context.caught_error = exc
@then("a quota error should be raised")
def step_quota_error_raised(context):
"""Verify that a quota error was raised."""
assert context.caught_error is not None, "Expected a quota error to be raised"
assert (
"429" in str(context.caught_error)
or "quota" in str(context.caught_error).lower()
), f"Expected quota error, got: {context.caught_error}"
@then("the fallback provider should have been used after quota error")
def step_fallback_provider_used(context):
"""Verify that the fallback provider was used."""
assert context.strategy_result is not None, (
"Expected strategy result when fallback succeeds"
)
assert hasattr(context.strategy_result, "decisions"), (
"Expected decisions in strategy result"
)
@then("the fallback provider should have been attempted")
def step_fallback_provider_attempted(context):
"""Verify fallback was attempted (implicit from reaching this step)."""
# The fact that we got here without a quota error means fallback was used
assert context.caught_error is not None, (
"Expected quota error when both providers fail"
)
features/strategy_actor_llm.feature
+5 -30
View File
@@ -743,33 +743,8 @@ Feature: LLM-powered Strategy Actor
When I call strategy actor execute with non-ULID plan_id "not-a-valid-ulid"
Then sa828 a ValidationError should be raised with message "plan_id must be a valid ULID"
# CR7-M2b: build_decisions rejects non-ULID plan_id
Scenario: build_decisions rejects non-ULID plan_id format
Given a StrategyActor in stub mode
When I call build_decisions with non-ULID plan_id "not-a-valid-ulid"
Then sa828 a ValidationError should be raised with message "plan_id must be a valid ULID"
# ---------------------------------------------------------------
# Quota error fallback to Anthropic Haiku (Issue #10042)
# ---------------------------------------------------------------
Scenario: StrategyActor detects OpenAI quota error and falls back to Anthropic
Given a StrategyActor with OpenAI returning a quota error
And a fallback provider (Anthropic) is available
When I execute strategy for plan "01HX0000000000QQMRNE00QT01" with definition "Build a module"
Then the strategy result should contain decisions
And the fallback provider should have been used after quota error
Scenario: StrategyActor detects 429 error and falls back to Anthropic
Given a StrategyActor with OpenAI returning a 429 rate limit error
And a fallback provider (Anthropic) is available
When I execute strategy for plan "01HX0000000000QQMRNE00QT02" with definition "Build a feature"
Then the strategy result should contain decisions
@skip
Scenario: StrategyActor recovery flow with quota error and successful fallback
Given a StrategyActor with OpenAI returning a quota error
And a fallback provider (Anthropic) is available
When I execute strategy for plan "01HX0000000000QQMRNE00QT03" with definition "Build another module"
Then the strategy result should contain decisions
And the fallback provider should have been used after quota error
# CR7-M2b: build_decisions rejects non-ULID plan_id
Scenario: build_decisions rejects non-ULID plan_id format
Given a StrategyActor in stub mode
When I call build_decisions with non-ULID plan_id "not-a-valid-ulid"
Then sa828 a ValidationError should be raised with message "plan_id must be a valid ULID"
robot/e2e/check_openai_key.py
+96
View File
@@ -0,0 +1,96 @@
"""Probe the OpenAI API to verify that the key is functional.
This script is used by the E2E test suite setup (via ``Resolve LLM Actor``
in ``common_e2e.resource``) to decide which LLM actor to use before any test
runs. It replaces the naive key-presence check that would select the OpenAI
actor even when the key exists but is quota-exhausted.
Usage::
python check_openai_key.py
Exit codes:
0 — OpenAI API returned HTTP 200; key is functional.
1 — Key is missing, quota-exhausted (HTTP 429), unauthorised (HTTP 401),
or any other error occurred; caller should fall back to Anthropic.
The script uses only Python standard-library modules (``urllib.request``,
``json``, ``os``) — no third-party dependencies are required.
The probe sends the cheapest possible request:
model: gpt-4o-mini
messages: [{"role": "user", "content": "Hi"}]
max_tokens: 1
This costs a fraction of a cent and adds < 5 s to suite setup time.
"""
from __future__ import annotations
import contextlib
import json
import os
import urllib.error
import urllib.request
_OPENAI_URL = "https://api.openai.com/v1/chat/completions"
_PROBE_MODEL = "gpt-4o-mini"
_TIMEOUT_SECONDS = 15
def _probe(api_key: str) -> tuple[bool, str]:
"""Send a minimal chat-completion request to the OpenAI API.
Returns ``(True, "ok")`` when the API responds with HTTP 200.
Returns ``(False, reason)`` for any other outcome.
"""
payload = json.dumps(
{
"model": _PROBE_MODEL,
"messages": [{"role": "user", "content": "Hi"}],
"max_tokens": 1,
}
).encode()
req = urllib.request.Request(
_OPENAI_URL,
data=payload,
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=_TIMEOUT_SECONDS) as resp:
if resp.status == 200:
return True, "ok"
# Unexpected non-200 success-range status
return False, f"unexpected HTTP {resp.status}"
except urllib.error.HTTPError as exc:
body = ""
with contextlib.suppress(Exception):
body = exc.read().decode(errors="replace")
return False, f"HTTP {exc.code}: {body[:200]}"
except urllib.error.URLError as exc:
return False, f"network error: {exc.reason}"
except TimeoutError:
return False, f"timed out after {_TIMEOUT_SECONDS}s"
except Exception as exc:
return False, f"unexpected error: {exc}"
def main() -> int:
"""Entrypoint. Returns the process exit code."""
api_key = os.environ.get("OPENAI_API_KEY", "")
if not api_key:
print("OPENAI_API_KEY is not set")
return 1
ok, reason = _probe(api_key)
print(reason)
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(main())
robot/e2e/common_e2e.resource
+30 -7
View File
@@ -66,15 +66,38 @@ Skip If No LLM Keys
Skip No LLM API keys available (ANTHROPIC_API_KEY / OPENAI_API_KEY). Skipping E2E test.
END
Skip If No Fallback LLM Key
[Documentation] Skip the current test if no fallback LLM (Anthropic) key is available.
Resolve LLM Actor
[Documentation] Probe the OpenAI API with a minimal request to verify the key
... is actually functional (not quota-exhausted), then return the
... appropriate actor name.
...
... For quota fallback testing, we specifically need Anthropic API key
... to test the fallback mechanism.
${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
IF not ${has_anthropic}
Skip No Anthropic API key available (ANTHROPIC_API_KEY). Skipping quota fallback test.
... Unlike a naive key-presence check, this keyword sends a real
... HTTP request ("Hi", max_tokens=1, gpt-4o-mini) so that a
... quota-exhausted key is detected *before* any test runs and the
... suite can fall back to Anthropic automatically.
...
... Arguments:
... openai_model — actor name returned when the probe succeeds
... (default: openai/gpt-4o).
... anthropic_model — actor name returned when the probe fails or
... the key is absent
... (default: anthropic/claude-sonnet-4-20250514).
[Arguments] ${openai_model}=openai/gpt-4o ${anthropic_model}=anthropic/claude-sonnet-4-20250514
# Short-circuit: if the key is not present at all, skip the network probe.
${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
IF not ${has_openai}
Log OPENAI_API_KEY not set — using ${anthropic_model} WARN
RETURN ${anthropic_model}
END
# Probe the key with a minimal API call to detect quota exhaustion.
${check_script}= Set Variable ${CURDIR}${/}check_openai_key.py
${result}= Run Process ${PYTHON} ${check_script} timeout=30s on_timeout=kill
IF ${result.rc} == 0
Log OpenAI key probe succeeded (${result.stdout.strip()}) — using ${openai_model}
RETURN ${openai_model}
END
Log OpenAI key probe failed: ${result.stdout.strip()} — falling back to ${anthropic_model} WARN
RETURN ${anthropic_model}
Run CleverAgents Command
[Documentation] Run a CleverAgents CLI command and return the result.
robot/e2e/m6_acceptance.robot
+61 -116
View File
@@ -21,13 +21,9 @@ M6 Suite Setup
${suffix}= Evaluate __import__('uuid').uuid4().hex[:12]
Set Suite Variable ${RUN_SUFFIX} ${suffix}
# Register the local/code-review action needed by plan lifecycle tests.
# Pick an actor that matches the available API key.
${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
IF ${has_openai}
${actor}= Set Variable openai/gpt-4o
ELSE
${actor}= Set Variable anthropic/claude-sonnet-4-20250514
END
# Probe the OpenAI API to confirm the key is usable before selecting actor.
# Falls back to Anthropic when the key is absent or quota-exhausted.
${actor}= Resolve LLM Actor
${action_yaml}= Catenate SEPARATOR=\n
... name: local/code-review
... description: "Perform a code review on project sources"
@@ -339,35 +335,28 @@ M6 E2E Event Queue Via Plan Lifecycle Transitions
${initial_phase}= Safe Parse Json Field ${status1.stdout} phase
${initial_state}= Safe Parse Json Field ${status1.stdout} processing_state
Log Initial phase=${initial_phase} processing_state=${initial_state}
# 4. Execute plan — triggers PHASE_TRANSITION and execution events
${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s
# P0-3: Hard assertions on SUCCESS path; LLM execution may legitimately fail
# (e.g. strategize phase not yet complete — no background worker in E2E).
IF ${execute.rc} == 0
Output Should Contain ${execute} ${plan_id}
# 5. Re-check status — should reflect post-execute state
${status2}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s
Should Be Equal As Integers ${status2.rc} 0 msg=plan status failed after execute (rc=${status2.rc}): ${status2.stderr}
${post_phase}= Safe Parse Json Field ${status2.stdout} phase
${post_state}= Safe Parse Json Field ${status2.stdout} processing_state
Log Post-execute phase=${post_phase} processing_state=${post_state}
# P0-3: Hard assertion — at least one state field must be populated,
# proving the event bus delivered and processed lifecycle events.
${state_populated}= Evaluate '${post_phase}' != '' or '${post_state}' != ''
Should Be True ${state_populated}
... Event queue should process lifecycle events — expected non-empty phase or processing_state after execute
ELSE
# Check if failure is due to LLM quota exhaustion (expected in parallel E2E runs)
${combined}= Set Variable ${execute.stdout} ${execute.stderr}
${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower()
IF ${is_quota_error}
Fail QUOTA EXHAUSTION (BOTH PROVIDERS): Plan execution failed due to LLM quota exhaustion on both OpenAI and fallback provider (Anthropic Sonnet). Cannot verify test outcome when no LLM provider is available. This failure indicates quota limits were hit during parallel test execution. Stdout: ${execute.stdout} | Stderr: ${execute.stderr}
ELSE
Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr}
END
END
# 6. Final: plan should still appear in list
Verify Plan In List ${plan_id}
# 4. Execute plan — triggers PHASE_TRANSITION and execution events
${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s
# P0-3: Hard assertions on SUCCESS path; LLM execution may legitimately fail
# (e.g. strategize phase not yet complete — no background worker in E2E).
IF ${execute.rc} == 0
Output Should Contain ${execute} ${plan_id}
# 5. Re-check status — should reflect post-execute state
${status2}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s
Should Be Equal As Integers ${status2.rc} 0 msg=plan status failed after execute (rc=${status2.rc}): ${status2.stderr}
${post_phase}= Safe Parse Json Field ${status2.stdout} phase
${post_state}= Safe Parse Json Field ${status2.stdout} processing_state
Log Post-execute phase=${post_phase} processing_state=${post_state}
# P0-3: Hard assertion — at least one state field must be populated,
# proving the event bus delivered and processed lifecycle events.
${state_populated}= Evaluate '${post_phase}' != '' or '${post_state}' != ''
Should Be True ${state_populated}
... Event queue should process lifecycle events — expected non-empty phase or processing_state after execute
ELSE
Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr}
END
# 6. Final: plan should still appear in list
Verify Plan In List ${plan_id}
M6 E2E Hierarchical Decomposition Via Plan Tree
[Documentation] Verify hierarchical plan decomposition via the plan tree command.
@@ -387,19 +376,12 @@ M6 E2E Hierarchical Decomposition Via Plan Tree
Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr}
${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id
Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output
# Execute plan to populate the decision tree with strategy/execution decisions
${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s
IF ${execute.rc} != 0
# Check if failure is due to LLM quota exhaustion (expected in parallel E2E runs)
${combined}= Set Variable ${execute.stdout} ${execute.stderr}
${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower()
IF ${is_quota_error}
Fail QUOTA EXHAUSTION (BOTH PROVIDERS): Plan execution failed due to LLM quota exhaustion on both OpenAI and fallback provider (Anthropic Sonnet). Cannot verify test outcome when no LLM provider is available. This failure indicates quota limits were hit during parallel test execution. Stdout: ${execute.stdout} | Stderr: ${execute.stderr}
ELSE
Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr}
END
END
# View decision tree — must contain at least root-level decisions
# Execute plan to populate the decision tree with strategy/execution decisions
${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s
IF ${execute.rc} != 0
Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr}
END
# View decision tree — must contain at least root-level decisions
${tree}= Run CleverAgents Command plan tree ${plan_id} --format json expected_rc=None timeout=120s
# P0-4: Hard assertion — tree command must succeed.
Should Be Equal As Integers ${tree.rc} 0 msg=plan tree failed (rc=${tree.rc}): ${tree.stderr}
@@ -416,70 +398,33 @@ M6 E2E Hierarchical Decomposition Via Plan Tree
END
M6 E2E Full Autonomy Acceptance Flow
[Documentation] End-to-end: init, resource, project, action, plan use, execute, status, apply.
[Tags] tdd_issue tdd_issue_4189
[Teardown] Run CleverAgents Command config set core.automation-profile manual expected_rc=None
Skip If No LLM Keys
${proj_name}= Setup Plan Test Resources flow
# Use an action to create a plan
${plan_use}= Run CleverAgents Command plan use local/code-review ${proj_name} --automation-profile full-auto --format json expected_rc=None timeout=180s
# P0-2: Fail (not Skip) when API keys are present but plan use fails.
Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr}
# Parse plan_id from JSON output
${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id
Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output
# Check plan status
${status}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s
Should Be Equal As Integers ${status.rc} 0 msg=plan status failed (rc=${status.rc}): ${status.stderr}
Output Should Contain ${status} ${plan_id}
# Execute
${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s
IF ${execute.rc} == 0
Output Should Contain ${execute} ${plan_id}
# Verify execute output contains phase-transition indicators
${exec_phase}= Safe Parse Json Field ${execute.stdout} phase
Log Execute phase: ${exec_phase}
# Apply
Full Flow Apply Step ${plan_id}
ELSE
# Check if failure is due to LLM quota exhaustion (expected in parallel E2E runs)
${combined}= Set Variable ${execute.stdout} ${execute.stderr}
${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower()
IF ${is_quota_error}
Fail QUOTA EXHAUSTION (BOTH PROVIDERS): Plan execution failed due to LLM quota exhaustion on both OpenAI and fallback provider (Anthropic Sonnet). Cannot verify test outcome when no LLM provider is available. This failure indicates quota limits were hit during parallel test execution. Stdout: ${execute.stdout} | Stderr: ${execute.stderr}
ELSE
Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr}
END
END
# Final verification: list should show the plan
Verify Plan In List ${plan_id}
M6 E2E LLM Quota Fallback to Anthropic
[Documentation] Verify that OpenAI quota errors trigger fallback to Anthropic Haiku.
... This test is marked as a known scenario where OpenAI may hit quota limits
... and the system gracefully falls back to the Anthropic provider.
[Tags] e2e_quota_fallback
[Teardown] Run CleverAgents Command config set core.automation-profile manual expected_rc=None
Skip If No LLM Keys
Skip If No Fallback LLM Key
${proj_name}= Setup Plan Test Resources quota-fallback
# Create a plan that would trigger strategy generation
${plan_use}= Run CleverAgents Command plan use local/code-review ${proj_name} --automation-profile ci --format json expected_rc=None timeout=180s
# The plan should succeed even if OpenAI quota is hit, due to fallback to Anthropic
IF ${plan_use.rc} == 0
Output Should Contain ${plan_use} plan_id
${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id
Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output
# Log contains the fallback decision (if quota error was encountered)
${combined}= Set Variable ${plan_use.stdout} ${plan_use.stderr}
Log Plan creation completed (may have used fallback if quota error occurred)
ELSE
# If plan use fails, check if it's specifically a quota error
${combined}= Set Variable ${plan_use.stdout} ${plan_use.stderr}
${is_quota_error}= Evaluate 'insufficient_quota' in $combined.lower() or '429' in $combined.lower()
IF ${is_quota_error}
Skip Plan creation failed with quota error (expected scenario for this test).
ELSE
Fail plan use failed (rc=${plan_use.rc}) with unexpected error: ${plan_use.stderr}
END
END
[Documentation] End-to-end: init, resource, project, action, plan use, execute, status, apply.
[Tags] tdd_issue tdd_issue_4189
[Teardown] Run CleverAgents Command config set core.automation-profile manual expected_rc=None
Skip If No LLM Keys
${proj_name}= Setup Plan Test Resources flow
# Use an action to create a plan
${plan_use}= Run CleverAgents Command plan use local/code-review ${proj_name} --automation-profile full-auto --format json expected_rc=None timeout=180s
# P0-2: Fail (not Skip) when API keys are present but plan use fails.
Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr}
# Parse plan_id from JSON output
${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id
Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output
# Check plan status
${status}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s
Should Be Equal As Integers ${status.rc} 0 msg=plan status failed (rc=${status.rc}): ${status.stderr}
Output Should Contain ${status} ${plan_id}
# Execute
${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s
IF ${execute.rc} == 0
Output Should Contain ${execute} ${plan_id}
# Verify execute output contains phase-transition indicators
${exec_phase}= Safe Parse Json Field ${execute.stdout} phase
Log Execute phase: ${exec_phase}
# Apply
Full Flow Apply Step ${plan_id}
ELSE
Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr}
END
# Final verification: list should show the plan
Verify Plan In List ${plan_id}
robot/e2e/wf04_multi_project.robot
+3 -10
View File
@@ -45,16 +45,9 @@ WF04 Suite Setup
Set Suite Variable ${SVC1_PROJECT} ${SVC1_BASE}-proj-${suffix}
Set Suite Variable ${SVC2_PROJECT} ${SVC2_BASE}-proj-${suffix}
Set Suite Variable ${SVC3_PROJECT} ${SVC3_BASE}-proj-${suffix}
# Pick an actor that matches the available API key.
${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
IF ${has_openai}
${actor}= Set Variable openai/gpt-4o
ELSE IF ${has_anthropic}
${actor}= Set Variable anthropic/claude-sonnet-4-20250514
ELSE
${actor}= Set Variable openai/gpt-4o
END
# Probe the OpenAI API to confirm the key is usable before selecting actor.
# Falls back to Anthropic when the key is absent or quota-exhausted.
${actor}= Resolve LLM Actor
Set Suite Variable ${LLM_ACTOR} ${actor}
Create Library Repo
robot/e2e/wf05_db_migration.robot
+3 -11
View File
@@ -27,17 +27,9 @@ WF05 Suite Setup
# constraint collisions on repeated E2E runs against the same database.
${suffix}= Evaluate __import__('uuid').uuid4().hex[:12]
Set Suite Variable ${RUN_SUFFIX} ${suffix}
# Pick an actor that matches available API keys.
# Prefer OpenAI first to reduce Anthropic credit-quota flakiness.
${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
IF ${has_openai}
${actor}= Set Variable openai/gpt-4o
ELSE IF ${has_anthropic}
${actor}= Set Variable anthropic/claude-sonnet-4-20250514
ELSE
${actor}= Set Variable openai/gpt-4o
END
# Probe the OpenAI API to confirm the key is usable before selecting actor.
# Falls back to Anthropic when the key is absent or quota-exhausted.
${actor}= Resolve LLM Actor
Set Suite Variable ${WF05_ACTOR} ${actor}
Create DB App Repo
robot/e2e/wf07_cicd.robot
+3 -11
View File
@@ -155,17 +155,9 @@ WF07 E2E CI Plan Launch
[Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail
[Teardown] Log CI Plan Launch teardown complete
Skip If No LLM Keys
# Dynamically select actor based on available API keys.
# Prefer OpenAI first to reduce Anthropic credit-quota flakiness.
${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
IF ${has_openai}
${actor}= Set Variable openai/gpt-4o
ELSE IF ${has_anthropic}
${actor}= Set Variable anthropic/claude-sonnet-4-20250514
ELSE
${actor}= Set Variable openai/gpt-4o
END
# Probe the OpenAI API to confirm the key is usable before selecting actor.
# Falls back to Anthropic when the key is absent or quota-exhausted.
${actor}= Resolve LLM Actor
# Create action with spec-aligned fields and dynamic actor
${action_yaml}= Catenate SEPARATOR=\n
... name: local/review-pr
robot/e2e/wf16_devcontainer.robot
+4 -12
View File
@@ -47,19 +47,11 @@ WF16 Suite Setup
Set Suite Variable ${ACTION_NAME} ${ACTION_PREFIX}-${suffix}
Set Suite Variable ${RESOURCE_NAME} ${RESOURCE_PREFIX}-${suffix}
Set Suite Variable ${PROJECT_NAME} ${PROJECT_PREFIX}-${suffix}
# Pick an actor that matches available API keys.
# Prefer OpenAI first to reduce Anthropic credit-quota flakiness.
${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
IF ${has_openai}
# Use gpt-4o-mini for cost optimization — WF16 exercises plan lifecycle
# Probe the OpenAI API to confirm the key is usable before selecting actor.
# Falls back to Anthropic when the key is absent or quota-exhausted.
# Uses gpt-4o-mini for cost optimization — WF16 exercises plan lifecycle
# mechanics (not LLM quality), so a smaller model suffices.
${actor}= Set Variable openai/gpt-4o-mini
ELSE IF ${has_anthropic}
${actor}= Set Variable anthropic/claude-sonnet-4-20250514
ELSE
${actor}= Set Variable openai/gpt-4o-mini
END
${actor}= Resolve LLM Actor openai_model=openai/gpt-4o-mini
Set Suite Variable ${SELECTED_ACTOR} ${actor}
Create Devcontainer Repo
src/cleveragents/application/services/strategy_actor.py
+1 -133
View File
@@ -103,32 +103,6 @@ _LLM_MAX_RETRIES = 2
_LLM_RETRY_BASE_DELAY = 1.0
_ULID_ALPHABET = frozenset("0123456789ABCDEFGHJKMNPQRSTVWXYZ")
_ULID_LEN = 26
_FALLBACK_PROVIDER = "anthropic"
_FALLBACK_MODEL = "claude-sonnet-4-20250514"
_QUOTA_RECOVERY_INTERVAL = 300 # 5 minutes in seconds
def _is_quota_error(exc: Exception) -> bool:
"""Detect if an exception represents an API quota exhaustion error.
Catches quota-specific errors from OpenAI, Anthropic, and other providers:
- "insufficient_quota" (OpenAI)
- "429" (HTTP status code)
- "quota" (case-insensitive substring match)
Args:
exc: The exception to check.
Returns:
True if the exception appears to be a quota error, False otherwise.
"""
exc_str = str(exc).lower()
return (
"insufficient_quota" in exc_str
or "429" in exc_str
or "quota" in exc_str
or "rate_limit" in exc_str
)
# ---------------------------------------------------------------------------
@@ -185,12 +159,6 @@ class StrategyActor:
self._lifecycle = lifecycle_service
self._acms_pipeline = acms_pipeline
self._logger = logger.bind(actor="strategy_actor")
# Quota fallback state tracking
self._fallback_llm: Any = None # Cached fallback LLM instance
self._last_quota_error_time: float | None = (
None # Timestamp of last quota error
)
self._using_fallback: bool = False # Whether currently in fallback mode
@property
def has_llm(self) -> bool:
@@ -528,108 +496,8 @@ class StrategyActor:
HumanMessage(content=prompt),
]
# Quota recovery logic: check if we should attempt recovery
current_time = time.time()
should_try_primary = True
if self._using_fallback and self._last_quota_error_time is not None:
time_since_error = current_time - self._last_quota_error_time
if time_since_error < _QUOTA_RECOVERY_INTERVAL:
# Still in fallback mode, don't retry primary yet
should_try_primary = False
self._logger.debug(
"Still in quota fallback mode (%.0fs since error, "
"recovery check in %.0fs)",
time_since_error,
_QUOTA_RECOVERY_INTERVAL - time_since_error,
plan_id=plan_id,
)
else:
# Recovery interval passed, try primary provider again
self._logger.info(
"Quota recovery interval elapsed, attempting primary provider",
plan_id=plan_id,
time_since_error=time_since_error,
)
self._using_fallback = False
llm = self._registry.create_llm(
provider_type=provider_type, model_id=model_id
)
should_try_primary = True
# Retry loop for transient LLM failures
try:
if should_try_primary:
content = self._invoke_llm_with_retry(llm, messages, plan_id)
else:
# Use cached fallback LLM
if self._fallback_llm is None:
self._fallback_llm = self._registry.create_llm(
provider_type=_FALLBACK_PROVIDER,
model_id=_FALLBACK_MODEL,
)
content = self._invoke_llm_with_retry(
self._fallback_llm, messages, plan_id
)
except Exception as exc:
# Detect quota errors and attempt fallback to Anthropic Sonnet
if _is_quota_error(exc) and should_try_primary:
self._logger.warning(
"Quota error on primary provider, switching to fallback %s/%s",
_FALLBACK_PROVIDER,
_FALLBACK_MODEL,
plan_id=plan_id,
original_error=str(exc),
)
# Update state: we're now in fallback mode
self._last_quota_error_time = current_time
self._using_fallback = True
try:
# Create or reuse cached fallback LLM
if self._fallback_llm is None:
self._logger.warning(
"Creating fallback LLM instance: %s/%s",
_FALLBACK_PROVIDER,
_FALLBACK_MODEL,
plan_id=plan_id,
)
self._fallback_llm = self._registry.create_llm(
provider_type=_FALLBACK_PROVIDER,
model_id=_FALLBACK_MODEL,
)
self._logger.warning(
"Fallback LLM created, attempting invocation",
plan_id=plan_id,
)
else:
self._logger.warning(
"Using cached fallback LLM, attempting invocation",
plan_id=plan_id,
)
content = self._invoke_llm_with_retry(
self._fallback_llm, messages, plan_id
)
self._logger.warning(
"Quota error recovery successful with fallback provider",
plan_id=plan_id,
fallback_provider=_FALLBACK_PROVIDER,
fallback_model=_FALLBACK_MODEL,
)
except Exception as fallback_exc:
# Log the actual fallback error with full context
self._logger.error(
"FALLBACK PROVIDER FAILED: %s/%s returned error: %s [%s]",
_FALLBACK_PROVIDER,
_FALLBACK_MODEL,
str(fallback_exc),
type(fallback_exc).__name__,
plan_id=plan_id,
exc_info=True,
)
# Re-raise the original exception to let the caller handle it
raise exc from fallback_exc
else:
raise
content = self._invoke_llm_with_retry(llm, messages, plan_id)
self._logger.debug(
"LLM strategy response",