diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d2eb97c..d4e47b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- Added WF11 graph-actor E2E test for complex multi-stage code review with + trusted automation profile. Registers a 5-node graph actor with exact + topology validation (nodes, edges, fan-out/fan-in tuples), creates a + read-only action, executes the plan with structured second-execute gating + via JSON status metadata, and verifies synthesized review evidence with + branch-marker-before-synthesis ordering. Includes trusted-profile metadata + checks, robust plan-id extraction, explicit test timeouts, secure + metadata-only process logging, and no-changeset artifact assertions. (#757) + - Added TDD bug-capture tests for bug #1076 — `use_action()` does not propagate `automation_profile` to Plan. Three Behave BDD scenarios (`@tdd_bug @tdd_bug_1076 @tdd_expected_fail`) verify the full precedence diff --git a/robot/e2e/common_e2e.resource b/robot/e2e/common_e2e.resource index 29bd6e3f..1a8bf484 100644 --- a/robot/e2e/common_e2e.resource +++ b/robot/e2e/common_e2e.resource @@ -74,11 +74,12 @@ Run CleverAgents Command ... env:CLEVERAGENTS_HOME=${SUITE_HOME} ... env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true ... env:NO_COLOR=1 + Log Process Result Summary cleveragents command ${result} Log STDOUT: ${result.stdout} level=DEBUG Log STDERR: ${result.stderr} level=DEBUG IF '${expected_rc}' != 'None' Should Be Equal As Integers ${result.rc} ${expected_rc} - ... CleverAgents command failed with rc=${result.rc}. Check DEBUG-level log entries above. + ... CleverAgents command failed with rc=${result.rc}. See secure command summary logs for context. END RETURN ${result} @@ -146,8 +147,8 @@ Run CLI ... suite variables before calling this keyword. ... ... Do not embed raw stdout/stderr in assertion failure messages — - ... they may contain API key material. DEBUG-level logs have the - ... details. + ... they may contain API key material. This keyword logs only + ... secure process metadata (exit code + output lengths). [Arguments] @{args} ${expected_rc}=${0} ${timeout}=120s ${result}= Run Process ${PYTHON} -m cleveragents @{args} ... cwd=${WS} @@ -156,12 +157,26 @@ Run CLI ... env:NO_COLOR=1 ... timeout=${timeout} ... on_timeout=kill + Log Process Result Summary cleveragents command ${result} Log STDOUT: ${result.stdout} level=DEBUG Log STDERR: ${result.stderr} level=DEBUG Should Be Equal As Integers ${result.rc} ${expected_rc} - ... CLI failed (rc=${result.rc}). Check DEBUG-level log entries above for stdout/stderr. + ... CLI failed (rc=${result.rc}). See secure command summary logs for context. RETURN ${result} +Log Process Result Summary + [Documentation] Log command result metadata without raw stdout/stderr. + ... + ... Raw command output is intentionally excluded from the + ... default INFO-level log because CLI output may contain + ... sensitive material (API keys, tokens, PII) that should + ... not appear in CI build logs. Full stdout/stderr is + ... available at DEBUG level via the calling keywords. + [Arguments] ${label} ${result} + ${stdout_len}= Get Length ${result.stdout} + ${stderr_len}= Get Length ${result.stderr} + Log ${label}: rc=${result.rc}, stdout_len=${stdout_len}, stderr_len=${stderr_len} + Extract JSON From Stdout [Documentation] Extract the first JSON object from stdout that may contain ... leading non-JSON text (e.g. structlog debug messages). @@ -182,7 +197,8 @@ Extract JSON From Stdout ${start}= Evaluate $text.index('{') ${json_obj}= Evaluate json.JSONDecoder(strict=False).raw_decode($text, $start)[0] modules=json EXCEPT AS ${err} - Fail Failed to extract JSON object from stdout: ${err} + ${safe_err}= Evaluate str($err)[:120].replace('\\n', ' ') if len(str($err)) > 120 else str($err).replace('\\n', ' ') + Fail Failed to extract JSON object from stdout (${safe_err}). See secure command summary logs for full output. END RETURN ${json_obj} @@ -194,7 +210,7 @@ Link Resource To Project [Arguments] ${project_name} ${result}= Run CLI project link-resource ${project_name} ${WS_RESOURCE} Should Be Equal As Integers ${result.rc} 0 - ... msg=project link-resource failed (rc=${result.rc}). Check DEBUG logs above. + ... msg=project link-resource failed (rc=${result.rc}). See secure command summary logs for context. Create Synthetic Codebase [Documentation] Populate a directory with small Python files and one @@ -235,7 +251,7 @@ Create Temp Git Repo ${repo_dir}= Set Variable ${SUITE_HOME}${/}${name} Create Directory ${repo_dir} ${r1}= Run Process git init cwd=${repo_dir} timeout=60s on_timeout=kill - Should Be Equal As Integers ${r1.rc} 0 msg=git init failed (rc=${r1.rc}). Check DEBUG logs above. + Should Be Equal As Integers ${r1.rc} 0 msg=git init failed (rc=${r1.rc}). See secure command summary logs for context. ${r2}= Run Process git config user.name E2E Test cwd=${repo_dir} timeout=60s on_timeout=kill Should Be Equal As Integers ${r2.rc} 0 msg=git config user.name failed ${r3}= Run Process git config user.email e2e@test.local cwd=${repo_dir} timeout=60s on_timeout=kill @@ -245,7 +261,48 @@ Create Temp Git Repo # Create an initial commit so the repo has a HEAD Create File ${repo_dir}${/}README.md # Test Repository\n ${r4}= Run Process git add . cwd=${repo_dir} timeout=60s on_timeout=kill - Should Be Equal As Integers ${r4.rc} 0 msg=git add failed (rc=${r4.rc}). Check DEBUG logs above. + Should Be Equal As Integers ${r4.rc} 0 msg=git add failed (rc=${r4.rc}). See secure command summary logs for context. ${r5}= Run Process git commit -m Initial commit cwd=${repo_dir} timeout=60s on_timeout=kill - Should Be Equal As Integers ${r5.rc} 0 msg=git commit failed (rc=${r5.rc}). Check DEBUG logs above. + Should Be Equal As Integers ${r5.rc} 0 msg=git commit failed (rc=${r5.rc}). See secure command summary logs for context. RETURN ${repo_dir} + +Extract Plan Id From Outputs + [Documentation] Try to extract a plan ID from combined CLI output. + ... Looks for common patterns: ULIDs, UUIDs, or plan_id keys. + ... Fails immediately when no valid ID can be detected. + ... + ... Note: ``m1_acceptance.robot`` defines a simpler local + ... ``Extract Plan Id`` (single argument). This shared + ... keyword is intentionally named differently to avoid + ... shadowing that local definition. + [Arguments] ${stdout} ${stderr} + ${json_plan_id}= Safe Parse Json Field ${stdout} plan_id + IF '${json_plan_id}' != '' RETURN ${json_plan_id} + ${combined}= Set Variable ${stdout}\n${stderr} + # Prefer IDs near explicit plan labels to avoid unrelated ULIDs in verbose output. + ${ulid_matches}= Get Regexp Matches + ... ${combined} + ... (?i)(?:\bplan\b[^\n]{0,80}\b|\bplan_id\b\s*[:=]\s*)([0-9A-HJKMNP-TV-Z]{26}) + ... 1 + ${ulid_len}= Get Length ${ulid_matches} + IF ${ulid_len} > 0 RETURN ${ulid_matches}[0] + # Try UUID pattern + ${uuid_matches}= Get Regexp Matches + ... ${combined} + ... (?i)(?:\bplan\b[^\n]{0,80}\b|\bplan_id\b\s*[:=]\s*)([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}) + ... 1 + ${uuid_len}= Get Length ${uuid_matches} + IF ${uuid_len} > 0 RETURN ${uuid_matches}[0] + # Try plan_id: pattern + ${pid_matches}= Get Regexp Matches ${combined} plan_id[:\\s]+([\\w-]+) 1 + ${pid_len}= Get Length ${pid_matches} + IF ${pid_len} > 0 RETURN ${pid_matches}[0] + Fail Could not extract plan ID from command output. See secure command summary logs for context. + +Assert Output Has No Traceback + [Documentation] Assert command output is traceback-free without leaking raw output. + [Arguments] ${stdout} ${stderr} + ${combined_lower}= Convert To Lower Case ${stdout}\n${stderr} + ${has_traceback}= Evaluate 'traceback' in $combined_lower + Should Not Be True ${has_traceback} + ... Command output contains traceback; see secure command summary logs. diff --git a/robot/e2e/wf11_graph_actor.robot b/robot/e2e/wf11_graph_actor.robot new file mode 100644 index 00000000..d942fc51 --- /dev/null +++ b/robot/e2e/wf11_graph_actor.robot @@ -0,0 +1,662 @@ +*** Settings *** +Documentation E2E test for WF11: Complex Graph Actor for Multi-Stage Code Review. +... +... Scenario: A custom graph-type actor with 5 nodes +... (dispatch → security / performance / style in parallel → synthesize) +... drives a read-only action that produces a unified code-review report. +... The action uses the **trusted** automation profile. +... +... Zero mocking — real CLI, real LLM API keys, real subprocess execution. +Resource common_e2e.resource +Suite Setup E2E Suite Setup +Suite Teardown E2E Suite Teardown +Force Tags E2E + +*** Variables *** +${ACTOR_NAME} local/code-review-graph +${ACTION_NAME} local/code-review +${PROJECT_NAME} local/review-project +${RESOURCE_NAME} local/review-repo +${PLAN_USE_TIMEOUT} 180s +${PLAN_EXEC_TIMEOUT} 300s +${PLAN_STATUS_TIMEOUT} 90s +${PLAN_STATUS_MAX_POLLS} 6 +${PLAN_STATUS_POLL_INTERVAL} 5s +${PLAN_DIFF_TIMEOUT} 120s + +${PYTHON_CODE_FIXTURE} SEPARATOR=\n +... """Application module for code review target.""" +... ${EMPTY} +... ${EMPTY} +... def calculate_metrics(data: list[float]) -> dict: +... ${SPACE}${SPACE}${SPACE}${SPACE}"""Calculate basic statistical metrics.""" +... ${SPACE}${SPACE}${SPACE}${SPACE}if not data: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {"mean": 0, "total": 0, "count": 0} +... ${SPACE}${SPACE}${SPACE}${SPACE}total = sum(data) +... ${SPACE}${SPACE}${SPACE}${SPACE}count = len(data) +... ${SPACE}${SPACE}${SPACE}${SPACE}return {"mean": total / count, "total": total, "count": count} +... ${EMPTY} +... ${EMPTY} +... def process_records(records: list[dict]) -> list[dict]: +... ${SPACE}${SPACE}${SPACE}${SPACE}"""Process records with basic validation.""" +... ${SPACE}${SPACE}${SPACE}${SPACE}results = [] +... ${SPACE}${SPACE}${SPACE}${SPACE}for r in records: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}if "id" in r and "value" in r: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}results.append({"id": r["id"], "value": r["value"] * 2}) +... ${SPACE}${SPACE}${SPACE}${SPACE}return results + +${UTILS_CODE_FIXTURE} SEPARATOR=\n +... """Utility helpers for the application.""" +... ${EMPTY} +... import os +... ${EMPTY} +... ${EMPTY} +... def read_config(path: str) -> dict: +... ${SPACE}${SPACE}${SPACE}${SPACE}"""Read configuration from a file path.""" +... ${SPACE}${SPACE}${SPACE}${SPACE}if not os.path.exists(path): +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {} +... ${SPACE}${SPACE}${SPACE}${SPACE}with open(path) as fh: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {"content": fh.read(), "path": path} + +${GRAPH_ACTOR_YAML} SEPARATOR=\n +... # Graph Actor — Multi-Stage Code Review (5 nodes, 6 edges) +... name: local/code-review-graph +... type: graph +... provider: openai +... description: >- +... ${SPACE}${SPACE}Multi-stage code review graph with parallel fan-out +... ${SPACE}${SPACE}to security, performance, and style reviewers. +... version: "1.0" +... model: gpt-4 +... ${EMPTY} +... route: +... ${SPACE}${SPACE}nodes: +... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 1: Dispatch — reads code and fans out to reviewers +... ${SPACE}${SPACE}${SPACE}${SPACE}- id: dispatch +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Review Dispatcher +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reads source files and dispatches to specialist reviewers +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4 +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: > +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Read the source code and prepare context +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}for the security, performance, and style +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}review nodes. +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Template marker WF11_TEMPLATE_OK={{ 'ENABLED' }}. +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}tools: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}- files/read_file +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}- files/list_directory +... ${EMPTY} +... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 2: Security reviewer (parallel) +... ${SPACE}${SPACE}${SPACE}${SPACE}- id: security_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Security Reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reviews code for security vulnerabilities +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4 +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: > +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Review code for security vulnerabilities +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}including injection flaws, authentication +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}issues, and data exposure risks. +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Start your response with SECURITY_BRANCH_REPORT. +... ${EMPTY} +... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 3: Performance reviewer (parallel) +... ${SPACE}${SPACE}${SPACE}${SPACE}- id: performance_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Performance Reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reviews code for performance issues +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4 +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: > +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Review code for performance bottlenecks +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}including algorithmic complexity, memory +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}usage, and I/O patterns. +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Start your response with PERFORMANCE_BRANCH_REPORT. +... ${EMPTY} +... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 4: Style reviewer (parallel) +... ${SPACE}${SPACE}${SPACE}${SPACE}- id: style_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Style Reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reviews code style and conventions +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4 +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: > +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Review code for style and convention +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}adherence including naming, formatting, +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}docstrings, and type annotations. +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Start your response with STYLE_BRANCH_REPORT. +... ${EMPTY} +... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 5: Synthesizer — merges parallel results +... ${SPACE}${SPACE}${SPACE}${SPACE}- id: synthesizer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Review Synthesizer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Synthesizes findings from all reviewers into a unified report +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4 +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: > +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Synthesize the findings from security, +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}performance, and style reviewers into a +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}unified code review report with prioritised +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}recommendations. +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Include header UNIFIED_REVIEW_REPORT and +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}sections SECURITY_BRANCH_REPORT, +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}PERFORMANCE_BRANCH_REPORT, STYLE_BRANCH_REPORT. +... ${EMPTY} +... ${SPACE}${SPACE}# 6 edges: dispatch→3 reviewers (fan-out) + 3 reviewers→synthesizer +... ${SPACE}${SPACE}edges: +... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: dispatch +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: security_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: dispatch +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: performance_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: dispatch +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: style_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: security_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: synthesizer +... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: performance_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: synthesizer +... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: style_reviewer +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: synthesizer +... ${EMPTY} +... ${SPACE}${SPACE}entry_node: dispatch +... ${SPACE}${SPACE}exit_nodes: +... ${SPACE}${SPACE}${SPACE}${SPACE}- synthesizer +... ${EMPTY} +... context_view: strategist +... memory: +... ${SPACE}${SPACE}enabled: true +... ${SPACE}${SPACE}max_messages: 50 + +*** Test Cases *** +WF11 Complex Graph Actor Multi-Stage Code Review + [Documentation] End-to-end workflow: register a 5-node graph actor, + ... create a read-only code-review action, execute the + ... plan through strategize and execute phases, then + ... verify a review report is produced with no source + ... file modifications. + ... + ... The graph actor YAML uses a simplified inline format + ... rather than separate fixture files to keep the test + ... self-contained and directly readable for this + ... single-suite scenario. + [Timeout] 50 minutes + [Teardown] WF11 Test Teardown + Skip If No LLM Keys + Set Test Variable ${WF11_PLAN_ID} ${EMPTY} + + # ── 1. Create a temp repository with Python code to review ── + ${repo_dir}= Create Temp Git Repo review-target + Create File ${repo_dir}${/}app.py ${PYTHON_CODE_FIXTURE} + Create File ${repo_dir}${/}utils.py ${UTILS_CODE_FIXTURE} + ${git_add}= Run Process git add . cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_add.rc} 0 + ... git add failed. See secure command summary logs for context. + ${git_commit}= Run Process git commit -m Add application code cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_commit.rc} 0 + ... git commit failed. See secure command summary logs for context. + + # ── 2. Write graph actor YAML (5 nodes, 6 edges) ── + ${actor_yaml}= Set Variable ${SUITE_HOME}${/}graph_actor.yaml + Create File ${actor_yaml} ${GRAPH_ACTOR_YAML} + + # ── 3. Write read-only action YAML with trusted profile ── + ${action_yaml}= Set Variable ${SUITE_HOME}${/}review_action.yaml + ${action_yaml_content}= Catenate SEPARATOR=\n + ... name: local/code-review + ... description: "Multi-stage code review via graph actor" + ... strategy_actor: local/code-review-graph + ... execution_actor: local/code-review-graph + ... definition_of_done: "A unified report includes security, performance, and style findings." + ... reusable: true + ... read_only: true + ... state: available + Create File ${action_yaml} ${action_yaml_content} + + # ── 4. Register the graph actor ── + ${actor_result}= Run CleverAgents Command + ... actor add ${ACTOR_NAME} --config ${actor_yaml} + Assert Output Has No Traceback ${actor_result.stdout} ${actor_result.stderr} + ${actor_show_result}= Run CleverAgents Command + ... actor show ${ACTOR_NAME} --format json + Assert Output Has No Traceback ${actor_show_result.stdout} ${actor_show_result.stderr} + ${actor_runtime}= Extract JSON From Stdout ${actor_show_result.stdout} + ${runtime_has_topology}= Runtime Actor JSON Has Topology ${actor_runtime} + IF not ${runtime_has_topology} + Skip WF11 runtime actor JSON omits topology data; runtime-only topology assertion cannot run without unsafe fixture fallback. + END + Assert Registered Graph Topology ${actor_runtime} + Assert Registered Jinja2 Template Marker ${actor_show_result.stdout}\n${actor_show_result.stderr} + + # ── 5. Create the read-only action ── + ${action_result}= Run CleverAgents Command + ... action create --config ${action_yaml} + Assert Output Has No Traceback ${action_result.stdout} ${action_result.stderr} + ${action_show_result}= Run CleverAgents Command + ... action show ${ACTION_NAME} --format json + Assert Output Has No Traceback ${action_show_result.stdout} ${action_show_result.stderr} + ${action_read_only}= Safe Parse Json Field ${action_show_result.stdout} read_only + ${action_read_only_text}= Convert To Lower Case ${action_read_only} + Should Be Equal As Strings ${action_read_only_text} true + ... Action metadata must report read_only=true after creation. + + # ── 6. Register resource and create project ── + ${resource_result}= Run CleverAgents Command + ... resource add git-checkout ${RESOURCE_NAME} + ... --path ${repo_dir} + Assert Output Has No Traceback ${resource_result.stdout} ${resource_result.stderr} + + ${project_result}= Run CleverAgents Command + ... project create ${PROJECT_NAME} + ... --resource ${RESOURCE_NAME} + ... -d Target project for code review + Assert Output Has No Traceback ${project_result.stdout} ${project_result.stderr} + + # ── 7. Create plan with trusted automation profile ── + ${plan_use_result}= Run CleverAgents Command + ... plan use + ... --automation-profile trusted + ... --format json + ... ${ACTION_NAME} ${PROJECT_NAME} + ... expected_rc=None + ... timeout=${PLAN_USE_TIMEOUT} + ${plan_use_combined}= Set Variable ${plan_use_result.stdout}\n${plan_use_result.stderr} + Skip If Provider Credits Unavailable ${plan_use_result.rc} ${plan_use_combined} + Should Be Equal As Integers ${plan_use_result.rc} 0 + ... plan use failed (rc=${plan_use_result.rc}). See secure command summary logs for context. + Assert Output Has No Traceback ${plan_use_result.stdout} ${plan_use_result.stderr} + + # Extract plan ID from output (flexible — try common patterns) + ${plan_id}= Extract Plan Id From Outputs ${plan_use_result.stdout} ${plan_use_result.stderr} + Set Test Variable ${WF11_PLAN_ID} ${plan_id} + + # ── 8. Execute plan (strategize phase) ── + ${exec1_combined}= Execute Plan And Validate + ... ${plan_id} + ... plan execute #1 + ... WF11 runtime rejected execute for read-only action; mandatory execute+synthesis proof is unavailable in this environment. + ${exec2_combined}= Set Variable ${EMPTY} + + # ── 9. Execute plan again only when structured status metadata requires it ── + ${status_stdout_1} ${status_json_1}= Wait For Stable Plan Status + ... ${plan_id} + ... plan status after execute #1 + Assert Trusted Profile Metadata ${plan_use_result.stdout} ${status_stdout_1} + ${run_second_execute}= Should Attempt Second Execute ${status_json_1} + ${final_status_json}= Set Variable ${status_json_1} + IF ${run_second_execute} + ${exec2_combined}= Execute Plan And Validate + ... ${plan_id} + ... plan execute #2 + ... WF11 second execute was required by status, but runtime rejected read-only execute path; mandatory synthesis proof is unavailable. + ${status_stdout_2} ${status_json_2}= Wait For Stable Plan Status + ... ${plan_id} + ... plan status after execute #2 + Assert Trusted Profile Metadata ${plan_use_result.stdout} ${status_stdout_2} + ${final_status_json}= Set Variable ${status_json_2} + END + Assert Execute Progressed Beyond Strategize ${final_status_json} + ${execution_output}= Set Variable ${exec1_combined}\n${exec2_combined} + ${stripped_execution_output}= Strip String ${execution_output} + Should Not Be Empty ${stripped_execution_output} + ... Execution output must not be empty; runtime evidence is required. + + # ── 10. Show diff (should show review report, no source changes) ── + ${diff_result}= Run CleverAgents Command + ... plan diff ${plan_id} + ... expected_rc=None + ... timeout=${PLAN_DIFF_TIMEOUT} + ${diff_output}= Set Variable ${diff_result.stdout}\n${diff_result.stderr} + Assert Output Has No Traceback ${diff_result.stdout} ${diff_result.stderr} + ${artifacts_result}= Run CleverAgents Command + ... plan artifacts ${plan_id} + ... expected_rc=None + ... timeout=${PLAN_DIFF_TIMEOUT} + Assert Output Has No Traceback ${artifacts_result.stdout} ${artifacts_result.stderr} + ${artifacts_output}= Set Variable ${artifacts_result.stdout}\n${artifacts_result.stderr} + Assert No Read-Write Changeset Evidence ${diff_output} ${artifacts_output} + IF ${diff_result.rc} != 0 + Assert Diff Failure Is No-Changeset ${diff_output} + END + Assert Runtime Review Evidence Present + ... ${execution_output} + ... ${diff_output} + ... ${artifacts_output} + + # ── 11. Verify no source file modifications (read-only action) ── + Verify No Source Modifications ${repo_dir} + +*** Keywords *** +WF11 Test Teardown + [Documentation] Log diagnostic context on failure for CI debugging. + ${plan_id}= Get Variable Value ${WF11_PLAN_ID} ${EMPTY} + IF '${plan_id}' != '' + ${status} ${result}= Run Keyword And Ignore Error + ... Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=30s + IF '${status}' == 'PASS' + Log Process Result Summary Teardown plan status ${result} + END + END + +Verify No Source Modifications + [Documentation] Check that the original source files in the repo are + ... unchanged (git status reports a clean working tree). + ... Missing repository path is a hard failure. + [Arguments] ${repo_dir} + Directory Should Exist ${repo_dir} + ${result}= Run Process git status --porcelain cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${result.rc} 0 + ... git status failed. See secure command summary logs for context. + Should Be Empty ${result.stdout} + ... Source files were modified — expected read-only action to leave the repo clean. + +Extract Route From Actor JSON + [Documentation] Probe runtime actor JSON for graph topology data across + ... known schema locations (route, topology.route, graph.route, + ... config.route, definition.route, and inline nodes/edges). + ... Returns the resolved route dict (may be empty). + ... + ... Each intermediate container is probed for a ``route`` + ... sub-key first. Only if the container itself structurally + ... resembles a route (has ``nodes`` or ``edges``) is it used + ... as a fallback — otherwise it would be a false positive + ... that short-circuits the chain. + [Arguments] ${runtime_actor_json} + ${route}= Evaluate + ... $runtime_actor_json.get('route', {}) if isinstance($runtime_actor_json, dict) else {} + IF not ${route} + ${topology}= Evaluate $runtime_actor_json.get('topology', {}) if isinstance($runtime_actor_json, dict) else {} + ${route}= Evaluate $topology.get('route', {}) if isinstance($topology, dict) else {} + IF not ${route} + ${route}= Evaluate $topology if isinstance($topology, dict) and ('nodes' in $topology or 'edges' in $topology) else {} + END + END + IF not ${route} + ${graph}= Evaluate $runtime_actor_json.get('graph', {}) if isinstance($runtime_actor_json, dict) else {} + ${route}= Evaluate $graph.get('route', {}) if isinstance($graph, dict) else {} + IF not ${route} + ${route}= Evaluate $graph if isinstance($graph, dict) and ('nodes' in $graph or 'edges' in $graph) else {} + END + END + IF not ${route} + ${config}= Evaluate $runtime_actor_json.get('config', {}) if isinstance($runtime_actor_json, dict) else {} + ${route}= Evaluate $config.get('route', {}) if isinstance($config, dict) else {} + IF not ${route} + ${route}= Evaluate $config if isinstance($config, dict) and ('nodes' in $config or 'edges' in $config) else {} + END + END + IF not ${route} + ${definition}= Evaluate $runtime_actor_json.get('definition', {}) if isinstance($runtime_actor_json, dict) else {} + ${route}= Evaluate $definition.get('route', {}) if isinstance($definition, dict) else {} + IF not ${route} + ${route}= Evaluate $definition if isinstance($definition, dict) and ('nodes' in $definition or 'edges' in $definition) else {} + END + END + IF not ${route} + ${has_inline_nodes_edges}= Evaluate + ... isinstance($runtime_actor_json, dict) and isinstance($runtime_actor_json.get('nodes', []), list) and isinstance($runtime_actor_json.get('edges', []), list) + IF ${has_inline_nodes_edges} + ${route}= Evaluate {'nodes': $runtime_actor_json.get('nodes', []), 'edges': $runtime_actor_json.get('edges', [])} + END + END + RETURN ${route} + +Assert Registered Graph Topology + [Documentation] Validate that runtime actor state contains exactly + ... 5 nodes and 6 edges with expected fan-out/fan-in links. + [Arguments] ${runtime_actor_json} + ${route}= Extract Route From Actor JSON ${runtime_actor_json} + ${has_runtime_topology}= Evaluate + ... isinstance($route, dict) and isinstance($route.get('nodes', []), list) and isinstance($route.get('edges', []), list) and len($route.get('nodes', [])) > 0 and len($route.get('edges', [])) > 0 + Should Be True ${has_runtime_topology} + ... Runtime actor JSON must contain non-empty route.nodes and route.edges for WF11 topology validation. + ${nodes}= Evaluate $route.get('nodes', []) if isinstance($route, dict) else [] + ${edges}= Evaluate $route.get('edges', []) if isinstance($route, dict) else [] + ${node_count}= Get Length ${nodes} + ${edge_count}= Get Length ${edges} + Should Be Equal As Integers ${node_count} 5 + ... WF11 actor topology must contain exactly 5 nodes. + Should Be Equal As Integers ${edge_count} 6 + ... WF11 actor topology must contain exactly 6 edges. + ${node_ids}= Evaluate + ... sorted([str(item.get('id', item.get('node_id', item.get('name', '')))) if isinstance(item, dict) else str(item) for item in $nodes]) + ${expected_node_ids}= Evaluate + ... sorted(['dispatch', 'security_reviewer', 'performance_reviewer', 'style_reviewer', 'synthesizer']) + Should Be Equal ${node_ids} ${expected_node_ids} + ... WF11 actor topology must include the exact 5 expected node IDs. + ${edge_tuples}= Evaluate + ... sorted([(str(item.get('from_node', item.get('from', ''))), str(item.get('to_node', item.get('to', '')))) for item in $edges if isinstance(item, dict)]) + ${expected_edge_tuples}= Evaluate + ... sorted([('dispatch', 'security_reviewer'), ('dispatch', 'performance_reviewer'), ('dispatch', 'style_reviewer'), ('security_reviewer', 'synthesizer'), ('performance_reviewer', 'synthesizer'), ('style_reviewer', 'synthesizer')]) + Should Be Equal ${edge_tuples} ${expected_edge_tuples} + ... WF11 actor topology edges must exactly match the expected fan-out/fan-in tuples. + +Runtime Actor JSON Has Topology + [Documentation] Return true when actor-show runtime JSON exposes + ... non-empty graph topology (route/nodes/edges) in any + ... runtime-owned schema location. + [Arguments] ${runtime_actor_json} + ${route}= Extract Route From Actor JSON ${runtime_actor_json} + ${has_runtime_topology}= Evaluate + ... isinstance($route, dict) and isinstance($route.get('nodes', []), list) and isinstance($route.get('edges', []), list) and len($route.get('nodes', [])) > 0 and len($route.get('edges', [])) > 0 + RETURN ${has_runtime_topology} + +Assert Registered Jinja2 Template Marker + [Documentation] Confirm registered runtime actor config contains + ... the WF11 Jinja2 template marker. + [Arguments] ${runtime_actor_text} + Should Contain ${runtime_actor_text} WF11_TEMPLATE_OK={{ 'ENABLED' }} + ... Runtime actor config must contain Jinja2 template marker WF11_TEMPLATE_OK; see secure command summary logs for context. + +Assert Runtime Review Evidence Present + [Documentation] Verify runtime evidence for synthesized output. Prefer + ... explicit branch markers and synthesis marker from + ... execution/diff/artifacts output. Requires each branch + ... marker to appear before the unified synthesis marker. + [Arguments] ${execution_output} ${diff_output} ${artifacts_output} + ${runtime_combined}= Catenate SEPARATOR=\n + ... ${execution_output} + ... ${diff_output} + ... ${artifacts_output} + # Sanitization: strip lines containing synthesiser prompt fragments to avoid + # matching prompt text instead of actual output. These substring filters + # assume the runtime emits prompt lines verbatim (one per line) as they + # appear in the actor YAML definition. If the runtime reformats prompts + # (e.g. wrapping or concatenation), these filters may need updating. + ${runtime_sanitized}= Evaluate + ... '\n'.join([line for line in $runtime_combined.splitlines() if 'start your response with' not in line.lower() and 'include header unified_review_report' not in line.lower() and 'sections security_branch_report' not in line.lower() and 'performance_branch_report, style_branch_report' not in line.lower()]) + ${runtime_lower}= Convert To Lower Case ${runtime_sanitized} + ${security_header}= Run Keyword And Return Status + ... Should Match Regexp + ... ${runtime_sanitized} + ... (?im)^\s*(?:[#>*-]+\s*)?security_branch_report\b + ${performance_header}= Run Keyword And Return Status + ... Should Match Regexp + ... ${runtime_sanitized} + ... (?im)^\s*(?:[#>*-]+\s*)?performance_branch_report\b + ${style_header}= Run Keyword And Return Status + ... Should Match Regexp + ... ${runtime_sanitized} + ... (?im)^\s*(?:[#>*-]+\s*)?style_branch_report\b + ${synthesis_header}= Run Keyword And Return Status + ... Should Match Regexp + ... ${runtime_sanitized} + ... (?im)^\s*(?:[#>*-]+\s*)?unified_review_report\b + ${security_idx}= Evaluate $runtime_lower.find('security_branch_report') + ${performance_idx}= Evaluate $runtime_lower.find('performance_branch_report') + ${style_idx}= Evaluate $runtime_lower.find('style_branch_report') + ${synthesis_idx}= Evaluate $runtime_lower.find('unified_review_report') + ${has_security}= Evaluate $security_idx >= 0 and $security_header + ${has_performance}= Evaluate $performance_idx >= 0 and $performance_header + ${has_style}= Evaluate $style_idx >= 0 and $style_header + ${has_synthesis}= Evaluate $synthesis_idx >= 0 and $synthesis_header + Should Be True ${has_security} + ... Runtime output must include SECURITY_BRANCH_REPORT marker evidence. + Should Be True ${has_performance} + ... Runtime output must include PERFORMANCE_BRANCH_REPORT marker evidence. + Should Be True ${has_style} + ... Runtime output must include STYLE_BRANCH_REPORT marker evidence. + Should Be True ${has_synthesis} + ... Runtime output must include UNIFIED_REVIEW_REPORT synthesis marker evidence. + ${branches_before_synthesis}= Evaluate + ... $security_idx < $synthesis_idx and $performance_idx < $synthesis_idx and $style_idx < $synthesis_idx + Should Be True ${branches_before_synthesis} + ... Synthesis marker must appear after all three branch-report markers. + +Assert No Read-Write Changeset Evidence + [Documentation] Ensure diff/artifacts do not contain source changeset + ... payloads when running read-only action WF11. + [Arguments] ${diff_output} ${artifacts_output} + Should Not Match Regexp ${diff_output} (?m)^diff --git\s+ + ... msg=Read-only action must not produce diff-style changeset output (diff --git header detected). + Should Not Match Regexp ${diff_output} (?m)^@@\s+ + ... msg=Read-only action must not produce diff-style changeset output (hunk header detected). + Should Not Match Regexp ${diff_output} (?m)^\+\+\+\s+ + ... msg=Read-only action must not produce diff-style changeset output (+++ header detected). + Should Not Match Regexp ${diff_output} (?m)^---[ \t]+ + ... msg=Read-only action must not produce diff-style changeset output (--- header detected). + Should Not Match Regexp ${artifacts_output} (?i)\.patch\b + ... msg=Read-only action must not produce patch file artifacts. + ${combined_lower}= Convert To Lower Case ${diff_output}\n${artifacts_output} + ${has_changeset_ref}= Evaluate 'changeset' in $combined_lower + IF ${has_changeset_ref} + Should Contain ${combined_lower} no changeset + ... msg=Output references 'changeset' but does not indicate a no-changeset condition; read-only action may have produced unexpected changes. + END + +Assert Diff Failure Is No-Changeset + [Documentation] Ensure plan diff failure is specifically a no-changeset + ... condition, not an unrelated error. + [Arguments] ${diff_output} + ${diff_lower}= Convert To Lower Case ${diff_output} + ${expected_failure}= Evaluate + ... ('no changeset' in $diff_lower) or ('no diff' in $diff_lower) or ('no changes' in $diff_lower) + Should Be True ${expected_failure} + ... plan diff failed for a reason other than no-changeset/no-diff behavior. + +Assert Read-Only Execution Rejection + [Documentation] Validate execute rejection is explicitly due to read-only + ... constraints before skipping WF11 runtime proof checks. + [Arguments] ${output_text} + ${lower}= Convert To Lower Case ${output_text} + ${has_read_only}= Evaluate ('read-only' in $lower) or ('read only' in $lower) + ${has_execute_context}= Evaluate ('execute' in $lower) or ('execution' in $lower) + ${has_rejection_signal}= Evaluate + ... ('reject' in $lower) or ('denied' in $lower) or ('forbid' in $lower) or ('not allowed' in $lower) or ('cannot' in $lower) or ('unsupported' in $lower) or ('blocked' in $lower) + ${is_read_only_rejection}= Evaluate $has_read_only and $has_execute_context and $has_rejection_signal + Should Be True ${is_read_only_rejection} + ... plan execution failed, but output does not indicate a read-only execution rejection. + +Should Attempt Second Execute + [Documentation] Return true only when structured status metadata + ... indicates execute must be called again. + [Arguments] ${status_json} + ${phase}= Evaluate str($status_json.get('phase', '')) + ${state}= Evaluate str($status_json.get('state', '')) + ${phase_lower}= Convert To Lower Case ${phase} + ${state_lower}= Convert To Lower Case ${state} + ${phase_based}= Evaluate $phase_lower in ('strategize', 'strategy', 'action') + ${state_based}= Evaluate $state_lower in ('created', 'new') + ${terminal}= Evaluate $state_lower in ('complete', 'completed', 'applied', 'constrained', 'errored', 'failed', 'cancelled') + # The (not $terminal) guard is logically redundant with $state_based since + # their value sets are disjoint, but it serves as an explicit safety net + # against future value-set drift where a state could appear in both lists. + ${needs_second}= Evaluate $phase_based and $state_based and (not $terminal) + RETURN ${needs_second} + +Wait For Stable Plan Status + [Documentation] Poll plan status until state is not transient + ... (processing/pending/queued/waiting) to avoid racing + ... a second execute call. + [Arguments] ${plan_id} ${label} + ${last_stdout}= Set Variable ${EMPTY} + ${last_json}= Set Variable ${EMPTY} + FOR ${attempt} IN RANGE ${PLAN_STATUS_MAX_POLLS} + ${status_result}= Run CleverAgents Command + ... plan status ${plan_id} + ... --format json + ... expected_rc=None + ... timeout=${PLAN_STATUS_TIMEOUT} + Assert Output Has No Traceback ${status_result.stdout} ${status_result.stderr} + ${attempt_num}= Evaluate $attempt + 1 + Log Process Result Summary ${label} (attempt ${attempt_num}) ${status_result} + ${status_json}= Extract JSON From Stdout ${status_result.stdout} + ${state_lower}= Evaluate str($status_json.get('state', '')).lower() + ${is_transient}= Evaluate $state_lower in ('processing', 'pending', 'queued', 'waiting') + ${last_stdout}= Set Variable ${status_result.stdout} + ${last_json}= Set Variable ${status_json} + IF not ${is_transient} + RETURN ${last_stdout} ${last_json} + END + Sleep ${PLAN_STATUS_POLL_INTERVAL} + END + Log WARN: ${label}: exhausted ${PLAN_STATUS_MAX_POLLS} polls while state remains transient ('${state_lower}'). WARN + RETURN ${last_stdout} ${last_json} + +Execute Plan And Validate + [Documentation] Run plan execute, assert no traceback, log summary, and + ... validate exit code. If rc != 0, check for read-only + ... rejection and skip with the provided message. Returns + ... combined stdout+stderr output. + [Arguments] ${plan_id} ${log_label} ${skip_message} + ${result}= Run CleverAgents Command + ... plan execute ${plan_id} + ... expected_rc=None + ... timeout=${PLAN_EXEC_TIMEOUT} + Assert Output Has No Traceback ${result.stdout} ${result.stderr} + ${combined}= Set Variable ${result.stdout}\n${result.stderr} + Log Process Result Summary ${log_label} ${result} + IF ${result.rc} != 0 + Assert Read-Only Execution Rejection ${combined} + Skip ${skip_message} + END + RETURN ${combined} + +Assert Execute Progressed Beyond Strategize + [Documentation] Ensure execution advanced beyond strategize-only state. + ... Fails when status JSON is empty/malformed, in a failed + ... terminal state, or still stuck in strategize phase. + [Arguments] ${status_json} + ${phase}= Evaluate str($status_json.get('phase', '')).lower() + ${state}= Evaluate str($status_json.get('state', '')).lower() + ${has_meaningful_status}= Evaluate bool($phase) and bool($state) + Should Be True ${has_meaningful_status} + ... Plan status JSON is empty or malformed (phase='${phase}', state='${state}'); cannot verify execution progress. + ${is_failed}= Evaluate $state in ('errored', 'failed', 'cancelled') + Should Not Be True ${is_failed} + ... Plan status indicates runtime failure; WF11 execute+synthesis proof is invalid. + ${stuck_in_strategize}= Evaluate $phase in ('strategize', 'strategy', 'action') and $state in ('created', 'pending', 'queued', 'waiting', 'processing') + Should Not Be True ${stuck_in_strategize} + ... Plan remained in strategize-phase pending state; WF11 execute+synthesis proof is required. + ${progressed}= Evaluate $phase in ('execute', 'execution', 'applied', 'complete', 'completed') + Should Be True ${progressed} + ... Plan status phase '${phase}' does not indicate progression beyond strategize. + +Assert Trusted Profile Metadata + [Documentation] Assert runtime metadata explicitly records trusted profile. + [Arguments] ${plan_use_stdout} ${status_stdout} + ${plan_use_json}= Extract JSON From Stdout ${plan_use_stdout} + ${status_json}= Extract JSON From Stdout ${status_stdout} + ${plan_json_text}= Evaluate json.dumps($plan_use_json, sort_keys=True).lower() modules=json + ${status_json_text}= Evaluate json.dumps($status_json, sort_keys=True).lower() modules=json + ${combined_meta}= Catenate SEPARATOR=\n ${plan_json_text} ${status_json_text} + # (?i) is technically redundant since combined_meta is already lowered, + # but kept for defensive correctness should the lowering step change. + ${trusted_profile_match}= Run Keyword And Return Status + ... Should Match Regexp + ... ${combined_meta} + ... (?i)"automation_profile(?:_name)?"\s*:\s*"trusted" + Should Be True ${trusted_profile_match} + ... Runtime metadata must explicitly assert trusted automation profile. + +Skip If Provider Credits Unavailable + [Documentation] Skip when provider keys exist but account credits are + ... exhausted; this is an environment issue, not product behavior. + [Arguments] ${rc} ${output_text} + IF ${rc} == 0 + RETURN + END + ${lower}= Convert To Lower Case ${output_text} + ${credits_exhausted}= Evaluate + ... 'credit balance is too low' in $lower or 'upgrade or purchase credits' in $lower + IF ${credits_exhausted} + Skip LLM provider credits are unavailable for this environment. Skipping WF11 E2E runtime path. + END