|
|
|
@@ -0,0 +1,662 @@
|
|
|
|
|
*** Settings ***
|
|
|
|
|
Documentation E2E test for WF11: Complex Graph Actor for Multi-Stage Code Review.
|
|
|
|
|
...
|
|
|
|
|
... Scenario: A custom graph-type actor with 5 nodes
|
|
|
|
|
... (dispatch → security / performance / style in parallel → synthesize)
|
|
|
|
|
... drives a read-only action that produces a unified code-review report.
|
|
|
|
|
... The action uses the **trusted** automation profile.
|
|
|
|
|
...
|
|
|
|
|
... Zero mocking — real CLI, real LLM API keys, real subprocess execution.
|
|
|
|
|
Resource common_e2e.resource
|
|
|
|
|
Suite Setup E2E Suite Setup
|
|
|
|
|
Suite Teardown E2E Suite Teardown
|
|
|
|
|
Force Tags E2E
|
|
|
|
|
|
|
|
|
|
*** Variables ***
|
|
|
|
|
${ACTOR_NAME} local/code-review-graph
|
|
|
|
|
${ACTION_NAME} local/code-review
|
|
|
|
|
${PROJECT_NAME} local/review-project
|
|
|
|
|
${RESOURCE_NAME} local/review-repo
|
|
|
|
|
${PLAN_USE_TIMEOUT} 180s
|
|
|
|
|
${PLAN_EXEC_TIMEOUT} 300s
|
|
|
|
|
${PLAN_STATUS_TIMEOUT} 90s
|
|
|
|
|
${PLAN_STATUS_MAX_POLLS} 6
|
|
|
|
|
${PLAN_STATUS_POLL_INTERVAL} 5s
|
|
|
|
|
${PLAN_DIFF_TIMEOUT} 120s
|
|
|
|
|
|
|
|
|
|
${PYTHON_CODE_FIXTURE} SEPARATOR=\n
|
|
|
|
|
... """Application module for code review target."""
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... def calculate_metrics(data: list[float]) -> dict:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Calculate basic statistical metrics."""
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}if not data:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {"mean": 0, "total": 0, "count": 0}
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}total = sum(data)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}count = len(data)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}return {"mean": total / count, "total": total, "count": count}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... def process_records(records: list[dict]) -> list[dict]:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Process records with basic validation."""
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}results = []
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}for r in records:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}if "id" in r and "value" in r:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}results.append({"id": r["id"], "value": r["value"] * 2})
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}return results
|
|
|
|
|
|
|
|
|
|
${UTILS_CODE_FIXTURE} SEPARATOR=\n
|
|
|
|
|
... """Utility helpers for the application."""
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... import os
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... def read_config(path: str) -> dict:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Read configuration from a file path."""
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}if not os.path.exists(path):
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {}
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}with open(path) as fh:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {"content": fh.read(), "path": path}
|
|
|
|
|
|
|
|
|
|
${GRAPH_ACTOR_YAML} SEPARATOR=\n
|
|
|
|
|
... # Graph Actor — Multi-Stage Code Review (5 nodes, 6 edges)
|
|
|
|
|
... name: local/code-review-graph
|
|
|
|
|
... type: graph
|
|
|
|
|
... provider: openai
|
|
|
|
|
... description: >-
|
|
|
|
|
... ${SPACE}${SPACE}Multi-stage code review graph with parallel fan-out
|
|
|
|
|
... ${SPACE}${SPACE}to security, performance, and style reviewers.
|
|
|
|
|
... version: "1.0"
|
|
|
|
|
... model: gpt-4
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... route:
|
|
|
|
|
... ${SPACE}${SPACE}nodes:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 1: Dispatch — reads code and fans out to reviewers
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- id: dispatch
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Review Dispatcher
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reads source files and dispatches to specialist reviewers
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: >
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Read the source code and prepare context
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}for the security, performance, and style
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}review nodes.
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Template marker WF11_TEMPLATE_OK={{ 'ENABLED' }}.
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}tools:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}- files/read_file
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}- files/list_directory
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 2: Security reviewer (parallel)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- id: security_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Security Reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reviews code for security vulnerabilities
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: >
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Review code for security vulnerabilities
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}including injection flaws, authentication
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}issues, and data exposure risks.
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Start your response with SECURITY_BRANCH_REPORT.
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 3: Performance reviewer (parallel)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- id: performance_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Performance Reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reviews code for performance issues
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: >
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Review code for performance bottlenecks
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}including algorithmic complexity, memory
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}usage, and I/O patterns.
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Start your response with PERFORMANCE_BRANCH_REPORT.
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 4: Style reviewer (parallel)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- id: style_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Style Reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Reviews code style and conventions
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: >
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Review code for style and convention
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}adherence including naming, formatting,
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}docstrings, and type annotations.
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Start your response with STYLE_BRANCH_REPORT.
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}# Node 5: Synthesizer — merges parallel results
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- id: synthesizer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type: agent
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name: Review Synthesizer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}description: Synthesizes findings from all reviewers into a unified report
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}config:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}model: gpt-4
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}prompt: >
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Synthesize the findings from security,
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}performance, and style reviewers into a
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}unified code review report with prioritised
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}recommendations.
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}Include header UNIFIED_REVIEW_REPORT and
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}sections SECURITY_BRANCH_REPORT,
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}PERFORMANCE_BRANCH_REPORT, STYLE_BRANCH_REPORT.
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${SPACE}${SPACE}# 6 edges: dispatch→3 reviewers (fan-out) + 3 reviewers→synthesizer
|
|
|
|
|
... ${SPACE}${SPACE}edges:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: dispatch
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: security_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: dispatch
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: performance_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: dispatch
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: style_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: security_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: synthesizer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: performance_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: synthesizer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- from_node: style_reviewer
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}to_node: synthesizer
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${SPACE}${SPACE}entry_node: dispatch
|
|
|
|
|
... ${SPACE}${SPACE}exit_nodes:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}- synthesizer
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... context_view: strategist
|
|
|
|
|
... memory:
|
|
|
|
|
... ${SPACE}${SPACE}enabled: true
|
|
|
|
|
... ${SPACE}${SPACE}max_messages: 50
|
|
|
|
|
|
|
|
|
|
*** Test Cases ***
|
|
|
|
|
WF11 Complex Graph Actor Multi-Stage Code Review
|
|
|
|
|
[Documentation] End-to-end workflow: register a 5-node graph actor,
|
|
|
|
|
... create a read-only code-review action, execute the
|
|
|
|
|
... plan through strategize and execute phases, then
|
|
|
|
|
... verify a review report is produced with no source
|
|
|
|
|
... file modifications.
|
|
|
|
|
...
|
|
|
|
|
... The graph actor YAML uses a simplified inline format
|
|
|
|
|
... rather than separate fixture files to keep the test
|
|
|
|
|
... self-contained and directly readable for this
|
|
|
|
|
... single-suite scenario.
|
|
|
|
|
[Timeout] 50 minutes
|
|
|
|
|
[Teardown] WF11 Test Teardown
|
|
|
|
|
Skip If No LLM Keys
|
|
|
|
|
Set Test Variable ${WF11_PLAN_ID} ${EMPTY}
|
|
|
|
|
|
|
|
|
|
# ── 1. Create a temp repository with Python code to review ──
|
|
|
|
|
${repo_dir}= Create Temp Git Repo review-target
|
|
|
|
|
Create File ${repo_dir}${/}app.py ${PYTHON_CODE_FIXTURE}
|
|
|
|
|
Create File ${repo_dir}${/}utils.py ${UTILS_CODE_FIXTURE}
|
|
|
|
|
${git_add}= Run Process git add . cwd=${repo_dir} timeout=60s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${git_add.rc} 0
|
|
|
|
|
... git add failed. See secure command summary logs for context.
|
|
|
|
|
${git_commit}= Run Process git commit -m Add application code cwd=${repo_dir} timeout=60s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${git_commit.rc} 0
|
|
|
|
|
... git commit failed. See secure command summary logs for context.
|
|
|
|
|
|
|
|
|
|
# ── 2. Write graph actor YAML (5 nodes, 6 edges) ──
|
|
|
|
|
${actor_yaml}= Set Variable ${SUITE_HOME}${/}graph_actor.yaml
|
|
|
|
|
Create File ${actor_yaml} ${GRAPH_ACTOR_YAML}
|
|
|
|
|
|
|
|
|
|
# ── 3. Write read-only action YAML with trusted profile ──
|
|
|
|
|
${action_yaml}= Set Variable ${SUITE_HOME}${/}review_action.yaml
|
|
|
|
|
${action_yaml_content}= Catenate SEPARATOR=\n
|
|
|
|
|
... name: local/code-review
|
|
|
|
|
... description: "Multi-stage code review via graph actor"
|
|
|
|
|
... strategy_actor: local/code-review-graph
|
|
|
|
|
... execution_actor: local/code-review-graph
|
|
|
|
|
... definition_of_done: "A unified report includes security, performance, and style findings."
|
|
|
|
|
... reusable: true
|
|
|
|
|
... read_only: true
|
|
|
|
|
... state: available
|
|
|
|
|
Create File ${action_yaml} ${action_yaml_content}
|
|
|
|
|
|
|
|
|
|
# ── 4. Register the graph actor ──
|
|
|
|
|
${actor_result}= Run CleverAgents Command
|
|
|
|
|
... actor add ${ACTOR_NAME} --config ${actor_yaml}
|
|
|
|
|
Assert Output Has No Traceback ${actor_result.stdout} ${actor_result.stderr}
|
|
|
|
|
${actor_show_result}= Run CleverAgents Command
|
|
|
|
|
... actor show ${ACTOR_NAME} --format json
|
|
|
|
|
Assert Output Has No Traceback ${actor_show_result.stdout} ${actor_show_result.stderr}
|
|
|
|
|
${actor_runtime}= Extract JSON From Stdout ${actor_show_result.stdout}
|
|
|
|
|
${runtime_has_topology}= Runtime Actor JSON Has Topology ${actor_runtime}
|
|
|
|
|
IF not ${runtime_has_topology}
|
|
|
|
|
Skip WF11 runtime actor JSON omits topology data; runtime-only topology assertion cannot run without unsafe fixture fallback.
|
|
|
|
|
END
|
|
|
|
|
Assert Registered Graph Topology ${actor_runtime}
|
|
|
|
|
Assert Registered Jinja2 Template Marker ${actor_show_result.stdout}\n${actor_show_result.stderr}
|
|
|
|
|
|
|
|
|
|
# ── 5. Create the read-only action ──
|
|
|
|
|
${action_result}= Run CleverAgents Command
|
|
|
|
|
... action create --config ${action_yaml}
|
|
|
|
|
Assert Output Has No Traceback ${action_result.stdout} ${action_result.stderr}
|
|
|
|
|
${action_show_result}= Run CleverAgents Command
|
|
|
|
|
... action show ${ACTION_NAME} --format json
|
|
|
|
|
Assert Output Has No Traceback ${action_show_result.stdout} ${action_show_result.stderr}
|
|
|
|
|
${action_read_only}= Safe Parse Json Field ${action_show_result.stdout} read_only
|
|
|
|
|
${action_read_only_text}= Convert To Lower Case ${action_read_only}
|
|
|
|
|
Should Be Equal As Strings ${action_read_only_text} true
|
|
|
|
|
... Action metadata must report read_only=true after creation.
|
|
|
|
|
|
|
|
|
|
# ── 6. Register resource and create project ──
|
|
|
|
|
${resource_result}= Run CleverAgents Command
|
|
|
|
|
... resource add git-checkout ${RESOURCE_NAME}
|
|
|
|
|
... --path ${repo_dir}
|
|
|
|
|
Assert Output Has No Traceback ${resource_result.stdout} ${resource_result.stderr}
|
|
|
|
|
|
|
|
|
|
${project_result}= Run CleverAgents Command
|
|
|
|
|
... project create ${PROJECT_NAME}
|
|
|
|
|
... --resource ${RESOURCE_NAME}
|
|
|
|
|
... -d Target project for code review
|
|
|
|
|
Assert Output Has No Traceback ${project_result.stdout} ${project_result.stderr}
|
|
|
|
|
|
|
|
|
|
# ── 7. Create plan with trusted automation profile ──
|
|
|
|
|
${plan_use_result}= Run CleverAgents Command
|
|
|
|
|
... plan use
|
|
|
|
|
... --automation-profile trusted
|
|
|
|
|
... --format json
|
|
|
|
|
... ${ACTION_NAME} ${PROJECT_NAME}
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
... timeout=${PLAN_USE_TIMEOUT}
|
|
|
|
|
${plan_use_combined}= Set Variable ${plan_use_result.stdout}\n${plan_use_result.stderr}
|
|
|
|
|
Skip If Provider Credits Unavailable ${plan_use_result.rc} ${plan_use_combined}
|
|
|
|
|
Should Be Equal As Integers ${plan_use_result.rc} 0
|
|
|
|
|
... plan use failed (rc=${plan_use_result.rc}). See secure command summary logs for context.
|
|
|
|
|
Assert Output Has No Traceback ${plan_use_result.stdout} ${plan_use_result.stderr}
|
|
|
|
|
|
|
|
|
|
# Extract plan ID from output (flexible — try common patterns)
|
|
|
|
|
${plan_id}= Extract Plan Id From Outputs ${plan_use_result.stdout} ${plan_use_result.stderr}
|
|
|
|
|
Set Test Variable ${WF11_PLAN_ID} ${plan_id}
|
|
|
|
|
|
|
|
|
|
# ── 8. Execute plan (strategize phase) ──
|
|
|
|
|
${exec1_combined}= Execute Plan And Validate
|
|
|
|
|
... ${plan_id}
|
|
|
|
|
... plan execute #1
|
|
|
|
|
... WF11 runtime rejected execute for read-only action; mandatory execute+synthesis proof is unavailable in this environment.
|
|
|
|
|
${exec2_combined}= Set Variable ${EMPTY}
|
|
|
|
|
|
|
|
|
|
# ── 9. Execute plan again only when structured status metadata requires it ──
|
|
|
|
|
${status_stdout_1} ${status_json_1}= Wait For Stable Plan Status
|
|
|
|
|
... ${plan_id}
|
|
|
|
|
... plan status after execute #1
|
|
|
|
|
Assert Trusted Profile Metadata ${plan_use_result.stdout} ${status_stdout_1}
|
|
|
|
|
${run_second_execute}= Should Attempt Second Execute ${status_json_1}
|
|
|
|
|
${final_status_json}= Set Variable ${status_json_1}
|
|
|
|
|
IF ${run_second_execute}
|
|
|
|
|
${exec2_combined}= Execute Plan And Validate
|
|
|
|
|
... ${plan_id}
|
|
|
|
|
... plan execute #2
|
|
|
|
|
... WF11 second execute was required by status, but runtime rejected read-only execute path; mandatory synthesis proof is unavailable.
|
|
|
|
|
${status_stdout_2} ${status_json_2}= Wait For Stable Plan Status
|
|
|
|
|
... ${plan_id}
|
|
|
|
|
... plan status after execute #2
|
|
|
|
|
Assert Trusted Profile Metadata ${plan_use_result.stdout} ${status_stdout_2}
|
|
|
|
|
${final_status_json}= Set Variable ${status_json_2}
|
|
|
|
|
END
|
|
|
|
|
Assert Execute Progressed Beyond Strategize ${final_status_json}
|
|
|
|
|
${execution_output}= Set Variable ${exec1_combined}\n${exec2_combined}
|
|
|
|
|
${stripped_execution_output}= Strip String ${execution_output}
|
|
|
|
|
Should Not Be Empty ${stripped_execution_output}
|
|
|
|
|
... Execution output must not be empty; runtime evidence is required.
|
|
|
|
|
|
|
|
|
|
# ── 10. Show diff (should show review report, no source changes) ──
|
|
|
|
|
${diff_result}= Run CleverAgents Command
|
|
|
|
|
... plan diff ${plan_id}
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
... timeout=${PLAN_DIFF_TIMEOUT}
|
|
|
|
|
${diff_output}= Set Variable ${diff_result.stdout}\n${diff_result.stderr}
|
|
|
|
|
Assert Output Has No Traceback ${diff_result.stdout} ${diff_result.stderr}
|
|
|
|
|
${artifacts_result}= Run CleverAgents Command
|
|
|
|
|
... plan artifacts ${plan_id}
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
... timeout=${PLAN_DIFF_TIMEOUT}
|
|
|
|
|
Assert Output Has No Traceback ${artifacts_result.stdout} ${artifacts_result.stderr}
|
|
|
|
|
${artifacts_output}= Set Variable ${artifacts_result.stdout}\n${artifacts_result.stderr}
|
|
|
|
|
Assert No Read-Write Changeset Evidence ${diff_output} ${artifacts_output}
|
|
|
|
|
IF ${diff_result.rc} != 0
|
|
|
|
|
Assert Diff Failure Is No-Changeset ${diff_output}
|
|
|
|
|
END
|
|
|
|
|
Assert Runtime Review Evidence Present
|
|
|
|
|
... ${execution_output}
|
|
|
|
|
... ${diff_output}
|
|
|
|
|
... ${artifacts_output}
|
|
|
|
|
|
|
|
|
|
# ── 11. Verify no source file modifications (read-only action) ──
|
|
|
|
|
Verify No Source Modifications ${repo_dir}
|
|
|
|
|
|
|
|
|
|
*** Keywords ***
|
|
|
|
|
WF11 Test Teardown
|
|
|
|
|
[Documentation] Log diagnostic context on failure for CI debugging.
|
|
|
|
|
${plan_id}= Get Variable Value ${WF11_PLAN_ID} ${EMPTY}
|
|
|
|
|
IF '${plan_id}' != ''
|
|
|
|
|
${status} ${result}= Run Keyword And Ignore Error
|
|
|
|
|
... Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=30s
|
|
|
|
|
IF '${status}' == 'PASS'
|
|
|
|
|
Log Process Result Summary Teardown plan status ${result}
|
|
|
|
|
END
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
Verify No Source Modifications
|
|
|
|
|
[Documentation] Check that the original source files in the repo are
|
|
|
|
|
... unchanged (git status reports a clean working tree).
|
|
|
|
|
... Missing repository path is a hard failure.
|
|
|
|
|
[Arguments] ${repo_dir}
|
|
|
|
|
Directory Should Exist ${repo_dir}
|
|
|
|
|
${result}= Run Process git status --porcelain cwd=${repo_dir} timeout=60s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${result.rc} 0
|
|
|
|
|
... git status failed. See secure command summary logs for context.
|
|
|
|
|
Should Be Empty ${result.stdout}
|
|
|
|
|
... Source files were modified — expected read-only action to leave the repo clean.
|
|
|
|
|
|
|
|
|
|
Extract Route From Actor JSON
|
|
|
|
|
[Documentation] Probe runtime actor JSON for graph topology data across
|
|
|
|
|
... known schema locations (route, topology.route, graph.route,
|
|
|
|
|
... config.route, definition.route, and inline nodes/edges).
|
|
|
|
|
... Returns the resolved route dict (may be empty).
|
|
|
|
|
...
|
|
|
|
|
... Each intermediate container is probed for a ``route``
|
|
|
|
|
... sub-key first. Only if the container itself structurally
|
|
|
|
|
... resembles a route (has ``nodes`` or ``edges``) is it used
|
|
|
|
|
... as a fallback — otherwise it would be a false positive
|
|
|
|
|
... that short-circuits the chain.
|
|
|
|
|
[Arguments] ${runtime_actor_json}
|
|
|
|
|
${route}= Evaluate
|
|
|
|
|
... $runtime_actor_json.get('route', {}) if isinstance($runtime_actor_json, dict) else {}
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${topology}= Evaluate $runtime_actor_json.get('topology', {}) if isinstance($runtime_actor_json, dict) else {}
|
|
|
|
|
${route}= Evaluate $topology.get('route', {}) if isinstance($topology, dict) else {}
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${route}= Evaluate $topology if isinstance($topology, dict) and ('nodes' in $topology or 'edges' in $topology) else {}
|
|
|
|
|
END
|
|
|
|
|
END
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${graph}= Evaluate $runtime_actor_json.get('graph', {}) if isinstance($runtime_actor_json, dict) else {}
|
|
|
|
|
${route}= Evaluate $graph.get('route', {}) if isinstance($graph, dict) else {}
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${route}= Evaluate $graph if isinstance($graph, dict) and ('nodes' in $graph or 'edges' in $graph) else {}
|
|
|
|
|
END
|
|
|
|
|
END
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${config}= Evaluate $runtime_actor_json.get('config', {}) if isinstance($runtime_actor_json, dict) else {}
|
|
|
|
|
${route}= Evaluate $config.get('route', {}) if isinstance($config, dict) else {}
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${route}= Evaluate $config if isinstance($config, dict) and ('nodes' in $config or 'edges' in $config) else {}
|
|
|
|
|
END
|
|
|
|
|
END
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${definition}= Evaluate $runtime_actor_json.get('definition', {}) if isinstance($runtime_actor_json, dict) else {}
|
|
|
|
|
${route}= Evaluate $definition.get('route', {}) if isinstance($definition, dict) else {}
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${route}= Evaluate $definition if isinstance($definition, dict) and ('nodes' in $definition or 'edges' in $definition) else {}
|
|
|
|
|
END
|
|
|
|
|
END
|
|
|
|
|
IF not ${route}
|
|
|
|
|
${has_inline_nodes_edges}= Evaluate
|
|
|
|
|
... isinstance($runtime_actor_json, dict) and isinstance($runtime_actor_json.get('nodes', []), list) and isinstance($runtime_actor_json.get('edges', []), list)
|
|
|
|
|
IF ${has_inline_nodes_edges}
|
|
|
|
|
${route}= Evaluate {'nodes': $runtime_actor_json.get('nodes', []), 'edges': $runtime_actor_json.get('edges', [])}
|
|
|
|
|
END
|
|
|
|
|
END
|
|
|
|
|
RETURN ${route}
|
|
|
|
|
|
|
|
|
|
Assert Registered Graph Topology
|
|
|
|
|
[Documentation] Validate that runtime actor state contains exactly
|
|
|
|
|
... 5 nodes and 6 edges with expected fan-out/fan-in links.
|
|
|
|
|
[Arguments] ${runtime_actor_json}
|
|
|
|
|
${route}= Extract Route From Actor JSON ${runtime_actor_json}
|
|
|
|
|
${has_runtime_topology}= Evaluate
|
|
|
|
|
... isinstance($route, dict) and isinstance($route.get('nodes', []), list) and isinstance($route.get('edges', []), list) and len($route.get('nodes', [])) > 0 and len($route.get('edges', [])) > 0
|
|
|
|
|
Should Be True ${has_runtime_topology}
|
|
|
|
|
... Runtime actor JSON must contain non-empty route.nodes and route.edges for WF11 topology validation.
|
|
|
|
|
${nodes}= Evaluate $route.get('nodes', []) if isinstance($route, dict) else []
|
|
|
|
|
${edges}= Evaluate $route.get('edges', []) if isinstance($route, dict) else []
|
|
|
|
|
${node_count}= Get Length ${nodes}
|
|
|
|
|
${edge_count}= Get Length ${edges}
|
|
|
|
|
Should Be Equal As Integers ${node_count} 5
|
|
|
|
|
... WF11 actor topology must contain exactly 5 nodes.
|
|
|
|
|
Should Be Equal As Integers ${edge_count} 6
|
|
|
|
|
... WF11 actor topology must contain exactly 6 edges.
|
|
|
|
|
${node_ids}= Evaluate
|
|
|
|
|
... sorted([str(item.get('id', item.get('node_id', item.get('name', '')))) if isinstance(item, dict) else str(item) for item in $nodes])
|
|
|
|
|
${expected_node_ids}= Evaluate
|
|
|
|
|
... sorted(['dispatch', 'security_reviewer', 'performance_reviewer', 'style_reviewer', 'synthesizer'])
|
|
|
|
|
Should Be Equal ${node_ids} ${expected_node_ids}
|
|
|
|
|
... WF11 actor topology must include the exact 5 expected node IDs.
|
|
|
|
|
${edge_tuples}= Evaluate
|
|
|
|
|
... sorted([(str(item.get('from_node', item.get('from', ''))), str(item.get('to_node', item.get('to', '')))) for item in $edges if isinstance(item, dict)])
|
|
|
|
|
${expected_edge_tuples}= Evaluate
|
|
|
|
|
... sorted([('dispatch', 'security_reviewer'), ('dispatch', 'performance_reviewer'), ('dispatch', 'style_reviewer'), ('security_reviewer', 'synthesizer'), ('performance_reviewer', 'synthesizer'), ('style_reviewer', 'synthesizer')])
|
|
|
|
|
Should Be Equal ${edge_tuples} ${expected_edge_tuples}
|
|
|
|
|
... WF11 actor topology edges must exactly match the expected fan-out/fan-in tuples.
|
|
|
|
|
|
|
|
|
|
Runtime Actor JSON Has Topology
|
|
|
|
|
[Documentation] Return true when actor-show runtime JSON exposes
|
|
|
|
|
... non-empty graph topology (route/nodes/edges) in any
|
|
|
|
|
... runtime-owned schema location.
|
|
|
|
|
[Arguments] ${runtime_actor_json}
|
|
|
|
|
${route}= Extract Route From Actor JSON ${runtime_actor_json}
|
|
|
|
|
${has_runtime_topology}= Evaluate
|
|
|
|
|
... isinstance($route, dict) and isinstance($route.get('nodes', []), list) and isinstance($route.get('edges', []), list) and len($route.get('nodes', [])) > 0 and len($route.get('edges', [])) > 0
|
|
|
|
|
RETURN ${has_runtime_topology}
|
|
|
|
|
|
|
|
|
|
Assert Registered Jinja2 Template Marker
|
|
|
|
|
[Documentation] Confirm registered runtime actor config contains
|
|
|
|
|
... the WF11 Jinja2 template marker.
|
|
|
|
|
[Arguments] ${runtime_actor_text}
|
|
|
|
|
Should Contain ${runtime_actor_text} WF11_TEMPLATE_OK={{ 'ENABLED' }}
|
|
|
|
|
... Runtime actor config must contain Jinja2 template marker WF11_TEMPLATE_OK; see secure command summary logs for context.
|
|
|
|
|
|
|
|
|
|
Assert Runtime Review Evidence Present
|
|
|
|
|
[Documentation] Verify runtime evidence for synthesized output. Prefer
|
|
|
|
|
... explicit branch markers and synthesis marker from
|
|
|
|
|
... execution/diff/artifacts output. Requires each branch
|
|
|
|
|
... marker to appear before the unified synthesis marker.
|
|
|
|
|
[Arguments] ${execution_output} ${diff_output} ${artifacts_output}
|
|
|
|
|
${runtime_combined}= Catenate SEPARATOR=\n
|
|
|
|
|
... ${execution_output}
|
|
|
|
|
... ${diff_output}
|
|
|
|
|
... ${artifacts_output}
|
|
|
|
|
# Sanitization: strip lines containing synthesiser prompt fragments to avoid
|
|
|
|
|
# matching prompt text instead of actual output. These substring filters
|
|
|
|
|
# assume the runtime emits prompt lines verbatim (one per line) as they
|
|
|
|
|
# appear in the actor YAML definition. If the runtime reformats prompts
|
|
|
|
|
# (e.g. wrapping or concatenation), these filters may need updating.
|
|
|
|
|
${runtime_sanitized}= Evaluate
|
|
|
|
|
... '\n'.join([line for line in $runtime_combined.splitlines() if 'start your response with' not in line.lower() and 'include header unified_review_report' not in line.lower() and 'sections security_branch_report' not in line.lower() and 'performance_branch_report, style_branch_report' not in line.lower()])
|
|
|
|
|
${runtime_lower}= Convert To Lower Case ${runtime_sanitized}
|
|
|
|
|
${security_header}= Run Keyword And Return Status
|
|
|
|
|
... Should Match Regexp
|
|
|
|
|
... ${runtime_sanitized}
|
|
|
|
|
... (?im)^\s*(?:[#>*-]+\s*)?security_branch_report\b
|
|
|
|
|
${performance_header}= Run Keyword And Return Status
|
|
|
|
|
... Should Match Regexp
|
|
|
|
|
... ${runtime_sanitized}
|
|
|
|
|
... (?im)^\s*(?:[#>*-]+\s*)?performance_branch_report\b
|
|
|
|
|
${style_header}= Run Keyword And Return Status
|
|
|
|
|
... Should Match Regexp
|
|
|
|
|
... ${runtime_sanitized}
|
|
|
|
|
... (?im)^\s*(?:[#>*-]+\s*)?style_branch_report\b
|
|
|
|
|
${synthesis_header}= Run Keyword And Return Status
|
|
|
|
|
... Should Match Regexp
|
|
|
|
|
... ${runtime_sanitized}
|
|
|
|
|
... (?im)^\s*(?:[#>*-]+\s*)?unified_review_report\b
|
|
|
|
|
${security_idx}= Evaluate $runtime_lower.find('security_branch_report')
|
|
|
|
|
${performance_idx}= Evaluate $runtime_lower.find('performance_branch_report')
|
|
|
|
|
${style_idx}= Evaluate $runtime_lower.find('style_branch_report')
|
|
|
|
|
${synthesis_idx}= Evaluate $runtime_lower.find('unified_review_report')
|
|
|
|
|
${has_security}= Evaluate $security_idx >= 0 and $security_header
|
|
|
|
|
${has_performance}= Evaluate $performance_idx >= 0 and $performance_header
|
|
|
|
|
${has_style}= Evaluate $style_idx >= 0 and $style_header
|
|
|
|
|
${has_synthesis}= Evaluate $synthesis_idx >= 0 and $synthesis_header
|
|
|
|
|
Should Be True ${has_security}
|
|
|
|
|
... Runtime output must include SECURITY_BRANCH_REPORT marker evidence.
|
|
|
|
|
Should Be True ${has_performance}
|
|
|
|
|
... Runtime output must include PERFORMANCE_BRANCH_REPORT marker evidence.
|
|
|
|
|
Should Be True ${has_style}
|
|
|
|
|
... Runtime output must include STYLE_BRANCH_REPORT marker evidence.
|
|
|
|
|
Should Be True ${has_synthesis}
|
|
|
|
|
... Runtime output must include UNIFIED_REVIEW_REPORT synthesis marker evidence.
|
|
|
|
|
${branches_before_synthesis}= Evaluate
|
|
|
|
|
... $security_idx < $synthesis_idx and $performance_idx < $synthesis_idx and $style_idx < $synthesis_idx
|
|
|
|
|
Should Be True ${branches_before_synthesis}
|
|
|
|
|
... Synthesis marker must appear after all three branch-report markers.
|
|
|
|
|
|
|
|
|
|
Assert No Read-Write Changeset Evidence
|
|
|
|
|
[Documentation] Ensure diff/artifacts do not contain source changeset
|
|
|
|
|
... payloads when running read-only action WF11.
|
|
|
|
|
[Arguments] ${diff_output} ${artifacts_output}
|
|
|
|
|
Should Not Match Regexp ${diff_output} (?m)^diff --git\s+
|
|
|
|
|
... msg=Read-only action must not produce diff-style changeset output (diff --git header detected).
|
|
|
|
|
Should Not Match Regexp ${diff_output} (?m)^@@\s+
|
|
|
|
|
... msg=Read-only action must not produce diff-style changeset output (hunk header detected).
|
|
|
|
|
Should Not Match Regexp ${diff_output} (?m)^\+\+\+\s+
|
|
|
|
|
... msg=Read-only action must not produce diff-style changeset output (+++ header detected).
|
|
|
|
|
Should Not Match Regexp ${diff_output} (?m)^---[ \t]+
|
|
|
|
|
... msg=Read-only action must not produce diff-style changeset output (--- header detected).
|
|
|
|
|
Should Not Match Regexp ${artifacts_output} (?i)\.patch\b
|
|
|
|
|
... msg=Read-only action must not produce patch file artifacts.
|
|
|
|
|
${combined_lower}= Convert To Lower Case ${diff_output}\n${artifacts_output}
|
|
|
|
|
${has_changeset_ref}= Evaluate 'changeset' in $combined_lower
|
|
|
|
|
IF ${has_changeset_ref}
|
|
|
|
|
Should Contain ${combined_lower} no changeset
|
|
|
|
|
... msg=Output references 'changeset' but does not indicate a no-changeset condition; read-only action may have produced unexpected changes.
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
Assert Diff Failure Is No-Changeset
|
|
|
|
|
[Documentation] Ensure plan diff failure is specifically a no-changeset
|
|
|
|
|
... condition, not an unrelated error.
|
|
|
|
|
[Arguments] ${diff_output}
|
|
|
|
|
${diff_lower}= Convert To Lower Case ${diff_output}
|
|
|
|
|
${expected_failure}= Evaluate
|
|
|
|
|
... ('no changeset' in $diff_lower) or ('no diff' in $diff_lower) or ('no changes' in $diff_lower)
|
|
|
|
|
Should Be True ${expected_failure}
|
|
|
|
|
... plan diff failed for a reason other than no-changeset/no-diff behavior.
|
|
|
|
|
|
|
|
|
|
Assert Read-Only Execution Rejection
|
|
|
|
|
[Documentation] Validate execute rejection is explicitly due to read-only
|
|
|
|
|
... constraints before skipping WF11 runtime proof checks.
|
|
|
|
|
[Arguments] ${output_text}
|
|
|
|
|
${lower}= Convert To Lower Case ${output_text}
|
|
|
|
|
${has_read_only}= Evaluate ('read-only' in $lower) or ('read only' in $lower)
|
|
|
|
|
${has_execute_context}= Evaluate ('execute' in $lower) or ('execution' in $lower)
|
|
|
|
|
${has_rejection_signal}= Evaluate
|
|
|
|
|
... ('reject' in $lower) or ('denied' in $lower) or ('forbid' in $lower) or ('not allowed' in $lower) or ('cannot' in $lower) or ('unsupported' in $lower) or ('blocked' in $lower)
|
|
|
|
|
${is_read_only_rejection}= Evaluate $has_read_only and $has_execute_context and $has_rejection_signal
|
|
|
|
|
Should Be True ${is_read_only_rejection}
|
|
|
|
|
... plan execution failed, but output does not indicate a read-only execution rejection.
|
|
|
|
|
|
|
|
|
|
Should Attempt Second Execute
|
|
|
|
|
[Documentation] Return true only when structured status metadata
|
|
|
|
|
... indicates execute must be called again.
|
|
|
|
|
[Arguments] ${status_json}
|
|
|
|
|
${phase}= Evaluate str($status_json.get('phase', ''))
|
|
|
|
|
${state}= Evaluate str($status_json.get('state', ''))
|
|
|
|
|
${phase_lower}= Convert To Lower Case ${phase}
|
|
|
|
|
${state_lower}= Convert To Lower Case ${state}
|
|
|
|
|
${phase_based}= Evaluate $phase_lower in ('strategize', 'strategy', 'action')
|
|
|
|
|
${state_based}= Evaluate $state_lower in ('created', 'new')
|
|
|
|
|
${terminal}= Evaluate $state_lower in ('complete', 'completed', 'applied', 'constrained', 'errored', 'failed', 'cancelled')
|
|
|
|
|
# The (not $terminal) guard is logically redundant with $state_based since
|
|
|
|
|
# their value sets are disjoint, but it serves as an explicit safety net
|
|
|
|
|
# against future value-set drift where a state could appear in both lists.
|
|
|
|
|
${needs_second}= Evaluate $phase_based and $state_based and (not $terminal)
|
|
|
|
|
RETURN ${needs_second}
|
|
|
|
|
|
|
|
|
|
Wait For Stable Plan Status
|
|
|
|
|
[Documentation] Poll plan status until state is not transient
|
|
|
|
|
... (processing/pending/queued/waiting) to avoid racing
|
|
|
|
|
... a second execute call.
|
|
|
|
|
[Arguments] ${plan_id} ${label}
|
|
|
|
|
${last_stdout}= Set Variable ${EMPTY}
|
|
|
|
|
${last_json}= Set Variable ${EMPTY}
|
|
|
|
|
FOR ${attempt} IN RANGE ${PLAN_STATUS_MAX_POLLS}
|
|
|
|
|
${status_result}= Run CleverAgents Command
|
|
|
|
|
... plan status ${plan_id}
|
|
|
|
|
... --format json
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
... timeout=${PLAN_STATUS_TIMEOUT}
|
|
|
|
|
Assert Output Has No Traceback ${status_result.stdout} ${status_result.stderr}
|
|
|
|
|
${attempt_num}= Evaluate $attempt + 1
|
|
|
|
|
Log Process Result Summary ${label} (attempt ${attempt_num}) ${status_result}
|
|
|
|
|
${status_json}= Extract JSON From Stdout ${status_result.stdout}
|
|
|
|
|
${state_lower}= Evaluate str($status_json.get('state', '')).lower()
|
|
|
|
|
${is_transient}= Evaluate $state_lower in ('processing', 'pending', 'queued', 'waiting')
|
|
|
|
|
${last_stdout}= Set Variable ${status_result.stdout}
|
|
|
|
|
${last_json}= Set Variable ${status_json}
|
|
|
|
|
IF not ${is_transient}
|
|
|
|
|
RETURN ${last_stdout} ${last_json}
|
|
|
|
|
END
|
|
|
|
|
Sleep ${PLAN_STATUS_POLL_INTERVAL}
|
|
|
|
|
END
|
|
|
|
|
Log WARN: ${label}: exhausted ${PLAN_STATUS_MAX_POLLS} polls while state remains transient ('${state_lower}'). WARN
|
|
|
|
|
RETURN ${last_stdout} ${last_json}
|
|
|
|
|
|
|
|
|
|
Execute Plan And Validate
|
|
|
|
|
[Documentation] Run plan execute, assert no traceback, log summary, and
|
|
|
|
|
... validate exit code. If rc != 0, check for read-only
|
|
|
|
|
... rejection and skip with the provided message. Returns
|
|
|
|
|
... combined stdout+stderr output.
|
|
|
|
|
[Arguments] ${plan_id} ${log_label} ${skip_message}
|
|
|
|
|
${result}= Run CleverAgents Command
|
|
|
|
|
... plan execute ${plan_id}
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
... timeout=${PLAN_EXEC_TIMEOUT}
|
|
|
|
|
Assert Output Has No Traceback ${result.stdout} ${result.stderr}
|
|
|
|
|
${combined}= Set Variable ${result.stdout}\n${result.stderr}
|
|
|
|
|
Log Process Result Summary ${log_label} ${result}
|
|
|
|
|
IF ${result.rc} != 0
|
|
|
|
|
Assert Read-Only Execution Rejection ${combined}
|
|
|
|
|
Skip ${skip_message}
|
|
|
|
|
END
|
|
|
|
|
RETURN ${combined}
|
|
|
|
|
|
|
|
|
|
Assert Execute Progressed Beyond Strategize
|
|
|
|
|
[Documentation] Ensure execution advanced beyond strategize-only state.
|
|
|
|
|
... Fails when status JSON is empty/malformed, in a failed
|
|
|
|
|
... terminal state, or still stuck in strategize phase.
|
|
|
|
|
[Arguments] ${status_json}
|
|
|
|
|
${phase}= Evaluate str($status_json.get('phase', '')).lower()
|
|
|
|
|
${state}= Evaluate str($status_json.get('state', '')).lower()
|
|
|
|
|
${has_meaningful_status}= Evaluate bool($phase) and bool($state)
|
|
|
|
|
Should Be True ${has_meaningful_status}
|
|
|
|
|
... Plan status JSON is empty or malformed (phase='${phase}', state='${state}'); cannot verify execution progress.
|
|
|
|
|
${is_failed}= Evaluate $state in ('errored', 'failed', 'cancelled')
|
|
|
|
|
Should Not Be True ${is_failed}
|
|
|
|
|
... Plan status indicates runtime failure; WF11 execute+synthesis proof is invalid.
|
|
|
|
|
${stuck_in_strategize}= Evaluate $phase in ('strategize', 'strategy', 'action') and $state in ('created', 'pending', 'queued', 'waiting', 'processing')
|
|
|
|
|
Should Not Be True ${stuck_in_strategize}
|
|
|
|
|
... Plan remained in strategize-phase pending state; WF11 execute+synthesis proof is required.
|
|
|
|
|
${progressed}= Evaluate $phase in ('execute', 'execution', 'applied', 'complete', 'completed')
|
|
|
|
|
Should Be True ${progressed}
|
|
|
|
|
... Plan status phase '${phase}' does not indicate progression beyond strategize.
|
|
|
|
|
|
|
|
|
|
Assert Trusted Profile Metadata
|
|
|
|
|
[Documentation] Assert runtime metadata explicitly records trusted profile.
|
|
|
|
|
[Arguments] ${plan_use_stdout} ${status_stdout}
|
|
|
|
|
${plan_use_json}= Extract JSON From Stdout ${plan_use_stdout}
|
|
|
|
|
${status_json}= Extract JSON From Stdout ${status_stdout}
|
|
|
|
|
${plan_json_text}= Evaluate json.dumps($plan_use_json, sort_keys=True).lower() modules=json
|
|
|
|
|
${status_json_text}= Evaluate json.dumps($status_json, sort_keys=True).lower() modules=json
|
|
|
|
|
${combined_meta}= Catenate SEPARATOR=\n ${plan_json_text} ${status_json_text}
|
|
|
|
|
# (?i) is technically redundant since combined_meta is already lowered,
|
|
|
|
|
# but kept for defensive correctness should the lowering step change.
|
|
|
|
|
${trusted_profile_match}= Run Keyword And Return Status
|
|
|
|
|
... Should Match Regexp
|
|
|
|
|
... ${combined_meta}
|
|
|
|
|
... (?i)"automation_profile(?:_name)?"\s*:\s*"trusted"
|
|
|
|
|
Should Be True ${trusted_profile_match}
|
|
|
|
|
... Runtime metadata must explicitly assert trusted automation profile.
|
|
|
|
|
|
|
|
|
|
Skip If Provider Credits Unavailable
|
|
|
|
|
[Documentation] Skip when provider keys exist but account credits are
|
|
|
|
|
... exhausted; this is an environment issue, not product behavior.
|
|
|
|
|
[Arguments] ${rc} ${output_text}
|
|
|
|
|
IF ${rc} == 0
|
|
|
|
|
RETURN
|
|
|
|
|
END
|
|
|
|
|
${lower}= Convert To Lower Case ${output_text}
|
|
|
|
|
${credits_exhausted}= Evaluate
|
|
|
|
|
... 'credit balance is too low' in $lower or 'upgrade or purchase credits' in $lower
|
|
|
|
|
IF ${credits_exhausted}
|
|
|
|
|
Skip LLM provider credits are unavailable for this environment. Skipping WF11 E2E runtime path.
|
|
|
|
|
END
|
hurui200320