|
|
|
@@ -0,0 +1,459 @@
|
|
|
|
|
*** Settings ***
|
|
|
|
|
Documentation E2E test for WF03: Multi-File Refactoring with Invariants.
|
|
|
|
|
...
|
|
|
|
|
... Intermediate-Advanced scenario: a senior engineer refactors an
|
|
|
|
|
... authentication module to replace raw SQL with SQLAlchemy ORM
|
|
|
|
|
... while maintaining API compatibility. Uses the ``cautious``
|
|
|
|
|
... automation profile which auto-approves high-confidence decisions
|
|
|
|
|
... but pauses on low-confidence ones.
|
|
|
|
|
...
|
|
|
|
|
... Exercises invariants at multiple scopes (global, project, action),
|
|
|
|
|
... custom actor registration, plan tree, plan explain, plan correct
|
|
|
|
|
... (revert mode), and plan prompt.
|
|
|
|
|
...
|
|
|
|
|
... Zero mocking — real CLI, real LLM API keys, real subprocess execution.
|
|
|
|
|
Resource common_e2e.resource
|
|
|
|
|
Suite Setup E2E Suite Setup
|
|
|
|
|
Suite Teardown E2E Suite Teardown
|
|
|
|
|
|
|
|
|
|
*** Variables ***
|
|
|
|
|
${ACTION_NAME} local/refactor-auth
|
|
|
|
|
${ACTOR_NAME} local/wf03-strategist
|
|
|
|
|
${RESOURCE_NAME} local/wf03-repo
|
|
|
|
|
${PROJECT_NAME} local/wf03-project
|
|
|
|
|
# Timeouts for LLM-backed commands (strategize, execute, correct, etc.)
|
|
|
|
|
${LLM_TIMEOUT} 300s
|
|
|
|
|
${TREE_TIMEOUT} 120s
|
|
|
|
|
|
|
|
|
|
${AUTH_MODULE_FIXTURE} SEPARATOR=\n
|
|
|
|
|
... """Authentication module using raw SQL queries."""
|
|
|
|
|
... import sqlite3
|
|
|
|
|
... from typing import Optional
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... DB_PATH = "users.db"
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... def authenticate(username: str, password: str) -> bool:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Check credentials against the users table."""
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}conn = sqlite3.connect(DB_PATH)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}cur = conn.cursor()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}cur.execute("SELECT 1 FROM users WHERE name=? AND pass=?", (username, password))
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}found = cur.fetchone() is not None
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}conn.close()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}return found
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... def get_user(user_id: int) -> Optional[dict]:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Retrieve a user record by primary key."""
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}conn = sqlite3.connect(DB_PATH)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}cur = conn.cursor()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}cur.execute("SELECT id, name, email FROM users WHERE id=?", (user_id,))
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}row = cur.fetchone()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}conn.close()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}if row is None:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return None
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}return {"id": row[0], "name": row[1], "email": row[2]}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... ${EMPTY}
|
|
|
|
|
... def list_users() -> list[dict]:
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Return all users."""
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}conn = sqlite3.connect(DB_PATH)
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}cur = conn.cursor()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}cur.execute("SELECT id, name, email FROM users")
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}rows = cur.fetchall()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}conn.close()
|
|
|
|
|
... ${SPACE}${SPACE}${SPACE}${SPACE}return [{"id": r[0], "name": r[1], "email": r[2]} for r in rows]
|
|
|
|
|
|
|
|
|
|
${MODELS_FIXTURE} SEPARATOR=\n
|
|
|
|
|
... """SQLAlchemy models — target for refactoring."""
|
|
|
|
|
... # This file is intentionally empty to serve as a placeholder
|
|
|
|
|
... # for the ORM models that should be created by the refactoring.
|
|
|
|
|
|
|
|
|
|
*** Test Cases ***
|
|
|
|
|
WF03 Multi-File Refactoring With Cautious Profile
|
|
|
|
|
[Documentation] End-to-end workflow: set up multi-scope invariants (global,
|
|
|
|
|
... project, action), register a custom actor, create a refactoring
|
|
|
|
|
... action with cautious profile, execute plan, exercise plan tree,
|
|
|
|
|
... plan explain, plan correct --mode revert, plan prompt, and verify
|
|
|
|
|
... corrected execution produces meaningful output.
|
|
|
|
|
[Tags] E2E
|
|
|
|
|
[Timeout] 30 minutes
|
|
|
|
|
|
|
|
|
|
# ── Graceful skip when no LLM API keys are available ──────────
|
|
|
|
|
Skip If No LLM Keys
|
|
|
|
|
|
|
|
|
|
# ── Select actor based on available API keys ──────────────────
|
|
|
|
|
${has_anthropic}= Evaluate
|
|
|
|
|
... bool(os.environ.get('ANTHROPIC_API_KEY', '')) modules=os
|
|
|
|
|
IF ${has_anthropic}
|
|
|
|
|
${provider}= Set Variable anthropic
|
|
|
|
|
${model}= Set Variable claude-sonnet-4-20250514
|
|
|
|
|
ELSE
|
|
|
|
|
${provider}= Set Variable openai
|
|
|
|
|
${model}= Set Variable gpt-4o
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
# ── Initialize database ───────────────────────────────────────
|
|
|
|
|
# Note: `init` does not support `--format` — no format flag here.
|
|
|
|
|
${r_init}= Run CleverAgents Command
|
|
|
|
|
... init --yes --force
|
|
|
|
|
Should Not Contain ${r_init.stdout}\n${r_init.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_init.stdout}\n${r_init.stderr} INTERNAL
|
|
|
|
|
|
|
|
|
|
# ── 1. Create temp repo with raw SQL auth module ──────────────
|
|
|
|
|
${repo_dir}= Create Temp Git Repo wf03-refactor-repo
|
|
|
|
|
Create Directory ${repo_dir}${/}src
|
|
|
|
|
Create File ${repo_dir}${/}src${/}auth.py ${AUTH_MODULE_FIXTURE}
|
|
|
|
|
Create File ${repo_dir}${/}src${/}models.py ${MODELS_FIXTURE}
|
|
|
|
|
${git_add}= Run Process git add . cwd=${repo_dir} timeout=60s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr}
|
|
|
|
|
${git_commit}= Run Process git commit -m Add raw SQL auth module cwd=${repo_dir} timeout=60s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr}
|
|
|
|
|
${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${repo_dir} timeout=60s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${branch_result.rc} 0 git rev-parse failed: ${branch_result.stderr}
|
|
|
|
|
${branch}= Strip String ${branch_result.stdout}
|
|
|
|
|
|
|
|
|
|
# ── 2. Register resource and project ──────────────────────────
|
|
|
|
|
${r_resource}= Run CleverAgents Command
|
|
|
|
|
... resource add git-checkout ${RESOURCE_NAME}
|
|
|
|
|
... --path ${repo_dir} --branch ${branch}
|
|
|
|
|
Should Not Contain ${r_resource.stdout}\n${r_resource.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_resource.stdout}\n${r_resource.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${r_resource} ${RESOURCE_NAME}
|
|
|
|
|
|
|
|
|
|
${r_project}= Run CleverAgents Command
|
|
|
|
|
... project create ${PROJECT_NAME}
|
|
|
|
|
... --description WF03 refactoring project
|
|
|
|
|
... --resource ${RESOURCE_NAME}
|
|
|
|
|
Should Not Contain ${r_project.stdout}\n${r_project.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_project.stdout}\n${r_project.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${r_project} ${PROJECT_NAME}
|
|
|
|
|
|
|
|
|
|
# ── 3. Add invariants at multiple scopes (global + project) ───
|
|
|
|
|
# Global invariant — text aligned with spec WF03 Step 1.
|
|
|
|
|
${inv_global}= Run CleverAgents Command
|
|
|
|
|
... invariant add --global
|
|
|
|
|
... All public APIs must maintain backward compatibility
|
|
|
|
|
Should Not Contain ${inv_global.stdout}\n${inv_global.stderr} Traceback
|
|
|
|
|
Should Not Contain ${inv_global.stdout}\n${inv_global.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${inv_global} backward compatibility
|
|
|
|
|
|
|
|
|
|
# Project-level invariant — text aligned with spec WF03 Step 1.
|
|
|
|
|
${inv_project}= Run CleverAgents Command
|
|
|
|
|
... invariant add --project ${PROJECT_NAME}
|
|
|
|
|
... Database queries must use the SQLAlchemy ORM, not raw SQL
|
|
|
|
|
Should Not Contain ${inv_project.stdout}\n${inv_project.stderr} Traceback
|
|
|
|
|
Should Not Contain ${inv_project.stdout}\n${inv_project.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${inv_project} SQLAlchemy ORM
|
|
|
|
|
|
|
|
|
|
# ── 4. Register a custom actor for refactoring strategy (AC3) ─
|
|
|
|
|
${actor_config}= Evaluate
|
|
|
|
|
... json.dumps({"provider": $provider, "model": $model, "options": {"temperature": 0.2}}, indent=2)
|
|
|
|
|
... modules=json
|
|
|
|
|
${actor_config_path}= Set Variable ${SUITE_HOME}${/}wf03_actor_config.json
|
|
|
|
|
Create File ${actor_config_path} ${actor_config}
|
|
|
|
|
${r_actor}= Run CleverAgents Command
|
|
|
|
|
... actor add ${ACTOR_NAME} --config ${actor_config_path} --format plain
|
|
|
|
|
Should Not Contain ${r_actor.stdout}\n${r_actor.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_actor.stdout}\n${r_actor.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${r_actor} ${ACTOR_NAME}
|
|
|
|
|
|
|
|
|
|
# Verify the custom actor appears in actor list (AC3 — registration is
|
|
|
|
|
# confirmed, even though the action must still reference provider/model
|
|
|
|
|
# directly; see comment below).
|
|
|
|
|
${r_actor_list}= Run CleverAgents Command
|
|
|
|
|
... actor list --format plain
|
|
|
|
|
Output Should Contain ${r_actor_list} ${ACTOR_NAME}
|
|
|
|
|
|
|
|
|
|
# ── 5. Create action with cautious profile and action-level invariants ─
|
|
|
|
|
# Action YAML includes action-scoped invariants (AC2) plus an
|
|
|
|
|
# estimation_actor per the spec's WF03.
|
|
|
|
|
# NOTE (strategy_actor): The spec WF03 Step 2 shows
|
|
|
|
|
# `strategy_actor: local/refactoring-strategist` (the custom actor name).
|
|
|
|
|
# However, the runtime resolves strategy_actor as a provider/model pair —
|
|
|
|
|
# not an actor registry lookup — so `local/wf03-strategist` yields
|
|
|
|
|
# "Unknown provider type: local" at execution time. Until the actor
|
|
|
|
|
# registry lookup is wired into the plan executor, we use the provider/model
|
|
|
|
|
# path directly. The custom actor registration above is still exercised
|
|
|
|
|
# for AC3 (actor add + actor list verification).
|
|
|
|
|
# NOTE (args): The spec WF03 Step 2 shows an `args:` section in the action
|
|
|
|
|
# YAML with a `target_module` parameter. However, ActionConfigSchema does
|
|
|
|
|
# not yet accept `args` as a config field (extra_forbidden). Omitted
|
|
|
|
|
# until schema support is added.
|
|
|
|
|
${action_yaml}= Catenate SEPARATOR=\n
|
|
|
|
|
... name: ${ACTION_NAME}
|
|
|
|
|
... description: Refactor raw SQL authentication to use SQLAlchemy ORM
|
|
|
|
|
... definition_of_done: All raw SQL replaced with ORM while preserving public API
|
|
|
|
|
... strategy_actor: ${provider}/${model}
|
|
|
|
|
... execution_actor: ${provider}/${model}
|
|
|
|
|
... estimation_actor: ${provider}/${model}
|
|
|
|
|
... automation_profile: cautious
|
|
|
|
|
... invariants:
|
|
|
|
|
... ${SPACE}${SPACE}- "Each file must be refactored in a separate commit-sized change"
|
|
|
|
|
... ${SPACE}${SPACE}- "ORM models must be defined before queries are converted"
|
|
|
|
|
... ${SPACE}${SPACE}- "All raw SQL must be replaced — no partial conversion"
|
|
|
|
|
${yaml_path}= Set Variable ${SUITE_HOME}${/}wf03_action.yaml
|
|
|
|
|
Create File ${yaml_path} ${action_yaml}
|
|
|
|
|
${r_action}= Run CleverAgents Command
|
|
|
|
|
... action create --config ${yaml_path}
|
|
|
|
|
Should Not Contain ${r_action.stdout}\n${r_action.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_action.stdout}\n${r_action.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${r_action} ${ACTION_NAME}
|
|
|
|
|
|
|
|
|
|
# ── 6. Plan use — create plan with cautious profile ───────────
|
|
|
|
|
# The --automation-profile flag is intentionally included even though the
|
|
|
|
|
# action YAML already sets automation_profile: cautious. This exercises
|
|
|
|
|
# the CLI-level override path and makes the test self-documenting.
|
|
|
|
|
# NOTE: Spec WF03 Step 3 shows `--arg target_module="src/auth"` here, but
|
|
|
|
|
# ActionConfigSchema does not yet support `args` declaration, so the CLI
|
|
|
|
|
# rejects unknown arguments. Omit --arg until schema support is added.
|
|
|
|
|
${r_use}= Run CleverAgents Command
|
|
|
|
|
... plan use ${ACTION_NAME} ${PROJECT_NAME}
|
|
|
|
|
... --automation-profile cautious --format plain
|
|
|
|
|
Should Not Be Empty ${r_use.stdout}
|
|
|
|
|
Should Not Contain ${r_use.stdout}\n${r_use.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_use.stdout}\n${r_use.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${r_use} cautious
|
|
|
|
|
${plan_id}= Extract Plan Id ${r_use.stdout} ${r_use.stderr}
|
|
|
|
|
Should Not Be Empty ${plan_id} Could not extract plan ID from plan use output
|
|
|
|
|
Log Plan ID: ${plan_id} level=INFO
|
|
|
|
|
|
|
|
|
|
# ── 7. Plan execute — strategize (may pause on low confidence) ─
|
|
|
|
|
${r_strategize}= Run CleverAgents Command
|
|
|
|
|
... plan execute ${plan_id} --format plain
|
|
|
|
|
... timeout=${LLM_TIMEOUT}
|
|
|
|
|
Should Not Contain ${r_strategize.stdout}\n${r_strategize.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_strategize.stdout}\n${r_strategize.stderr} INTERNAL
|
|
|
|
|
Should Not Be Empty ${r_strategize.stdout} Strategize produced no output
|
|
|
|
|
# Best-effort check that invariants were acknowledged during strategize
|
|
|
|
|
# (AC2 invariant enforcement). LLM output is non-deterministic, so we
|
|
|
|
|
# check for any invariant-related keyword rather than exact text.
|
|
|
|
|
${strat_combined}= Set Variable ${r_strategize.stdout}\n${r_strategize.stderr}
|
|
|
|
|
${strat_lower}= Evaluate ($strat_combined).lower()
|
|
|
|
|
${has_invariant_ref}= Evaluate 'invariant' in $strat_lower or 'constraint' in $strat_lower or 'backward' in $strat_lower or 'compatibility' in $strat_lower or 'orm' in $strat_lower
|
|
|
|
|
Run Keyword And Warn On Failure Should Be True ${has_invariant_ref}
|
|
|
|
|
... Strategize output should reference invariants/constraints (AC2)
|
|
|
|
|
|
|
|
|
|
# ── 8. Plan tree — inspect decision tree (spec WF03 Step 3) ──
|
|
|
|
|
# Spec WF03 Step 3 flow: execute → tree → explain → prompt → status.
|
|
|
|
|
# Step 4 continues: correct → tree --show-superseded → execute → diff → apply.
|
|
|
|
|
${r_tree}= Run CleverAgents Command
|
|
|
|
|
... plan tree ${plan_id} --format plain
|
|
|
|
|
... timeout=${TREE_TIMEOUT} expected_rc=None
|
|
|
|
|
Log Plan tree output: ${r_tree.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_tree.stdout}\n${r_tree.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_tree.stdout}\n${r_tree.stderr} INTERNAL
|
|
|
|
|
IF ${r_tree.rc} != 0
|
|
|
|
|
Fail plan tree failed (rc=${r_tree.rc}): ${r_tree.stderr}
|
|
|
|
|
END
|
|
|
|
|
Should Not Be Empty ${r_tree.stdout} Plan tree produced no output
|
|
|
|
|
|
|
|
|
|
# ── 9. Plan explain — inspect decision (AC5, partial) ─────────
|
|
|
|
|
# PARTIAL AC5 COVERAGE: The spec uses a decision_id for plan explain
|
|
|
|
|
# (plan-level only, not decision-level). Plan explain also accepts
|
|
|
|
|
# plan_id for plan-level explanation (see #1057). We use plan_id here
|
|
|
|
|
# because the dynamic decision_id is not deterministically extractable
|
|
|
|
|
# from plan tree output in all output formats.
|
|
|
|
|
# TODO(#1057): Add decision_id extraction from plan tree output and
|
|
|
|
|
# exercise decision-level explain for full AC5 coverage.
|
|
|
|
|
${r_explain}= Run CleverAgents Command
|
|
|
|
|
... plan explain ${plan_id} --format plain
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
Log Plan explain output: ${r_explain.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_explain.stdout}\n${r_explain.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_explain.stdout}\n${r_explain.stderr} INTERNAL
|
|
|
|
|
IF ${r_explain.rc} != 0
|
|
|
|
|
Fail plan explain failed (rc=${r_explain.rc}): ${r_explain.stderr}
|
|
|
|
|
END
|
|
|
|
|
Should Not Be Empty ${r_explain.stdout} Plan explain produced no output
|
|
|
|
|
|
|
|
|
|
# ── 10. Plan prompt — provide user guidance (AC7) ─────────────
|
|
|
|
|
# Spec WF03 Step 3: tree → explain → prompt → status → correct.
|
|
|
|
|
# The spec defines `plan prompt` for providing user guidance, but it may
|
|
|
|
|
# not yet be implemented as a CLI subcommand. Use expected_rc=None and
|
|
|
|
|
# validate only when the command succeeds.
|
|
|
|
|
${r_prompt}= Run CleverAgents Command
|
|
|
|
|
... plan prompt ${plan_id}
|
|
|
|
|
... Ensure all database operations use context managers for connection handling
|
|
|
|
|
... --format plain timeout=${LLM_TIMEOUT} expected_rc=None
|
|
|
|
|
Log Plan prompt output: ${r_prompt.stdout} level=DEBUG
|
|
|
|
|
IF ${r_prompt.rc} == 0
|
|
|
|
|
# Command succeeded — check for error markers that indicate a
|
|
|
|
|
# code-level defect (Traceback, INTERNAL).
|
|
|
|
|
Should Not Contain ${r_prompt.stdout}\n${r_prompt.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_prompt.stdout}\n${r_prompt.stderr} INTERNAL
|
|
|
|
|
Should Not Be Empty ${r_prompt.stdout} Plan prompt produced no output
|
|
|
|
|
ELSE
|
|
|
|
|
# Command failed — check for Traceback (indicates unhandled exception
|
|
|
|
|
# rather than a clean "not implemented" error). INTERNAL is expected
|
|
|
|
|
# in the error output for unrecognized subcommands.
|
|
|
|
|
Should Not Contain ${r_prompt.stdout}\n${r_prompt.stderr} Traceback
|
|
|
|
|
Log Plan prompt command returned rc=${r_prompt.rc} (may not be implemented yet) WARN
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
# ── 11. Plan status — verify plan progression (AC4) ───────────
|
|
|
|
|
# Placed after prompt per spec WF03 Step 3 ordering:
|
|
|
|
|
# tree → explain → prompt → status.
|
|
|
|
|
# The cautious profile should pause on low-confidence decisions.
|
|
|
|
|
${r_status_after_strat}= Run CleverAgents Command
|
|
|
|
|
... plan status ${plan_id} --format plain
|
|
|
|
|
Should Not Be Empty ${r_status_after_strat.stdout}
|
|
|
|
|
Should Not Contain ${r_status_after_strat.stdout}\n${r_status_after_strat.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_status_after_strat.stdout}\n${r_status_after_strat.stderr} INTERNAL
|
|
|
|
|
Log Status after strategize: ${r_status_after_strat.stdout} level=DEBUG
|
|
|
|
|
# Verify the plan_id appears in status output (confirms we are querying
|
|
|
|
|
# the correct plan).
|
|
|
|
|
Output Should Contain ${r_status_after_strat} ${plan_id}
|
|
|
|
|
# KNOWN AC4 VERIFICATION GAP: Robust verification that the cautious
|
|
|
|
|
# profile caused a pause on a low-confidence decision is infeasible
|
|
|
|
|
# due to LLM non-determinism — the LLM may not always produce a
|
|
|
|
|
# low-confidence decision. We verify the plan_id is present and
|
|
|
|
|
# soft-check for pause-related keywords only.
|
|
|
|
|
${status_combined}= Set Variable ${r_status_after_strat.stdout}\n${r_status_after_strat.stderr}
|
|
|
|
|
${status_lower}= Evaluate ($status_combined).lower()
|
|
|
|
|
${has_pause_indicator}= Evaluate 'paused' in $status_lower or 'awaiting' in $status_lower
|
|
|
|
|
Run Keyword And Warn On Failure Should Be True ${has_pause_indicator}
|
|
|
|
|
... Plan status should indicate a paused/awaiting state after cautious-profile strategize (AC4)
|
|
|
|
|
|
|
|
|
|
# ── 12. Plan correct — revert mode with guidance (AC6, partial) ─
|
|
|
|
|
# PARTIAL AC6 COVERAGE: Spec WF03 flow places correct after prompt.
|
|
|
|
|
# The spec uses a decision_id for plan correct; however, plan correct
|
|
|
|
|
# also accepts plan_id as primary identifier (see #1055). We use
|
|
|
|
|
# plan_id here because the dynamic decision_id is not deterministically
|
|
|
|
|
# extractable.
|
|
|
|
|
# TODO(#1055): Add decision_id extraction from plan tree output and
|
|
|
|
|
# exercise decision-level correct for full AC6 coverage.
|
|
|
|
|
${r_correct}= Run CleverAgents Command
|
|
|
|
|
... plan correct ${plan_id} --mode revert
|
|
|
|
|
... --guidance Use SQLAlchemy ORM with session context managers
|
|
|
|
|
... --yes --format plain
|
|
|
|
|
... timeout=${LLM_TIMEOUT} expected_rc=None
|
|
|
|
|
Log Plan correct output: ${r_correct.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_correct.stdout}\n${r_correct.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_correct.stdout}\n${r_correct.stderr} INTERNAL
|
|
|
|
|
IF ${r_correct.rc} != 0
|
|
|
|
|
Fail plan correct failed (rc=${r_correct.rc}): ${r_correct.stderr}
|
|
|
|
|
END
|
|
|
|
|
Should Not Be Empty ${r_correct.stdout} Plan correct produced no output
|
|
|
|
|
# Validate that correction output references revert semantics.
|
|
|
|
|
# Soft-checked because CLI output content depends on LLM-generated text;
|
|
|
|
|
# the CLI does not currently echo the correction mode as a fixed string.
|
|
|
|
|
${correct_lower}= Evaluate ($r_correct.stdout).lower()
|
|
|
|
|
${has_revert_ref}= Evaluate 'revert' in $correct_lower or 'correction' in $correct_lower or 'rollback' in $correct_lower or 'supersed' in $correct_lower
|
|
|
|
|
Run Keyword And Warn On Failure Should Be True ${has_revert_ref}
|
|
|
|
|
... Plan correct output should reference revert/correction semantics
|
|
|
|
|
|
|
|
|
|
# ── 13. Plan tree --show-superseded — verify superseded decisions ─
|
|
|
|
|
# Spec WF03 Step 4 shows `plan tree --show-superseded` after correction
|
|
|
|
|
# to visualise the corrected decision alongside the original.
|
|
|
|
|
${r_tree_super}= Run CleverAgents Command
|
|
|
|
|
... plan tree ${plan_id} --show-superseded --format plain
|
|
|
|
|
... timeout=${TREE_TIMEOUT} expected_rc=None
|
|
|
|
|
Log Plan tree (show-superseded) output: ${r_tree_super.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_tree_super.stdout}\n${r_tree_super.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_tree_super.stdout}\n${r_tree_super.stderr} INTERNAL
|
|
|
|
|
IF ${r_tree_super.rc} != 0
|
|
|
|
|
Fail plan tree --show-superseded failed (rc=${r_tree_super.rc}): ${r_tree_super.stderr}
|
|
|
|
|
END
|
|
|
|
|
Should Not Be Empty ${r_tree_super.stdout} Plan tree --show-superseded produced no output
|
|
|
|
|
|
|
|
|
|
# ── 14. Plan execute — continue execution after correction ────
|
|
|
|
|
${r_execute}= Run CleverAgents Command
|
|
|
|
|
... plan execute ${plan_id} --format plain
|
|
|
|
|
... timeout=${LLM_TIMEOUT} expected_rc=None
|
|
|
|
|
Log Execute output: ${r_execute.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_execute.stdout}\n${r_execute.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_execute.stdout}\n${r_execute.stderr} INTERNAL
|
|
|
|
|
IF ${r_execute.rc} != 0
|
|
|
|
|
Fail Post-correction execute failed (rc=${r_execute.rc}): ${r_execute.stderr}
|
|
|
|
|
END
|
|
|
|
|
Should Not Be Empty ${r_execute.stdout} Plan execute produced no output
|
|
|
|
|
|
|
|
|
|
# ── 15. Plan diff — review corrected changes (AC8) ────────────
|
|
|
|
|
# TODO(#749): Add `plan diff --correction <CORRECTION_ID>` variant once
|
|
|
|
|
# correction_id can be reliably extracted from plan correct output
|
|
|
|
|
# (spec WF03 Step 4 shows correction-specific diff).
|
|
|
|
|
${r_diff}= Run CleverAgents Command
|
|
|
|
|
... plan diff ${plan_id} --format plain
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
Log Diff output: ${r_diff.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_diff.stdout}\n${r_diff.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_diff.stdout}\n${r_diff.stderr} INTERNAL
|
|
|
|
|
IF ${r_diff.rc} == 0
|
|
|
|
|
Should Not Be Empty ${r_diff.stdout} Plan diff produced no output — expected file changes
|
|
|
|
|
# Content assertion: diff output should reference at least one of
|
|
|
|
|
# the refactoring target files (AC8 validation). Use warn-on-failure
|
|
|
|
|
# because LLM-generated diff content is non-deterministic.
|
|
|
|
|
${diff_lower}= Evaluate ($r_diff.stdout).lower()
|
|
|
|
|
${has_file_ref}= Evaluate 'auth' in $diff_lower or '.py' in $diff_lower or 'model' in $diff_lower or 'sql' in $diff_lower
|
|
|
|
|
Run Keyword And Warn On Failure Should Be True ${has_file_ref}
|
|
|
|
|
... Diff output should reference refactoring-related content (AC8)
|
|
|
|
|
ELSE
|
|
|
|
|
Fail plan diff failed (rc=${r_diff.rc}): ${r_diff.stderr}
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
# ── 16. Plan apply ────────────────────────────────────────────
|
|
|
|
|
# Note: `lifecycle-apply` is the v3 subcommand that does NOT support
|
|
|
|
|
# `--yes` (unlike `plan apply --yes` shown in the spec). The
|
|
|
|
|
# `lifecycle-apply` subcommand has no confirmation prompt by design.
|
|
|
|
|
# TODO(#749): Align with spec's `plan apply --yes` when the v3 apply
|
|
|
|
|
# subcommand supports the --yes confirmation flag.
|
|
|
|
|
${r_apply}= Run CleverAgents Command
|
|
|
|
|
... plan lifecycle-apply ${plan_id} --format plain
|
|
|
|
|
... timeout=${LLM_TIMEOUT} expected_rc=None
|
|
|
|
|
Log Apply output: ${r_apply.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_apply.stdout}\n${r_apply.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_apply.stdout}\n${r_apply.stderr} INTERNAL
|
|
|
|
|
IF ${r_apply.rc} != 0
|
|
|
|
|
Fail plan lifecycle-apply failed (rc=${r_apply.rc}): ${r_apply.stderr}
|
|
|
|
|
END
|
|
|
|
|
Should Not Be Empty ${r_apply.stdout} Plan lifecycle-apply produced no output
|
|
|
|
|
|
|
|
|
|
# ── 16b. Post-apply repository verification (AC8) ─────────────
|
|
|
|
|
# Verify that lifecycle-apply actually committed changes to the target repo.
|
|
|
|
|
${git_log_post_apply}= Run Process git log -1 --oneline
|
|
|
|
|
... cwd=${repo_dir} timeout=60s on_timeout=kill
|
|
|
|
|
Log Git log after apply: ${git_log_post_apply.stdout} level=DEBUG
|
|
|
|
|
Should Not Be Empty ${git_log_post_apply.stdout}
|
|
|
|
|
... No commits found in target repo after lifecycle-apply
|
|
|
|
|
|
|
|
|
|
# ── 17. Invariant list — verify command works without error ───
|
|
|
|
|
# TODO: Spec WF03 Step 1 places `invariant list --effective` immediately
|
|
|
|
|
# after invariant setup, before action creation. Here it is placed at
|
|
|
|
|
# the end because invariants are currently in-memory only (per
|
|
|
|
|
# InvariantService design), so invariants added in earlier CLI
|
|
|
|
|
# invocations are not visible in subsequent ones. Revisit placement
|
|
|
|
|
# when invariant persistence is implemented.
|
|
|
|
|
# NOTE: Expected results are empty/minimal — each CLI invocation starts
|
|
|
|
|
# a fresh InvariantService, so previously-added invariants are not
|
|
|
|
|
# retained. This step validates the command executes without error.
|
|
|
|
|
${r_inv_list}= Run CleverAgents Command
|
|
|
|
|
... invariant list --effective --project ${PROJECT_NAME}
|
|
|
|
|
... --format plain expected_rc=None
|
|
|
|
|
Log Invariant list: ${r_inv_list.stdout} level=DEBUG
|
|
|
|
|
Should Not Contain ${r_inv_list.stdout}\n${r_inv_list.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_inv_list.stdout}\n${r_inv_list.stderr} INTERNAL
|
|
|
|
|
IF ${r_inv_list.rc} != 0
|
|
|
|
|
Log invariant list returned rc=${r_inv_list.rc}: ${r_inv_list.stderr} WARN
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
# ── 18. Final status check ────────────────────────────────────
|
|
|
|
|
${r_status}= Run CleverAgents Command
|
|
|
|
|
... plan status ${plan_id} --format plain
|
|
|
|
|
Should Not Be Empty ${r_status.stdout} Final status produced no output
|
|
|
|
|
Should Not Contain ${r_status.stdout}\n${r_status.stderr} Traceback
|
|
|
|
|
Should Not Contain ${r_status.stdout}\n${r_status.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${r_status} ${plan_id}
|
|
|
|
|
# Soft-check for terminal state — after lifecycle-apply the plan should
|
|
|
|
|
# be in a terminal state (applied/completed). LLM non-determinism may
|
|
|
|
|
# prevent the plan from reaching terminal state in all runs.
|
|
|
|
|
${final_status_lower}= Evaluate ($r_status.stdout).lower()
|
|
|
|
|
${is_terminal}= Evaluate 'applied' in $final_status_lower or 'completed' in $final_status_lower
|
|
|
|
|
Run Keyword And Warn On Failure Should Be True ${is_terminal}
|
|
|
|
|
... Final status should indicate a terminal state (applied/completed)
|
|
|
|
|
Log Final status: ${r_status.stdout} level=DEBUG
|
|
|
|
|
Log WF03 Multi-File Refactoring E2E test completed
|
|
|
|
|
|
|
|
|
|
*** Keywords ***
|
|
|
|
|
# Extract Plan Id is provided by common_e2e.resource
|