From 91bcffe1b5f6ed3c5541259a11a777c50ab992f2 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Thu, 12 Mar 2026 23:37:33 +0000 Subject: [PATCH] =?UTF-8?q?test(e2e):=20workflow=20example=2015=20?= =?UTF-8?q?=E2=80=94=20disaster=20recovery,=20rollback=20a=20failed=20appl?= =?UTF-8?q?y=20(trusted=20profile)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add end-to-end Robot Framework test for WF15: Disaster Recovery — Rollback a Failed Apply. The test exercises plan execution with the trusted automation profile, then investigates via plan status, plan tree, plan explain (--show-context --show-reasoning), plan rollback, plan correct (--mode revert with corrective guidance), and plan diff (--correction) before re-executing and applying. Zero mocking — real CLI, real LLM API keys, real subprocess execution. Fixture code provides a connection-pool module with known exhaustion risk. ISSUES CLOSED: #761 --- CHANGELOG.md | 5 + robot/e2e/wf15_disaster_recovery.robot | 515 +++++++++++++++++++++++++ 2 files changed, 520 insertions(+) create mode 100644 robot/e2e/wf15_disaster_recovery.robot diff --git a/CHANGELOG.md b/CHANGELOG.md index 36aab2df1..7d2b2d262 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1021,6 +1021,11 @@ attribute 'db'` after `agents init`. Root cause: `_get_session_service()` when API keys are absent, and `--exclude E2E` on the standard integration test session. Includes a minimal smoke test exercising `agents --version` and `agents --help`. (#740) +- Added E2E test for workflow example 15: disaster recovery — rollback a + failed apply with trusted profile. Tests plan status, plan tree, plan + explain with --show-context --show-reasoning, plan rollback, plan correct + --mode revert, and plan diff --correction with real LLM API keys and zero + mocking. (#761) - Implemented `tdd_expected_fail` tag handling in Robot Framework via a Listener v3 module (`robot/tdd_expected_fail_listener.py`). Tests tagged `tdd_expected_fail` that fail have their result inverted to pass (expected failure); tests that diff --git a/robot/e2e/wf15_disaster_recovery.robot b/robot/e2e/wf15_disaster_recovery.robot new file mode 100644 index 000000000..7048b68c4 --- /dev/null +++ b/robot/e2e/wf15_disaster_recovery.robot @@ -0,0 +1,515 @@ +*** Settings *** +Documentation E2E test for WF15: Disaster Recovery — Rollback a Failed Apply. +... +... Intermediate scenario using the ``trusted`` automation profile. +... A plan to optimize database connection handling is executed, +... then the team investigates using plan status, plan tree, plan diff, +... plan explain (with forensic flags), plan rollback, plan correct +... (revert mode), and plan diff --correction. +... +... Zero mocking — real CLI, real LLM API keys, real subprocess execution. +Resource common_e2e.resource +Suite Setup E2E Suite Setup +Suite Teardown E2E Suite Teardown + +*** Variables *** +${ACTION_NAME_BASE} local/wf15-optimize-db +${RESOURCE_NAME_BASE} local/wf15-repo +${PROJECT_NAME_BASE} local/wf15-project + +${CONNPOOL_FIXTURE} SEPARATOR=\n +... """Database connection pool module with known issues.""" +... import sqlite3 +... ${EMPTY} +... ${EMPTY} +... class ConnectionPool: +... ${SPACE}${SPACE}${SPACE}${SPACE}"""Simple connection pool with fixed size.""" +... ${EMPTY} +... ${SPACE}${SPACE}${SPACE}${SPACE}def __init__(self, db_path: str, size: int = 5): +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}self.db_path = db_path +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}self.size = size +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}self._pool: list[sqlite3.Connection] = [] +... ${EMPTY} +... ${SPACE}${SPACE}${SPACE}${SPACE}def acquire(self) -> sqlite3.Connection: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}"""Acquire a connection from the pool.""" +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}if self._pool: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return self._pool.pop() +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return sqlite3.connect(self.db_path) +... ${EMPTY} +... ${SPACE}${SPACE}${SPACE}${SPACE}def release(self, conn: sqlite3.Connection) -> None: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}"""Return a connection to the pool.""" +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}if len(self._pool) < self.size: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}self._pool.append(conn) +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}else: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}conn.close() +... ${EMPTY} +... ${EMPTY} +... # Global pool instance — potential exhaustion if not managed +... pool = ConnectionPool("app.db", size=5) + +${APP_FIXTURE} SEPARATOR=\n +... """Application layer that uses the connection pool.""" +... from connpool import pool +... ${EMPTY} +... ${EMPTY} +... def get_users() -> list[dict]: +... ${SPACE}${SPACE}${SPACE}${SPACE}"""Fetch all users from the database.""" +... ${SPACE}${SPACE}${SPACE}${SPACE}conn = pool.acquire() +... ${SPACE}${SPACE}${SPACE}${SPACE}try: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}cursor = conn.cursor() +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}cursor.execute("SELECT id, name FROM users") +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return [{"id": r[0], "name": r[1]} for r in cursor.fetchall()] +... ${SPACE}${SPACE}${SPACE}${SPACE}finally: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}pool.release(conn) +... ${EMPTY} +... ${EMPTY} +... def update_user(user_id: int, name: str) -> None: +... ${SPACE}${SPACE}${SPACE}${SPACE}"""Update a user's name.""" +... ${SPACE}${SPACE}${SPACE}${SPACE}conn = pool.acquire() +... ${SPACE}${SPACE}${SPACE}${SPACE}try: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}cursor = conn.cursor() +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}cursor.execute("UPDATE users SET name=? WHERE id=?", (name, user_id)) +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}conn.commit() +... ${SPACE}${SPACE}${SPACE}${SPACE}finally: +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}pool.release(conn) + +*** Test Cases *** +WF15 Disaster Recovery Rollback Workflow + [Documentation] End-to-end disaster recovery workflow: execute plan with + ... trusted profile, investigate via status/tree/diff/explain, + ... rollback to checkpoint, correct with guidance, re-apply. + [Tags] E2E + [Timeout] 30 minutes + + Skip If No LLM Keys + + # ── Generate run-unique suffix for parallel CI safety ── + ${run_suffix}= Evaluate __import__('uuid').uuid4().hex[:12] + ${action_name}= Set Variable ${ACTION_NAME_BASE}-${run_suffix} + ${resource_name}= Set Variable ${RESOURCE_NAME_BASE}-${run_suffix} + ${project_name}= Set Variable ${PROJECT_NAME_BASE}-${run_suffix} + + # ── Initialize database ── + ${r_init}= Run CleverAgents Command + ... init --yes --force + Should Not Contain ${r_init.stdout}${r_init.stderr} Traceback + Should Not Contain ${r_init.stdout}${r_init.stderr} INTERNAL + + # ── 1. Create temp repo with connection pool module ── + ${repo_dir}= Create Temp Git Repo wf15-disaster-repo-${run_suffix} + Create Directory ${repo_dir}${/}src + Create File ${repo_dir}${/}src${/}connpool.py ${CONNPOOL_FIXTURE} + Create File ${repo_dir}${/}src${/}app.py ${APP_FIXTURE} + ${git_add}= Run Process git add . cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr} + ${git_commit}= Run Process git commit -m Add connection pool module cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr} + ${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${repo_dir} timeout=60s on_timeout=kill + ${branch}= Strip String ${branch_result.stdout} + + # ── 2. Register resource and project ── + ${r_resource}= Run CleverAgents Command + ... resource add git-checkout ${resource_name} + ... --path ${repo_dir} --branch ${branch} + Should Not Contain ${r_resource.stdout}${r_resource.stderr} Traceback + Should Not Contain ${r_resource.stdout}${r_resource.stderr} INTERNAL + Output Should Contain ${r_resource} ${resource_name} + + ${r_project}= Run CleverAgents Command + ... project create ${project_name} + ... --description WF15 disaster recovery project + ... --resource ${resource_name} + Should Not Contain ${r_project.stdout}${r_project.stderr} Traceback + Should Not Contain ${r_project.stdout}${r_project.stderr} INTERNAL + Output Should Contain ${r_project} ${project_name} + + # ── 3. Create action with trusted profile ── + ${action_yaml}= Catenate SEPARATOR=\n + ... name: ${action_name} + ... description: Optimize database connection pool handling + ... definition_of_done: Connection pool properly manages connections with overflow handling + ... strategy_actor: openai/gpt-4 + ... execution_actor: openai/gpt-4 + ... automation_profile: trusted + ${yaml_path}= Set Variable ${SUITE_HOME}${/}wf15_action.yaml + Create File ${yaml_path} ${action_yaml} + ${r_action}= Run CleverAgents Command + ... action create --config ${yaml_path} + Should Not Contain ${r_action.stdout}${r_action.stderr} Traceback + Should Not Contain ${r_action.stdout}${r_action.stderr} INTERNAL + Output Should Contain ${r_action} ${action_name} + + # ── 4. Plan use ── + ${r_use}= Run CleverAgents Command + ... plan use ${action_name} ${project_name} + ... --automation-profile trusted --format plain + Should Not Be Empty ${r_use.stdout} + ${plan_id}= Extract Plan Id ${r_use.stdout}${r_use.stderr} + Should Not Be Empty ${plan_id} Could not extract plan ID from plan use output + Log Plan ID: ${plan_id} + + # ── 5. Plan execute — strategize phase ── + ${r_strategize}= Run CleverAgents Command + ... plan execute ${plan_id} --format plain + ... expected_rc=${0} timeout=300s + Log Strategize stdout: ${r_strategize.stdout} + Should Not Contain ${r_strategize.stdout}${r_strategize.stderr} Traceback + Should Not Contain ${r_strategize.stdout}${r_strategize.stderr} INTERNAL + + # ── 5b. Verify phase advancement after strategize ── + ${r_post_strat}= Run CleverAgents Command + ... plan status ${plan_id} --format plain + ... timeout=120s + Log Post-strategize status: ${r_post_strat.stdout} + Should Not Contain ${r_post_strat.stdout}${r_post_strat.stderr} Traceback + Should Not Contain ${r_post_strat.stdout}${r_post_strat.stderr} INTERNAL + + # ── 6. Plan execute — execute phase ── + ${r_execute}= Run CleverAgents Command + ... plan execute ${plan_id} --format plain + ... expected_rc=${0} timeout=300s + Log Execute stdout: ${r_execute.stdout} + Should Not Contain ${r_execute.stdout}${r_execute.stderr} Traceback + Should Not Contain ${r_execute.stdout}${r_execute.stderr} INTERNAL + + # ── 7. Plan apply — trigger the apply that may error ── + ${r_initial_apply}= Run CleverAgents Command + ... plan apply ${plan_id} --yes --format plain + ... expected_rc=None timeout=300s + Log Initial apply stdout: ${r_initial_apply.stdout} + Log Initial apply stderr: ${r_initial_apply.stderr} + Should Not Contain ${r_initial_apply.stdout}${r_initial_apply.stderr} Traceback + Should Not Contain ${r_initial_apply.stdout}${r_initial_apply.stderr} INTERNAL + + # ── 8. Plan status — inspect current state ── + ${r_status}= Run CleverAgents Command + ... plan status ${plan_id} --format plain + ... timeout=120s + Log Plan status: ${r_status.stdout} + Should Not Contain ${r_status.stdout}${r_status.stderr} Traceback + Should Not Contain ${r_status.stdout}${r_status.stderr} INTERNAL + Should Not Be Empty ${r_status.stdout} + # AC#3: Verify status output contains specific state values (not generic field labels) + ${status_lower}= Evaluate ($r_status.stdout + $r_status.stderr).lower() + ${has_state_info}= Evaluate 'processing' in $status_lower or 'errored' in $status_lower or 'applied' in $status_lower + Should Be True ${has_state_info} + ... Plan status should contain state information (processing, errored, or applied) + # Issue #3: Verify plan is in errored/failed state before recovery (spec WF15 shows State: errored) + ${has_error_state}= Evaluate 'errored' in $status_lower or 'failed' in $status_lower or 'error' in $status_lower + IF not ${has_error_state} + Log Plan did not enter errored/failed state after apply — LLM may have produced a successful plan. Recovery steps still exercised. WARN + END + + # ── 9. Plan tree — view decision tree and extract decision ID ── + ${r_tree}= Run CleverAgents Command + ... plan tree ${plan_id} --format json + ... timeout=120s + Log Plan tree: ${r_tree.stdout} + Should Not Contain ${r_tree.stdout}${r_tree.stderr} Traceback + Should Not Contain ${r_tree.stdout}${r_tree.stderr} INTERNAL + Should Not Be Empty ${r_tree.stdout} Plan tree output should not be empty + # AC#4: Verify the tree contains decision nodes + Should Contain ${r_tree.stdout} decision_id + ... Plan tree should contain at least one decision_id + # AC#4: Check for ROOT CAUSE annotation — request plain format for display annotation + ${r_tree_plain}= Run CleverAgents Command + ... plan tree ${plan_id} --format plain + ... timeout=120s + Log Plan tree (plain): ${r_tree_plain.stdout} + ${tree_plain_lower}= Evaluate ($r_tree_plain.stdout + $r_tree_plain.stderr).lower() + # Also check JSON output and status output for root cause references + ${tree_json_lower}= Evaluate ($r_tree.stdout + $r_tree.stderr).lower() + ${has_root_cause}= Evaluate 'root_cause' in $tree_plain_lower or 'root cause' in $tree_plain_lower or 'root_cause' in $tree_json_lower or 'root cause' in $tree_json_lower or 'root_cause' in $status_lower or 'root cause' in $status_lower + IF not ${has_root_cause} + Log ROOT CAUSE annotation not found in tree or status output. The LLM may not have annotated a root cause for this plan. WARN + END + # Extract the first decision_id specifically from JSON tree output + ${decision_id}= Extract Decision Id From Tree ${r_tree.stdout} + Should Not Be Empty ${decision_id} + ... Could not extract decision ID from plan tree output + Log Extracted decision ID: ${decision_id} + + # ── 9b. Plan diff — investigate applied changes (spec WF15 step) ── + ${r_diff_invest}= Run CleverAgents Command + ... plan diff ${plan_id} --format plain + ... expected_rc=None timeout=120s + Log Investigation diff stdout: ${r_diff_invest.stdout} + Log Investigation diff stderr: ${r_diff_invest.stderr} + Should Be Equal As Integers ${r_diff_invest.rc} 0 + ... Plan diff command failed with rc=${r_diff_invest.rc}: ${r_diff_invest.stderr} + Should Not Contain ${r_diff_invest.stdout}${r_diff_invest.stderr} Traceback + Should Not Contain ${r_diff_invest.stdout}${r_diff_invest.stderr} INTERNAL + + # ── 10. Plan explain with forensic flags ── + ${r_explain}= Run CleverAgents Command + ... plan explain ${decision_id} + ... --show-context --show-reasoning --format plain + ... timeout=120s + Log Explain stdout: ${r_explain.stdout} + Log Explain stderr: ${r_explain.stderr} + Should Not Contain ${r_explain.stdout}${r_explain.stderr} Traceback + Should Not Contain ${r_explain.stdout}${r_explain.stderr} INTERNAL + # AC#5: Verify explain output contains meaningful forensic content + Should Not Be Empty ${r_explain.stdout} + ... Plan explain should produce non-empty output + ${explain_lower}= Evaluate ($r_explain.stdout + $r_explain.stderr).lower() + ${has_forensic_content}= Evaluate 'rationale' in $explain_lower or 'snapshot' in $explain_lower or 'alternative' in $explain_lower or 'chosen' in $explain_lower or 'confidence' in $explain_lower or 'context' in $explain_lower + Should Be True ${has_forensic_content} + ... Plan explain output should contain forensic content (rationale, snapshot, alternative, chosen, confidence, context) + + # ── 11. Extract checkpoint ID from plan status JSON ── + ${r_status_json}= Run CleverAgents Command + ... plan status ${plan_id} --format json + ... expected_rc=None timeout=120s + Log Status JSON stdout: ${r_status_json.stdout} + Log Status JSON stderr: ${r_status_json.stderr} + Should Not Contain ${r_status_json.stdout}${r_status_json.stderr} Traceback + Should Not Contain ${r_status_json.stdout}${r_status_json.stderr} INTERNAL + # Extract checkpoint_id from status JSON (last_checkpoint_id or checkpoint_id field) + ${checkpoint_id}= Extract Checkpoint Id From Status ${r_status_json.stdout} + Log Extracted checkpoint ID: ${checkpoint_id} + + # Also exercise plan artifacts (AC requirement) and try extracting checkpoint from there + ${r_artifacts}= Run CleverAgents Command + ... plan artifacts ${plan_id} --format json + ... expected_rc=None timeout=120s + Log Artifacts stdout: ${r_artifacts.stdout} + Log Artifacts stderr: ${r_artifacts.stderr} + Should Not Contain ${r_artifacts.stdout}${r_artifacts.stderr} Traceback + Should Not Contain ${r_artifacts.stdout}${r_artifacts.stderr} INTERNAL + IF '${checkpoint_id}' == '' + ${checkpoint_id}= Extract Checkpoint Id From Status ${r_artifacts.stdout} + Log Checkpoint ID from artifacts: ${checkpoint_id} + END + + # ── 12. Plan rollback — restore to pre-change state ── + # Checkpoint creation depends on LLM behavior during apply — not all plan + # executions produce checkpoints (e.g. if apply succeeds without errors, or + # if the execution environment doesn't support checkpointing). When no + # checkpoint is available, the remaining recovery steps (AC#6–AC#9) cannot + # be exercised and the test skips gracefully. Investigate if this skip + # occurs consistently — a persistent skip indicates the fixture or LLM + # prompt needs adjustment so that apply produces a checkpoint. + Skip If '${checkpoint_id}' == '' + ... No checkpoint ID available — checkpoint creation depends on LLM behavior during apply. Recovery steps (AC6–AC9) skipped. + ${r_rollback}= Run CleverAgents Command + ... plan rollback ${plan_id} ${checkpoint_id} --yes + ... expected_rc=None timeout=180s + Log Rollback stdout: ${r_rollback.stdout} + Log Rollback stderr: ${r_rollback.stderr} + Should Be Equal As Integers ${r_rollback.rc} 0 + ... Plan rollback command failed with rc=${r_rollback.rc}: ${r_rollback.stderr} + Should Not Contain ${r_rollback.stdout}${r_rollback.stderr} Traceback + Should Not Contain ${r_rollback.stdout}${r_rollback.stderr} INTERNAL + # AC#6: Verify rollback output references rollback/restore + ${rollback_lower}= Evaluate ($r_rollback.stdout + $r_rollback.stderr).lower() + ${has_rollback_info}= Evaluate 'rollback' in $rollback_lower or 'restored' in $rollback_lower or 'revert' in $rollback_lower or 'checkpoint' in $rollback_lower + Should Be True ${has_rollback_info} + ... Rollback output should contain rollback/restore confirmation + + # ── 13. Plan correct — revert with corrective guidance ── + ${r_correct}= Run CleverAgents Command + ... plan correct ${decision_id} --mode revert + ... --guidance Use context managers and limit pool overflow to prevent exhaustion + ... --yes --format plain + ... expected_rc=None timeout=300s + Log Correct stdout: ${r_correct.stdout} + Log Correct stderr: ${r_correct.stderr} + Should Be Equal As Integers ${r_correct.rc} 0 + ... Plan correct command failed with rc=${r_correct.rc}: ${r_correct.stderr} + Should Not Contain ${r_correct.stdout}${r_correct.stderr} Traceback + Should Not Contain ${r_correct.stdout}${r_correct.stderr} INTERNAL + # AC#7: Verify correction output references correction/applied + ${correct_lower}= Evaluate ($r_correct.stdout + $r_correct.stderr).lower() + ${has_correct_info}= Evaluate 'correction_id' in $correct_lower or 'corrected' in $correct_lower or 'applied' in $correct_lower or 'revert' in $correct_lower + Should Be True ${has_correct_info} + ... Correct output should contain correction confirmation (correction_id, corrected, applied, revert) + # Extract correction_id for plan diff --correction (rc==0 already guaranteed by assertion above) + ${correction_id}= Extract Correction Id From Output ${r_correct.stdout} + Log Extracted correction ID: ${correction_id} + + # ── 14. Plan diff --correction — compare original vs corrected ── + # Spec shows: plan diff (--correction |) + IF '${correction_id}' != '' + ${r_diff_corr}= Run CleverAgents Command + ... plan diff --correction ${correction_id} --format plain + ... expected_rc=None timeout=120s + ELSE + ${r_diff_corr}= Run CleverAgents Command + ... plan diff ${plan_id} --format plain + ... expected_rc=None timeout=120s + END + Log Correction diff stdout: ${r_diff_corr.stdout} + Log Correction diff stderr: ${r_diff_corr.stderr} + Should Be Equal As Integers ${r_diff_corr.rc} 0 + ... Plan diff command failed with rc=${r_diff_corr.rc}: ${r_diff_corr.stderr} + Should Not Contain ${r_diff_corr.stdout}${r_diff_corr.stderr} Traceback + Should Not Contain ${r_diff_corr.stdout}${r_diff_corr.stderr} INTERNAL + # AC#8: Verify diff output contains meaningful diff content + ${diff_combined}= Set Variable ${r_diff_corr.stdout}${r_diff_corr.stderr} + Should Not Be Empty ${diff_combined} + ... Plan diff output should not be empty + ${diff_lower}= Evaluate $diff_combined.lower() + ${has_diff_content}= Evaluate 'diff' in $diff_lower or 'correction' in $diff_lower or '---' in $diff_lower or '+++' in $diff_lower or 'original' in $diff_lower or 'modified' in $diff_lower + Should Be True ${has_diff_content} + ... Plan diff output should contain diff-related content (diff, correction, ---, +++, original, modified) + + # ── 15. Plan execute — re-execute after correction ── + ${r_reexec}= Run CleverAgents Command + ... plan execute ${plan_id} --format plain + ... expected_rc=None timeout=300s + Log Re-execute stdout: ${r_reexec.stdout} + Log Re-execute stderr: ${r_reexec.stderr} + Should Be Equal As Integers ${r_reexec.rc} 0 + ... Plan re-execute command failed with rc=${r_reexec.rc}: ${r_reexec.stderr} + Should Not Contain ${r_reexec.stdout}${r_reexec.stderr} Traceback + Should Not Contain ${r_reexec.stdout}${r_reexec.stderr} INTERNAL + + # ── 16. Plan apply — final apply ── + ${r_apply}= Run CleverAgents Command + ... plan apply ${plan_id} --yes --format plain + ... expected_rc=None timeout=300s + Log Apply output: ${r_apply.stdout} + Log Apply stderr: ${r_apply.stderr} + Should Be Equal As Integers ${r_apply.rc} 0 + ... Final apply command failed with rc=${r_apply.rc}: ${r_apply.stderr} + Should Not Contain ${r_apply.stdout}${r_apply.stderr} Traceback + Should Not Contain ${r_apply.stdout}${r_apply.stderr} INTERNAL + + # ── 17. Final status check ── + ${r_final}= Run CleverAgents Command + ... plan status ${plan_id} --format plain + ... timeout=120s + Should Not Be Empty ${r_final.stdout} + Should Not Contain ${r_final.stdout}${r_final.stderr} Traceback + Should Not Contain ${r_final.stdout}${r_final.stderr} INTERNAL + # AC#9: Verify final status shows a successful recovery state + ${final_lower}= Evaluate ($r_final.stdout + $r_final.stderr).lower() + ${has_terminal_state}= Evaluate 'applied' in $final_lower or 'completed' in $final_lower + Should Be True ${has_terminal_state} + ... Final plan status should indicate successful recovery (applied or completed) + Log Final status: ${r_final.stdout} + Log WF15 Disaster Recovery E2E test completed + +*** Keywords *** +Extract Plan Id + [Documentation] Extract a plan ID (ULID/UUID) from CLI output text. + ... + ... Delegates to ``Extract First Id From Text`` for ULID/UUID + ... scanning, then falls back to ``plan_id`` field extraction. + ... Returns the first match or EMPTY. + ... + ... NOTE: The M1 test suite defines a similar keyword with a + ... different signature. Consolidation into common_e2e.resource + ... is deferred to a separate ticket to avoid touching unrelated + ... test files. + [Arguments] ${text} + # Strategy 1: ULID or UUID via shared keyword + ${first_id}= Extract First Id From Text ${text} + IF '${first_id}' != '' + RETURN ${first_id} + END + # Strategy 2: plan_id field extraction (tightened regex) + ${field_matches}= Get Regexp Matches ${text} plan_id[:\\s]+([0-9A-HJ-KM-NP-TV-Z]{26}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}) 1 flags=IGNORECASE + ${field_count}= Get Length ${field_matches} + IF ${field_count} > 0 + RETURN ${field_matches}[0] + END + RETURN ${EMPTY} + +Extract First Id From Text + [Documentation] Extract the first ULID or UUID from arbitrary text. + ... + ... Useful for extracting decision IDs, checkpoint IDs, or + ... correction IDs from CLI output. Returns EMPTY if none found. + [Arguments] ${text} + # ULID — 26 Crockford Base32 chars (excludes I, L, O, U) + ${ulid_matches}= Get Regexp Matches ${text} [0-9A-HJ-KM-NP-TV-Z]{26} flags=IGNORECASE + ${ulid_count}= Get Length ${ulid_matches} + IF ${ulid_count} > 0 + RETURN ${ulid_matches}[0] + END + # UUID — 8-4-4-4-12 hex pattern + ${uuid_matches}= Get Regexp Matches ${text} [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} flags=IGNORECASE + ${uuid_count}= Get Length ${uuid_matches} + IF ${uuid_count} > 0 + RETURN ${uuid_matches}[0] + END + RETURN ${EMPTY} + +Extract Decision Id From Tree + [Documentation] Extract the first ``decision_id`` value from JSON tree output. + ... + ... Uses a targeted regex to match ``"decision_id": ""`` fields + ... in the JSON, avoiding extraction of ``plan_id`` or other IDs that + ... appear earlier in the output. Returns EMPTY if no decision_id found. + [Arguments] ${text} + # Target decision_id fields specifically in JSON output + ${decision_matches}= Get Regexp Matches ${text} "decision_id"\\s*:\\s*"([0-9A-HJ-KM-NP-TV-Z]{26})" 1 flags=IGNORECASE + ${match_count}= Get Length ${decision_matches} + IF ${match_count} > 0 + RETURN ${decision_matches}[0] + END + # Fallback: try UUID-style decision_id + ${uuid_matches}= Get Regexp Matches ${text} "decision_id"\\s*:\\s*"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" 1 flags=IGNORECASE + ${uuid_count}= Get Length ${uuid_matches} + IF ${uuid_count} > 0 + RETURN ${uuid_matches}[0] + END + RETURN ${EMPTY} + +Extract Correction Id From Output + [Documentation] Extract the ``correction_id`` from ``plan correct`` stdout. + ... + ... Uses a targeted regex to match the ``"correction_id"`` field + ... in the output, avoiding stray ULIDs from unrelated fields. + ... Falls back to ``Extract First Id From Text`` on stdout only. + ... Returns EMPTY if not found. + [Arguments] ${text} + # Target correction_id / correction field specifically + ${corr_matches}= Get Regexp Matches ${text} "?correction_id"?\\s*[:=]\\s*"?([0-9A-HJ-KM-NP-TV-Z]{26})"? 1 flags=IGNORECASE + ${corr_count}= Get Length ${corr_matches} + IF ${corr_count} > 0 + RETURN ${corr_matches}[0] + END + # Try UUID-style correction_id + ${corr_uuid}= Get Regexp Matches ${text} "?correction_id"?\\s*[:=]\\s*"?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"? 1 flags=IGNORECASE + ${corr_uuid_count}= Get Length ${corr_uuid} + IF ${corr_uuid_count} > 0 + RETURN ${corr_uuid}[0] + END + # Fallback: first ULID/UUID from stdout only (not stderr) + ${fallback_id}= Extract First Id From Text ${text} + RETURN ${fallback_id} + +Extract Checkpoint Id From Status + [Documentation] Extract ``last_checkpoint_id`` from ``plan status --format json`` output. + ... + ... Uses a targeted regex to match the ``last_checkpoint_id`` or + ... ``checkpoint_id`` field in JSON output. Returns EMPTY if not found. + [Arguments] ${text} + # Try last_checkpoint_id first (ULID) + ${lcp_matches}= Get Regexp Matches ${text} "last_checkpoint_id"\\s*:\\s*"([0-9A-HJ-KM-NP-TV-Z]{26})" 1 flags=IGNORECASE + ${lcp_count}= Get Length ${lcp_matches} + IF ${lcp_count} > 0 + RETURN ${lcp_matches}[0] + END + # Try checkpoint_id (ULID) + ${cp_matches}= Get Regexp Matches ${text} "checkpoint_id"\\s*:\\s*"([0-9A-HJ-KM-NP-TV-Z]{26})" 1 flags=IGNORECASE + ${cp_count}= Get Length ${cp_matches} + IF ${cp_count} > 0 + RETURN ${cp_matches}[0] + END + # Try last_checkpoint_id with UUID pattern + ${lcp_uuid}= Get Regexp Matches ${text} "last_checkpoint_id"\\s*:\\s*"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" 1 flags=IGNORECASE + ${lcp_uuid_count}= Get Length ${lcp_uuid} + IF ${lcp_uuid_count} > 0 + RETURN ${lcp_uuid}[0] + END + # Try checkpoint_id with UUID pattern + ${cp_uuid}= Get Regexp Matches ${text} "checkpoint_id"\\s*:\\s*"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" 1 flags=IGNORECASE + ${cp_uuid_count}= Get Length ${cp_uuid} + IF ${cp_uuid_count} > 0 + RETURN ${cp_uuid}[0] + END + RETURN ${EMPTY} -- 2.52.0