diff --git a/CHANGELOG.md b/CHANGELOG.md index 36aab2df1..c5d88c90f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1021,6 +1021,11 @@ attribute 'db'` after `agents init`. Root cause: `_get_session_service()` when API keys are absent, and `--exclude E2E` on the standard integration test session. Includes a minimal smoke test exercising `agents --version` and `agents --help`. (#740) +- Added E2E acceptance test for M3 (v3.2.0) milestone: decisions, validations, + and invariants. Tests decision recording, decision tree visualization + (plan tree), decision explanation (plan explain), invariant management + (invariant add/list), and decision correction (plan correct --mode=revert) + with real LLM API keys and zero mocking. (#743) - Implemented `tdd_expected_fail` tag handling in Robot Framework via a Listener v3 module (`robot/tdd_expected_fail_listener.py`). Tests tagged `tdd_expected_fail` that fail have their result inverted to pass (expected failure); tests that diff --git a/robot/e2e/common_e2e.resource b/robot/e2e/common_e2e.resource index 29bd6e3ff..2f4525050 100644 --- a/robot/e2e/common_e2e.resource +++ b/robot/e2e/common_e2e.resource @@ -186,6 +186,30 @@ Extract JSON From Stdout END RETURN ${json_obj} +Extract Plan Id + [Documentation] Extract a plan ID (ULID or UUID) from CLI output. + ... + ... Searches stdout (and optionally stderr) for a 26-character + ... ULID (Crockford Base32), a standard UUID, or a + ... ``plan_id: `` pattern. + ... Returns the first match, or EMPTY if none found. + [Arguments] ${stdout} ${stderr}=${EMPTY} + ${combined}= Set Variable ${stdout}\n${stderr} + # M5: Use IF/RETURN instead of deprecated Return From Keyword If + # Try ULID (Crockford Base32: excludes I, L, O, U) + @{ulid_matches}= Get Regexp Matches ${combined} [0-9A-HJKMNP-TV-Z]{26} + ${ulid_count}= Get Length ${ulid_matches} + IF ${ulid_count} > 0 RETURN ${ulid_matches}[0] + # Try UUID + @{uuid_matches}= Get Regexp Matches ${combined} [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ${uuid_count}= Get Length ${uuid_matches} + IF ${uuid_count} > 0 RETURN ${uuid_matches}[0] + # Try plan_id: pattern + @{id_matches}= Get Regexp Matches ${combined} plan_id[:\\\\s]+(\\w+) 1 + ${id_count}= Get Length ${id_matches} + IF ${id_count} > 0 RETURN ${id_matches}[0] + RETURN ${EMPTY} + Link Resource To Project [Documentation] Link the suite-level workspace resource to a project. ... diff --git a/robot/e2e/m1_acceptance.robot b/robot/e2e/m1_acceptance.robot index 014c41baa..73e68f0e9 100644 --- a/robot/e2e/m1_acceptance.robot +++ b/robot/e2e/m1_acceptance.robot @@ -121,15 +121,3 @@ M1 Full Plan Lifecycle # Git repo still has commits (verified above) Log M1 Full Plan Lifecycle E2E test completed successfully -*** Keywords *** -Extract Plan Id - [Documentation] Extract a ULID-style plan ID from command output. - ... - ... Searches for a 26-character alphanumeric ULID pattern - ... in the output text. Returns the first match or EMPTY. - [Arguments] ${text} - # ULID pattern: 26 uppercase alphanumeric characters (Crockford Base32) - ${matches}= Get Regexp Matches ${text} [0-9A-HJ-NP-Z]{26} flags=IGNORECASE - ${count}= Get Length ${matches} - ${plan_id}= Set Variable If ${count} > 0 ${matches}[0] ${EMPTY} - RETURN ${plan_id} diff --git a/robot/e2e/m2_acceptance.robot b/robot/e2e/m2_acceptance.robot index 104521bca..8a3d18df6 100644 --- a/robot/e2e/m2_acceptance.robot +++ b/robot/e2e/m2_acceptance.robot @@ -88,9 +88,9 @@ M2 Full Actor Compiler And LLM Integration ${r_use}= Run CleverAgents Command ... plan use ${ACTION_NAME} ${PROJECT_NAME} --format plain Should Not Be Empty ${r_use.stdout} - ${plan_ids}= Get Regexp Matches ${r_use.stdout} [0-9A-Z]{26} - Should Not Be Empty ${plan_ids} msg=Expected a ULID plan ID in plan use output - ${plan_id}= Set Variable ${plan_ids}[0] + ${plan_id}= Extract Plan Id ${r_use.stdout} ${r_use.stderr} + Should Not Be Empty ${plan_id} + ... Could not extract plan ID from plan use output.\nSTDOUT: ${r_use.stdout}\nSTDERR: ${r_use.stderr} Log Extracted plan_id: ${plan_id} # ---- Step 6: Plan execute — strategize phase ---- diff --git a/robot/e2e/m3_acceptance.robot b/robot/e2e/m3_acceptance.robot new file mode 100644 index 000000000..b00649c0b --- /dev/null +++ b/robot/e2e/m3_acceptance.robot @@ -0,0 +1,271 @@ +*** Settings *** +Documentation E2E acceptance test for M3 (v3.2.0): Decisions, Validations, and Invariants. +... +... Exercises decision recording during plan execution, decision tree +... visualization (plan tree), decision explanation (plan explain), +... invariant management (invariant add/list), and decision correction +... (plan correct --mode=revert) with real LLM API keys. +... +... Zero mocking — all CLI invocations use real providers. +Resource common_e2e.resource +Library String +Suite Setup E2E Suite Setup +Suite Teardown E2E Suite Teardown + +*** Variables *** +${ACTION_NAME} local/m3-e2e-action +${RESOURCE_NAME} local/m3-e2e-repo +${PROJECT_NAME} local/m3-e2e-project + +*** Test Cases *** +M3 Decision Recording And Tree Visualization + [Documentation] End-to-end test for M3 milestone: decision recording, tree + ... visualization, explanation, invariant management, and correction. + ... + ... Exercises the full M3 feature set through the CLI with real + ... LLM API keys. Validates structural output, not exact text. + [Tags] E2E tdd_issue tdd_issue_1022 tdd_expected_fail + [Timeout] 15 minutes + Skip If No LLM Keys + + # ── Initialize database ── + ${r_init}= Run CleverAgents Command + ... init --yes --force + Should Not Contain ${r_init.stdout}${r_init.stderr} Traceback + + # ── 1. Create temp git repo with sample project files ── + ${repo_dir}= Create Temp Git Repo m3-acceptance-repo + Create Directory ${repo_dir}${/}src + ${auth_code}= Catenate SEPARATOR=\n + ... """Authentication module with raw SQL queries.""" + ... import sqlite3 + ... ${EMPTY} + ... ${EMPTY} + ... def authenticate(username: str, password: str) -> bool: + ... ${SPACE}${SPACE}${SPACE}${SPACE}"""Authenticate user against database.""" + ... ${SPACE}${SPACE}${SPACE}${SPACE}conn = sqlite3.connect("users.db") + ... ${SPACE}${SPACE}${SPACE}${SPACE}cursor = conn.cursor() + ... ${SPACE}${SPACE}${SPACE}${SPACE}cursor.execute( + ... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}"SELECT * FROM users WHERE name=? AND pass=?", + ... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}(username, password), + ... ${SPACE}${SPACE}${SPACE}${SPACE}) + ... ${SPACE}${SPACE}${SPACE}${SPACE}result = cursor.fetchone() + ... ${SPACE}${SPACE}${SPACE}${SPACE}conn.close() + ... ${SPACE}${SPACE}${SPACE}${SPACE}return result is not None + ... ${EMPTY} + ... ${EMPTY} + ... def get_user(user_id: int) -> dict: + ... ${SPACE}${SPACE}${SPACE}${SPACE}"""Fetch user by ID.""" + ... ${SPACE}${SPACE}${SPACE}${SPACE}conn = sqlite3.connect("users.db") + ... ${SPACE}${SPACE}${SPACE}${SPACE}cursor = conn.cursor() + ... ${SPACE}${SPACE}${SPACE}${SPACE}cursor.execute("SELECT * FROM users WHERE id=?", (user_id,)) + ... ${SPACE}${SPACE}${SPACE}${SPACE}row = cursor.fetchone() + ... ${SPACE}${SPACE}${SPACE}${SPACE}conn.close() + ... ${SPACE}${SPACE}${SPACE}${SPACE}return {"id": row[0], "name": row[1]} if row else {} + Create File ${repo_dir}${/}src${/}auth.py ${auth_code} + # H1: Assert rc on git add/commit + ${r_git_add}= Run Process git add . cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${r_git_add.rc} 0 msg=git add failed (rc=${r_git_add.rc}) + ${r_git_commit}= Run Process git commit -m Add auth module cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${r_git_commit.rc} 0 msg=git commit failed (rc=${r_git_commit.rc}) + ${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${repo_dir} timeout=60s on_timeout=kill + ${branch}= Strip String ${branch_result.stdout} + + # ── 2. Register resource and create project ── + ${r_resource}= Run CleverAgents Command + ... resource add git-checkout ${RESOURCE_NAME} + ... --path ${repo_dir} --branch ${branch} + Should Not Contain ${r_resource.stdout}${r_resource.stderr} Traceback + Output Should Contain ${r_resource} ${RESOURCE_NAME} + + ${r_project}= Run CleverAgents Command + ... project create ${PROJECT_NAME} + ... --description M3 E2E acceptance project + ... --resource ${RESOURCE_NAME} + Should Not Contain ${r_project.stdout}${r_project.stderr} Traceback + Output Should Contain ${r_project} ${PROJECT_NAME} + + # ── 3. Add invariants at project scope ── + ${inv_result}= Run CleverAgents Command + ... invariant add --project ${PROJECT_NAME} + ... API function signatures must not change + Log Invariant add stdout: ${inv_result.stdout} + Log Invariant add stderr: ${inv_result.stderr} + Should Not Contain ${inv_result.stdout}${inv_result.stderr} Traceback + Output Should Contain ${inv_result} invariant + Output Should Contain ${inv_result} API function signatures + + # ── 4. List invariants to verify persistence ── + ${inv_list}= Run CleverAgents Command + ... invariant list --project ${PROJECT_NAME} + Log Invariant list stdout: ${inv_list.stdout} + Log Invariant list stderr: ${inv_list.stderr} + Should Not Contain ${inv_list.stdout}${inv_list.stderr} Traceback + Output Should Contain ${inv_list} API function signatures + + # ── 5. Create action YAML — C2: dynamic actor selection ── + # Select actor based on available API key to avoid misleading failures + # when only one provider's key is available. + ${actor}= Set Variable If '%{ANTHROPIC_API_KEY}' != '' + ... anthropic/claude-sonnet-4 openai/gpt-4 + ${action_yaml}= Catenate SEPARATOR=\n + ... name: ${ACTION_NAME} + ... description: Refactor auth module from raw SQL to ORM + ... definition_of_done: Replace raw SQL queries with ORM while preserving API + ... strategy_actor: ${actor} + ... execution_actor: ${actor} + ${action_yaml_path}= Set Variable ${SUITE_HOME}${/}m3_action.yaml + Create File ${action_yaml_path} ${action_yaml} + ${r_action}= Run CleverAgents Command + ... action create --config ${action_yaml_path} + Should Not Contain ${r_action.stdout}${r_action.stderr} Traceback + Output Should Contain ${r_action} ${ACTION_NAME} + + # ── 6. Plan use — create plan ── + ${r_use}= Run CleverAgents Command + ... plan use ${ACTION_NAME} ${PROJECT_NAME} --format plain + Should Not Be Empty ${r_use.stdout} + ${plan_id}= Extract Plan Id ${r_use.stdout} ${r_use.stderr} + Should Not Be Empty ${plan_id} + ... Could not extract plan ID from plan use output. + Log Extracted plan_id: ${plan_id} + + # ── 7. Plan execute — strategize phase (decisions recorded) ── + ${r_strategize}= Run CleverAgents Command + ... plan execute ${plan_id} --format plain + ... timeout=300s + Should Not Contain ${r_strategize.stdout}${r_strategize.stderr} Traceback + Should Not Contain ${r_strategize.stdout}${r_strategize.stderr} INTERNAL + Log Strategize output: ${r_strategize.stdout} + + # ── 8. Plan tree JSON — extract decision IDs ── + # C1: Use plan tree JSON to extract a real decision_id for explain/correct + ${r_tree_json}= Run CleverAgents Command + ... plan tree ${plan_id} --format json + Should Not Contain ${r_tree_json.stdout}${r_tree_json.stderr} Traceback + Should Not Be Empty ${r_tree_json.stdout} + ... Plan tree --format json produced no output + Log Plan tree JSON: ${r_tree_json.stdout} + # H5: Verify decision_id key exists (specific structural check) + Should Contain ${r_tree_json.stdout} decision_id + ... Plan tree JSON missing 'decision_id' — decisions not recorded + # AC-5: invariant enforcement should be represented structurally in the + # decision tree as an invariant_enforced decision. This is intentionally + # a hard assertion; the test remains tdd_expected_fail until the CLI + # strategize path persists/surfaces invariant enforcement decisions. + Should Contain ${r_tree_json.stdout} invariant_enforced + ... Plan tree JSON does not contain an invariant_enforced decision for the added invariant + # Extract first decision_id via regex + ${decision_ids}= Get Regexp Matches ${r_tree_json.stdout} [0-9A-HJKMNP-TV-Z]{26} + ${has_decisions}= Get Length ${decision_ids} + Should Be True ${has_decisions} > 0 + ... No decision IDs found in plan tree JSON output + ${decision_id}= Set Variable ${decision_ids}[0] + Log Extracted decision_id: ${decision_id} + + # ── 8b. Plan tree plain — structural check ── + ${r_tree}= Run CleverAgents Command + ... plan tree ${plan_id} --format plain + Should Not Contain ${r_tree.stdout}${r_tree.stderr} Traceback + Should Not Be Empty ${r_tree.stdout} + ... Plan tree produced no output for plan ${plan_id} + + # ── 9. Plan explain JSON — C1 + AC-2 + H3 ── + # Use decision_id (not plan_id) and request structured context/reasoning so + # the test can assert that the decision carries a context snapshot and that + # alternatives are represented in the explain output. + ${r_explain}= Run CleverAgents Command + ... plan explain ${decision_id} --format json --show-context --show-reasoning + Log Plan explain stdout: ${r_explain.stdout} + Log Plan explain stderr: ${r_explain.stderr} + Should Not Contain ${r_explain.stdout}${r_explain.stderr} Traceback + Should Not Be Empty ${r_explain.stdout} + ... Plan explain produced no output for decision ${decision_id} + Should Contain ${r_explain.stdout} decision_id + ... Explain JSON missing decision_id field + Should Contain ${r_explain.stdout} alternatives_considered + ... Explain JSON missing alternatives_considered field + Should Contain ${r_explain.stdout} context_snapshot + ... Explain JSON missing context_snapshot field + Should Contain ${r_explain.stdout} hot_context_hash + ... Explain JSON missing hot_context_hash within context_snapshot + ${explain_json}= Extract JSON From Stdout ${r_explain.stdout} + ${explained_id}= Evaluate $explain_json.get('decision_id', '') + Should Be Equal ${explained_id} ${decision_id} + ... plan explain returned a different decision_id than requested + ${hot_context_hash}= Evaluate $explain_json.get('context_snapshot', {}).get('hot_context_hash', '') + Should Not Be Empty ${hot_context_hash} + ... context_snapshot.hot_context_hash is empty — snapshot not recorded + ${has_alternatives_key}= Evaluate 'alternatives_considered' in $explain_json + Should Be True ${has_alternatives_key} + ... Explain JSON does not include alternatives_considered key + + # ── 10. Plan execute — advance to execute phase ── + ${r_execute}= Run CleverAgents Command + ... plan execute ${plan_id} --format plain + ... timeout=300s + Should Not Contain ${r_execute.stdout}${r_execute.stderr} Traceback + Should Not Contain ${r_execute.stdout}${r_execute.stderr} INTERNAL + Log Execute phase output: ${r_execute.stdout} + + # ── 11. Plan correct — C1: use decision_id, not plan_id ── + ${r_correct}= Run CleverAgents Command + ... plan correct ${decision_id} --mode revert + ... --guidance Use SQLAlchemy ORM instead of raw SQL + ... --yes --format plain + ... timeout=300s + Log Plan correct stdout: ${r_correct.stdout} + Log Plan correct stderr: ${r_correct.stderr} + Should Not Contain ${r_correct.stdout}${r_correct.stderr} Traceback + Should Not Contain ${r_correct.stdout}${r_correct.stderr} INTERNAL + Should Not Be Empty ${r_correct.stdout} + ... Plan correct produced no output for decision ${decision_id} + # H4: Use $variable syntax + Should Be True + ... 'revert' in $r_correct.stdout.lower() or 'correct' in $r_correct.stdout.lower() or 'decision' in $r_correct.stdout.lower() + ... Plan correct output missing revert/correct/decision keywords. + + # ── 11b. Verify correction created a superseded decision ── + # P1-C: After plan correct, the targeted decision should be superseded + # and a new correction decision should exist in the tree. + ${r_tree_post}= Run CleverAgents Command + ... plan tree ${plan_id} --format json + Should Not Contain ${r_tree_post.stdout}${r_tree_post.stderr} Traceback + Should Not Be Empty ${r_tree_post.stdout} + # The corrected decision should now be marked superseded in the tree + Should Contain ${r_tree_post.stdout} "superseded": true + ... Plan tree after correction does not show any superseded decisions — correction may not have taken effect + Log Post-correction tree JSON: ${r_tree_post.stdout} + + # ── 12. Plan diff — show changes ── + ${r_diff}= Run CleverAgents Command + ... plan diff ${plan_id} --format plain + Log Plan diff stdout: ${r_diff.stdout} + Log Plan diff stderr: ${r_diff.stderr} + Should Not Contain ${r_diff.stdout}${r_diff.stderr} Traceback + Should Not Contain ${r_diff.stdout}${r_diff.stderr} INTERNAL + + # ── 13. Plan apply ── + ${r_apply}= Run CleverAgents Command + ... plan lifecycle-apply ${plan_id} --format plain + ... timeout=300s + Log Plan apply stdout: ${r_apply.stdout} + Log Plan apply stderr: ${r_apply.stderr} + Should Not Contain ${r_apply.stdout}${r_apply.stderr} Traceback + Should Not Contain ${r_apply.stdout}${r_apply.stderr} INTERNAL + + # ── 14. Final plan status — P1-D: use JSON for structured terminal check ── + ${r_status}= Run CleverAgents Command + ... plan status ${plan_id} --format json + Should Not Be Empty ${r_status.stdout} + Should Not Contain ${r_status.stdout}${r_status.stderr} Traceback + Log Final plan status JSON: ${r_status.stdout} + # Verify plan reached the Apply phase + Should Match Regexp ${r_status.stdout} + ... "phase"\\s*:\\s*"apply" + ... Plan did not reach Apply phase after lifecycle-apply + # Verify processing state is terminal or queued (lifecycle-apply is async) + Should Match Regexp ${r_status.stdout} + ... "processing_state"\\s*:\\s*"(queued|processing|complete|applied)" + ... Plan is in unexpected processing state after lifecycle-apply + Log M3 acceptance E2E test completed successfully