diff --git a/robot/e2e/common_e2e.resource b/robot/e2e/common_e2e.resource index 9ac3cf851..0fbb137d8 100644 --- a/robot/e2e/common_e2e.resource +++ b/robot/e2e/common_e2e.resource @@ -297,6 +297,24 @@ Create Synthetic Codebase ${large_content}= Evaluate "# auto-generated large file\\n" + ("x = 1\\n" * 250) Create File ${base_dir}${/}large_file.py ${large_content} +Create Temp Directory + [Documentation] Create a temporary directory for testing. + ... + ... Uses Python's tempfile module to create an isolated + ... temp directory with the given name prefix. Ideal for + ... project creation tests that need their own workspace. + [Arguments] ${name}=${EMPTY} + ${temp_dir}= Evaluate __import__("tempfile").mkdtemp(prefix="${name}-") + RETURN ${temp_dir} + +Remove Temp Directory + [Documentation] Remove a temporary directory created for testing. + ... + ... Safely removes the directory and all contents recursively, + ... ignoring errors if the directory does not exist. + [Arguments] ${dir} + Run Keyword And Ignore Error Remove Directory ${dir} recursive=True + Create Temp Git Repo [Documentation] Create a temporary git repository for E2E testing. ... diff --git a/robot/e2e/test_correction_workflow.robot b/robot/e2e/test_correction_workflow.robot new file mode 100644 index 000000000..dc6bd0050 --- /dev/null +++ b/robot/e2e/test_correction_workflow.robot @@ -0,0 +1,115 @@ +*** Settings *** +Documentation E2E workflow test for plan correction workflows. +... +... Tests plan correction via CLI: +... 1. Revert mode - previews revert correction using --dry-run +... 2. Append mode - previews append correction using --dry-run +... 3. State transition - applies a correction and verifies plan +... responds without error, confirming the correction workflow +... alters plan state. +... +... Requires real LLM API keys (ANTHROPIC_API_KEY or OPENAI_API_KEY). +... Tests are skipped automatically when no keys are available. +Resource common_e2e.resource +Suite Setup Correction Suite Setup +Suite Teardown E2E Suite Teardown +Force Tags E2E + +*** Keywords *** +Correction Suite Setup + [Documentation] E2E Suite Setup plus action registration. + E2E Suite Setup + ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] + Set Suite Variable ${RUN_SUFFIX} ${suffix} + ${actor}= Resolve LLM Actor + ${yaml}= Catenate SEPARATOR=\n + ... name: local/code-review + ... description: "Perform a code review on project sources" + ... strategy_actor: ${actor} + ... execution_actor: ${actor} + ... definition_of_done: "Code review completed." + ... reusable: true + ... read_only: false + ... state: available + ${yaml_path}= Set Variable ${SUITE_HOME}${/}code-review-action.yaml + Create File ${yaml_path} ${yaml} + ${reg}= Run CleverAgents Command action create --config ${yaml_path} expected_rc=None + IF ${reg.rc} != 0 + Log Could not register local/code-review action: ${reg.stderr} WARN + END + +Setup Correction Plan Resources + [Documentation] Create git repo resource and project for correction tests. + ... Resources are created inside SUITE_HOME and cleaned up + ... by E2E Suite Teardown automatically. + [Arguments] ${prefix} + ${repo}= Create Temp Git Repo ${prefix}-repo-${RUN_SUFFIX} + ${res}= Set Variable ${prefix}-res-${RUN_SUFFIX} + ${add}= Run CleverAgents Command resource add git-checkout ${res} --path ${repo} + Should Be Equal As Integers ${add.rc} 0 + ${proj}= Set Variable ${prefix}-proj-${RUN_SUFFIX} + ${create}= Run CleverAgents Command project create --resource ${res} ${proj} + Should Be Equal As Integers ${create.rc} 0 + RETURN ${proj} + +*** Test Cases *** + +Correction Revert Mode Workflow + [Documentation] Test plan correction in revert mode (dry-run preview). + ... + ... Creates a project, generates a plan via action, then previews + ... a revert-mode correction using --dry-run. Verifies the correction + ... command accepts the plan and returns output referencing it. + [Tags] correction-revert + [Timeout] 30 minutes + [Teardown] Run Keyword And Ignore Error Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj}= Setup Correction Plan Resources cr-revert + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj} --automation-profile ci --format json expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + ${correct}= Run CleverAgents Command plan correct --mode revert -g "Use a different framework" --dry-run --yes ${plan_id} --format json expected_rc=None timeout=180s + Output Should Contain ${correct} ${plan_id} + +Correction Append Mode Workflow + [Documentation] Test plan correction in append mode (dry-run preview). + ... + ... Creates a project, generates a plan via action, then previews + ... an append-mode correction using --dry-run. Verifies the correction + ... command accepts the plan and returns output referencing it. + [Tags] correction-append + [Timeout] 30 minutes + [Teardown] Run Keyword And Ignore Error Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj}= Setup Correction Plan Resources cr-append + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj} --automation-profile ci --format json expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + ${correct}= Run CleverAgents Command plan correct --mode append -g "Add error handling layer" --dry-run --yes ${plan_id} --format json expected_rc=None timeout=180s + Output Should Contain ${correct} ${plan_id} + +Correction State Transition Validation + [Documentation] Test correction state transitions via plan status. + ... + ... Creates a project, generates a plan, records its initial status, + ... applies a revert correction with --yes (no dry-run), and verifies + ... the plan is still queryable after the correction, confirming the + ... correction workflow alters plan state without corrupting it. + [Tags] correction-state + [Timeout] 30 minutes + [Teardown] Run Keyword And Ignore Error Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj}= Setup Correction Plan Resources cr-state + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj} --automation-profile ci --format json expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + ${status_before}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${status_before.rc} 0 msg=plan status failed: ${status_before.stderr} + ${correct}= Run CleverAgents Command plan correct --mode revert -g "State transition validation" --yes ${plan_id} --format json expected_rc=None timeout=180s + Output Should Contain ${correct} ${plan_id} + ${status_after}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${status_after.rc} 0 msg=plan status failed after correction: ${status_after.stderr} + Output Should Contain ${status_after} ${plan_id} diff --git a/robot/e2e/test_project_plan_workflow.robot b/robot/e2e/test_project_plan_workflow.robot new file mode 100644 index 000000000..499525b80 --- /dev/null +++ b/robot/e2e/test_project_plan_workflow.robot @@ -0,0 +1,104 @@ +*** Settings *** +Documentation E2E workflow test for project creation, actor setup, and plan execution. +... +... Exercises the complete workflow: +... 1. Create a project linked to a git resource and verify it appears in list +... 2. List actors available in the workspace +... 3. Create a plan via action, execute it, and verify plan status +... +... Requires real LLM API keys (ANTHROPIC_API_KEY or OPENAI_API_KEY) for +... the plan execution test. Non-LLM tests run unconditionally. +Resource common_e2e.resource +Suite Setup Plan Workflow Suite Setup +Suite Teardown E2E Suite Teardown +Force Tags E2E + +*** Keywords *** +Plan Workflow Suite Setup + [Documentation] E2E Suite Setup plus action registration. + E2E Suite Setup + ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] + Set Suite Variable ${RUN_SUFFIX} ${suffix} + ${actor}= Resolve LLM Actor + ${yaml}= Catenate SEPARATOR=\n + ... name: local/code-review + ... description: "Perform a code review on project sources" + ... strategy_actor: ${actor} + ... execution_actor: ${actor} + ... definition_of_done: "Code review completed." + ... reusable: true + ... read_only: false + ... state: available + ${yaml_path}= Set Variable ${SUITE_HOME}${/}code-review-action.yaml + Create File ${yaml_path} ${yaml} + ${reg}= Run CleverAgents Command action create --config ${yaml_path} expected_rc=None + IF ${reg.rc} != 0 + Log Could not register local/code-review action: ${reg.stderr} WARN + END + +Setup Project Resources + [Documentation] Create git repo resource and project for plan workflow tests. + ... Resources are created inside SUITE_HOME and cleaned up + ... by E2E Suite Teardown automatically. + [Arguments] ${prefix} + ${repo}= Create Temp Git Repo ${prefix}-repo-${RUN_SUFFIX} + ${res}= Set Variable ${prefix}-res-${RUN_SUFFIX} + ${add}= Run CleverAgents Command resource add git-checkout ${res} --path ${repo} + Should Be Equal As Integers ${add.rc} 0 + ${proj}= Set Variable ${prefix}-proj-${RUN_SUFFIX} + ${create}= Run CleverAgents Command project create --resource ${res} ${proj} + Should Be Equal As Integers ${create.rc} 0 + RETURN ${proj} + +*** Test Cases *** + +Project Creation Workflow + [Documentation] Test complete project creation workflow. + ... + ... Creates a project linked to a git-checkout resource, verifies the + ... project is created successfully, appears in project list, and + ... project show returns the expected project name. + [Tags] project-creation + [Timeout] 10 minutes + ${proj}= Setup Project Resources proj-create + ${list_result}= Run CleverAgents Command project list --format json + Should Be Equal As Integers ${list_result.rc} 0 + Output Should Contain ${list_result} ${proj} + ${show_result}= Run CleverAgents Command project show --name ${proj} --format json expected_rc=None + Should Be Equal As Integers ${show_result.rc} 0 + Output Should Contain ${show_result} ${proj} + +Actor Setup Workflow + [Documentation] Test actor listing in the workspace. + ... + ... Verifies that actor list returns successfully after workspace + ... initialization, confirming the actor subsystem is reachable. + [Tags] actor-setup + [Timeout] 10 minutes + ${list_result}= Run CleverAgents Command actor list --format json expected_rc=None + Should Be Equal As Integers ${list_result.rc} 0 + Should Not Be Empty ${list_result.stdout} msg=actor list returned empty output + +Plan Execution Workflow + [Documentation] Test complete plan execution workflow. + ... + ... Creates a project linked to a git resource, creates a plan via + ... the local/code-review action, executes it, and verifies the plan + ... status is retrievable after execution. + [Tags] plan-execution + [Timeout] 30 minutes + [Teardown] Run Keyword And Ignore Error Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj}= Setup Project Resources plan-exec + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj} --automation-profile ci --format json expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s + IF ${execute.rc} != 0 + Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} + END + Output Should Contain ${execute} ${plan_id} + ${status}= Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${status.rc} 0 msg=plan status failed: ${status.stderr} + Output Should Contain ${status} ${plan_id} diff --git a/robot/e2e/test_subplan_workflow.robot b/robot/e2e/test_subplan_workflow.robot new file mode 100644 index 000000000..c99e86914 --- /dev/null +++ b/robot/e2e/test_subplan_workflow.robot @@ -0,0 +1,144 @@ +*** Settings *** +Documentation E2E workflow test for subplan spawning and merge infrastructure. +... +... Exercises subplan and merge workflows via plan tree inspection: +... 1. Subplan Spawning - verifies plan tree contains child_plans field +... after plan execution, confirming subplan spawning infrastructure +... 2. Three-Way Merge - verifies plan apply runs after execution, +... exercising the plan finalisation / merge path +... 3. Merge Result Validation - verifies plan tree output conforms to +... the spec-required envelope (command, data, child_plans, decision_ids) +... +... Requires real LLM API keys (ANTHROPIC_API_KEY or OPENAI_API_KEY). +... Tests are skipped automatically when no keys are available. +Resource common_e2e.resource +Suite Setup Subplan Suite Setup +Suite Teardown E2E Suite Teardown +Force Tags E2E + +*** Keywords *** +Subplan Suite Setup + [Documentation] E2E Suite Setup plus action registration. + E2E Suite Setup + ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] + Set Suite Variable ${RUN_SUFFIX} ${suffix} + ${actor}= Resolve LLM Actor + ${yaml}= Catenate SEPARATOR=\n + ... name: local/code-review + ... description: "Perform a code review on project sources" + ... strategy_actor: ${actor} + ... execution_actor: ${actor} + ... definition_of_done: "Code review completed." + ... reusable: true + ... read_only: false + ... state: available + ${yaml_path}= Set Variable ${SUITE_HOME}${/}code-review-action.yaml + Create File ${yaml_path} ${yaml} + ${reg}= Run CleverAgents Command action create --config ${yaml_path} expected_rc=None + IF ${reg.rc} != 0 + Log Could not register local/code-review action: ${reg.stderr} WARN + END + +Setup Subplan Resources + [Documentation] Create git repo resource and project for subplan tests. + ... Resources are created inside SUITE_HOME and cleaned up + ... by E2E Suite Teardown automatically. + [Arguments] ${prefix} + ${repo}= Create Temp Git Repo ${prefix}-repo-${RUN_SUFFIX} + ${res}= Set Variable ${prefix}-res-${RUN_SUFFIX} + ${add}= Run CleverAgents Command resource add git-checkout ${res} --path ${repo} + Should Be Equal As Integers ${add.rc} 0 + ${proj}= Set Variable ${prefix}-proj-${RUN_SUFFIX} + ${create}= Run CleverAgents Command project create --resource ${res} ${proj} + Should Be Equal As Integers ${create.rc} 0 + RETURN ${proj} + +*** Test Cases *** + +Subplan Spawning Workflow + [Documentation] Test subplan spawning infrastructure via plan tree. + ... + ... Creates a project, generates a plan via action, executes it, then + ... inspects plan tree to confirm the spec-required child_plans field + ... is present. This verifies the subplan spawning infrastructure is + ... functional: the field is always present whether or not the LLM + ... chose to decompose the task into child subplans. + [Tags] subplan-spawn + [Timeout] 30 minutes + [Teardown] Run Keyword And Ignore Error Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj}= Setup Subplan Resources sp-spawn + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj} --automation-profile full-auto --format json expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s + IF ${execute.rc} != 0 + Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} + END + ${tree}= Run CleverAgents Command plan tree ${plan_id} --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${tree.rc} 0 msg=plan tree failed: ${tree.stderr} + Should Not Be Empty ${tree.stdout} msg=plan tree returned empty output + ${has_child_plans}= Evaluate '"child_plans"' in $tree.stdout + Should Be True ${has_child_plans} Plan tree output should contain spec-required child_plans field + +Three-Way Merge Workflow + [Documentation] Test plan apply path which exercises merge logic. + ... + ... Creates a project, generates a plan, executes it, then attempts + ... plan apply. The apply step exercises the plan finalisation and + ... merge path, verifying that completed execution can transition to + ... the apply phase without error. + [Tags] three-way-merge + [Timeout] 30 minutes + [Teardown] Run Keyword And Ignore Error Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj}= Setup Subplan Resources sp-merge + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj} --automation-profile full-auto --format json expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s + IF ${execute.rc} != 0 + Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} + END + ${apply}= Run CleverAgents Command plan apply --yes ${plan_id} --format json expected_rc=None timeout=180s + IF ${apply.rc} == 0 + Output Should Contain ${apply} ${plan_id} + ELSE + Fail plan apply failed (rc=${apply.rc}) stdout=${apply.stdout} stderr=${apply.stderr} + END + +Merge Result Validation + [Documentation] Test plan tree output conforms to spec-required envelope. + ... + ... Creates a project, generates a plan, executes it, then verifies + ... plan tree returns the spec-required envelope fields: command, + ... data, plan_id, child_plans, and decision_ids. These fields + ... provide the structural evidence that merge result data is + ... accessible via the CLI. + [Tags] merge-validation + [Timeout] 30 minutes + [Teardown] Run Keyword And Ignore Error Run CleverAgents Command config set core.automation-profile manual expected_rc=None + Skip If No LLM Keys + ${proj}= Setup Subplan Resources sp-result + ${plan_use}= Run CleverAgents Command plan use local/code-review ${proj} --automation-profile full-auto --format json expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_use.rc} 0 msg=plan use failed (rc=${plan_use.rc}): ${plan_use.stderr} + ${plan_id}= Safe Parse Json Field ${plan_use.stdout} plan_id + Should Not Be Empty ${plan_id} Could not parse plan_id from plan use output + ${execute}= Run CleverAgents Command plan execute ${plan_id} --format json expected_rc=None timeout=180s + IF ${execute.rc} != 0 + Fail plan execute failed (rc=${execute.rc}) stdout=${execute.stdout} stderr=${execute.stderr} + END + ${tree}= Run CleverAgents Command plan tree ${plan_id} --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${tree.rc} 0 msg=plan tree failed: ${tree.stderr} + ${has_command}= Evaluate '"command"' in $tree.stdout + Should Be True ${has_command} Plan tree output should contain spec-required "command" envelope key + ${has_data}= Evaluate '"data"' in $tree.stdout + Should Be True ${has_data} Plan tree output should contain spec-required "data" envelope key + ${has_plan_id}= Evaluate '"plan_id"' in $tree.stdout + Should Be True ${has_plan_id} Plan tree output should contain "plan_id" in envelope data + ${has_child_plans}= Evaluate '"child_plans"' in $tree.stdout + Should Be True ${has_child_plans} Plan tree output should contain spec-required "child_plans" field + ${has_decision_ids}= Evaluate '"decision_ids"' in $tree.stdout + Should Be True ${has_decision_ids} Plan tree output should contain "decision_ids" mapping after execution