diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cf9fe53e..dea044747 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +- Added E2E acceptance test for M2 (v3.1.0): Actor Compiler + Full LLM + Integration. Robot Framework test suite ``robot/e2e/m2_acceptance.robot`` + exercises actor YAML compilation into functional graphs, skill registry, + tool lifecycle, and plan execution with a custom actor using real LLM API + keys. Test flow: create temp git repo → register custom actor → register + resource and project → create action → run full plan lifecycle (use → + execute strategize → execute → diff → apply) → verify actor compilation + and plan integrity. Uses ``[Tags] E2E``, ``Skip If No LLM Keys``, and + flexible structural assertions with ``expected_rc=None`` for LLM-dependent + commands. (#742) - Fixed `agents session list`, `agents session create`, and other session subcommands raising `AttributeError: 'DynamicContainer' object has no attribute 'db'` after `agents init`. Root cause: `_get_session_service()` diff --git a/robot/e2e/m2_acceptance.robot b/robot/e2e/m2_acceptance.robot new file mode 100644 index 000000000..177c2f044 --- /dev/null +++ b/robot/e2e/m2_acceptance.robot @@ -0,0 +1,128 @@ +*** Settings *** +Documentation E2E acceptance test for M2 (v3.1.0): Actor Compiler + Full LLM Integration. +... +... Exercises actor YAML compilation into functional graphs, skill registry, +... tool lifecycle, and plan execution with a custom actor using real LLM keys. +... Zero mocking — all CLI invocations hit the real CleverAgents binary with +... real provider keys. +Resource common_e2e.resource +Suite Setup E2E Suite Setup +Suite Teardown E2E Suite Teardown + +*** Variables *** +${ACTOR_NAME} local/m2-e2e-actor +${ACTION_NAME} local/m2-e2e-action +${RESOURCE_NAME} local/m2-e2e-repo +${PROJECT_NAME} local/m2-e2e-project + +*** Test Cases *** +M2 Full Actor Compiler And LLM Integration + [Documentation] End-to-end acceptance test for the M2 milestone. + ... + ... Exercises actor YAML compilation, registration via CLI, + ... resource and project setup, action creation referencing a + ... custom actor, and the full plan lifecycle (use, execute + ... strategize, execute, diff, apply) with real LLM API keys. + [Tags] E2E + # ---- Step 1: Create temp git repo with sample project files ---- + ${repo_dir}= Create Temp Git Repo m2-e2e-repo + Create Directory ${repo_dir}${/}src + Create File ${repo_dir}${/}src${/}main.py print("hello world")\n + Run Process git add . cwd=${repo_dir} + Run Process git commit -m Add source files cwd=${repo_dir} + # Detect the default branch name created by git init + ${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${repo_dir} + ${branch}= Strip String ${branch_result.stdout} + Log Detected branch: ${branch} + + # ---- Step 2: Create and register custom actor YAML ---- + ${actor_yaml}= Catenate SEPARATOR=\n + ... name: ${ACTOR_NAME} + ... type: llm + ... description: M2 E2E acceptance actor for compiler and LLM integration + ... version: "1.0" + ... model: gpt-4 + ${actor_yaml_path}= Set Variable ${SUITE_HOME}${/}m2_actor.yaml + Create File ${actor_yaml_path} ${actor_yaml} + ${actor_config}= Catenate SEPARATOR=\n + ... { + ... "provider": "openai", + ... "model": "gpt-4", + ... "options": {"temperature": 0.2} + ... } + ${config_path}= Set Variable ${SUITE_HOME}${/}actor_config.json + Create File ${config_path} ${actor_config} + ${r_actor}= Run CleverAgents Command + ... actor add ${ACTOR_NAME} --config ${config_path} --format plain + Should Not Contain ${r_actor.stdout}${r_actor.stderr} Traceback + Log Actor registration: ${r_actor.stdout} + + # ---- Step 3: Register resource and create project ---- + ${r_resource}= Run CleverAgents Command + ... resource add git-checkout ${RESOURCE_NAME} + ... --path ${repo_dir} --branch ${branch} --format plain + Output Should Contain ${r_resource} ${RESOURCE_NAME} + ${r_project}= Run CleverAgents Command + ... project create ${PROJECT_NAME} + ... --description M2 E2E acceptance project + ... --resource ${RESOURCE_NAME} --format plain + Output Should Contain ${r_project} ${PROJECT_NAME} + + # ---- Step 4: Create action referencing the custom actor ---- + ${action_yaml}= Catenate SEPARATOR=\n + ... name: ${ACTION_NAME} + ... description: M2 acceptance test action for actor compiler and LLM integration + ... definition_of_done: Generate or modify at least one source file + ... strategy_actor: openai/gpt-4 + ... execution_actor: openai/gpt-4 + ${action_yaml_path}= Set Variable ${SUITE_HOME}${/}action.yaml + Create File ${action_yaml_path} ${action_yaml} + ${r_action}= Run CleverAgents Command + ... action create --config ${action_yaml_path} --format plain + Output Should Contain ${r_action} ${ACTION_NAME} + + # ---- Step 5: Plan use ---- + ${r_use}= Run CleverAgents Command + ... plan use ${ACTION_NAME} ${PROJECT_NAME} --format plain + Should Not Be Empty ${r_use.stdout} + ${plan_ids}= Get Regexp Matches ${r_use.stdout} [0-9A-Z]{26} + Should Not Be Empty ${plan_ids} msg=Expected a ULID plan ID in plan use output + ${plan_id}= Set Variable ${plan_ids}[0] + Log Extracted plan_id: ${plan_id} + + # ---- Step 6: Plan execute — strategize phase ---- + ${r_strategize}= Run CleverAgents Command + ... plan execute ${plan_id} --format plain + ... timeout=180s + Should Not Contain ${r_strategize.stdout}${r_strategize.stderr} INTERNAL + Should Not Contain ${r_strategize.stdout}${r_strategize.stderr} Traceback + Log Strategize phase output: ${r_strategize.stdout} + + # ---- Step 7: Plan execute — execute phase ---- + ${r_execute}= Run CleverAgents Command + ... plan execute ${plan_id} --format plain + ... timeout=180s + Should Not Contain ${r_execute.stdout}${r_execute.stderr} INTERNAL + Should Not Contain ${r_execute.stdout}${r_execute.stderr} Traceback + Log Execute phase output: ${r_execute.stdout} + + # ---- Step 8: Plan diff ---- + ${r_diff}= Run CleverAgents Command + ... plan diff ${plan_id} --format plain + Should Not Contain ${r_diff.stdout}${r_diff.stderr} INTERNAL + Should Not Contain ${r_diff.stdout}${r_diff.stderr} Traceback + Log Diff output: ${r_diff.stdout} + + # ---- Step 9: Plan apply ---- + ${r_apply}= Run CleverAgents Command + ... plan lifecycle-apply ${plan_id} --format plain + Should Not Contain ${r_apply.stdout}${r_apply.stderr} INTERNAL + Should Not Contain ${r_apply.stdout}${r_apply.stderr} Traceback + Log Apply output: ${r_apply.stdout} + + # ---- Step 10: Verify actor compilation and plan integrity ---- + ${r_status}= Run CleverAgents Command + ... plan status ${plan_id} --format plain + Should Not Be Empty ${r_status.stdout} + Output Should Contain ${r_status} ${plan_id} + Log Final plan status: ${r_status.stdout}