diff --git a/.gitignore b/.gitignore index 20c57982d..64a609379 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,6 @@ PIPE # Git worktrees for parallel task branches worktrees/ + +# Copy-on-write sandbox backup artifacts +ca-cow-backup-*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb09c386..faae94413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -854,6 +854,16 @@ attribute 'db'` after `agents init`. Root cause: `_get_session_service()` session subcommands. Includes Behave BDD regression scenarios, Robot Framework integration smoke tests, and structlog isolation for parallel test execution. (#554, #570, #680) +- Added E2E test for Workflow Example 9 — session-driven interactive + exploration. `robot/e2e/wf09_session.robot` exercises the full session + lifecycle: creates a temp git repo with a sample multi-module Python project, + registers it as a resource/project, creates a session with an actor via + `session create`, sends conversational queries via `session tell` (including + an action-creation request per the spec), verifies accumulated history via + `session show`, exports the conversation to JSON via `session export`, and + validates the exported JSON structure (session_id, messages array). + Uses `Force Tags E2E`, `common_e2e.resource`, and skips gracefully + when LLM API keys are absent via `Skip If No LLM Keys`. (#755) - Added Robot Framework E2E acceptance test for M1 (v3.0.0) milestone. Tests the complete plan lifecycle (action create → resource add → project create → plan use → plan execute strategize → plan execute → plan diff → diff --git a/robot/e2e/wf09_session.robot b/robot/e2e/wf09_session.robot new file mode 100644 index 000000000..f2ab8be19 --- /dev/null +++ b/robot/e2e/wf09_session.robot @@ -0,0 +1,365 @@ +*** Settings *** +Documentation E2E Workflow Example 9 — Session-Driven Interactive Exploration. +... +... A new developer uses sessions to interactively explore an +... unfamiliar codebase: creates a session, sends conversational +... queries via ``session tell``, asks the AI to create an action +... from the conversation, reviews accumulated history via +... ``session show``, and exports the conversation to JSON via +... ``session export``. +... +... **Zero mocking** — exercises the real CleverAgents CLI with +... real LLM API keys. LLM-dependent steps use +... ``Skip If No LLM Keys`` for graceful degradation in +... environments without API keys. +... +... **Note:** ``session tell`` is currently stubbed at the CLI +... layer (M3 milestone) — the assistant echoes an +... acknowledgement rather than invoking a real LLM. AC #5 +... (action creation) therefore always takes the WARN path. +... Assertions validate the echo response pattern until real +... LLM integration lands. +Library Collections +Resource common_e2e.resource +Suite Setup WF09 Suite Setup +Suite Teardown E2E Suite Teardown +Force Tags E2E + +*** Variables *** +${AUTH_PY} SEPARATOR=\n +... """Simple authentication module.""" +... +... +... def authenticate(username: str, password: str) -> bool: +... """Authenticate a user with username and password.""" +... # Placeholder implementation +... return username == "test-user" and password == "test-password" +... +... +... def create_token(user_id: str) -> str: +... """Create a JWT token for authenticated user.""" +... return f"token-{user_id}" +... +... +... def verify_token(token: str) -> bool: +... """Verify a JWT token is valid.""" +... return token.startswith("token-") + +${ROUTES_PY} SEPARATOR=\n +... """HTTP route definitions.""" +... +... from auth import authenticate, verify_token +... +... +... ROUTES = { +... "/login": "handle_login", +... "/dashboard": "handle_dashboard", +... "/api/users": "handle_users", +... } +... +... +... def handle_login(request): +... """Handle login requests.""" +... return {"status": "ok"} +... +... +... def handle_dashboard(request): +... """Handle dashboard requests.""" +... if not verify_token(request.get("token", "")): +... return {"status": "unauthorized"} +... return {"status": "ok", "data": "dashboard"} + +${MODELS_PY} SEPARATOR=\n +... """Data models for the application.""" +... +... +... class User: +... """User model with basic fields.""" +... +... def __init__(self, user_id: str, username: str, email: str): +... self.user_id = user_id +... self.username = username +... self.email = email +... +... def to_dict(self): +... return {"id": self.user_id, "username": self.username, "email": self.email} +... +... +... class Session: +... """Session model for tracking user sessions.""" +... +... def __init__(self, session_id: str, user: User): +... self.session_id = session_id +... self.user = user + +${README_CONTENT} SEPARATOR=\n +... # Sample Web Application +... +... A simple web application with authentication, routing, and data models. +... +... ## Modules +... +... - **auth** — authentication and token management +... - **routes** — HTTP route definitions and handlers +... - **models** — data models (User, Session) +... +... ## Getting Started +... +... Install dependencies and run the application. + +*** Keywords *** +WF09 Suite Setup + [Documentation] E2E Suite Setup plus unique suffix generation and actor selection. + E2E Suite Setup + # Initialise the database so session/config commands work in all tests. + ${init}= Run CleverAgents Command init --force --yes + Should Be Equal As Integers ${init.rc} 0 + Should Not Contain ${init.stdout}${init.stderr} Traceback + Should Not Contain ${init.stdout}${init.stderr} INTERNAL + # Generate a unique suffix for resource/project names to avoid UNIQUE + # constraint collisions on repeated or parallel CI runs. + ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] + Set Suite Variable ${RUN_SUFFIX} ${suffix} + # Pick an actor that matches the available API key. + # Prefer Anthropic for session-based exploration: Claude models tend to + # produce richer conversational replies that improve session-history + # assertions. WF04/WF05 prefer OpenAI to balance provider coverage. + # Unlike WF04/WF05 which check both providers, WF09 only checks + # Anthropic and falls through to OpenAI unconditionally — this is + # intentional: ``session tell`` is currently stubbed (M3), so the + # actor choice affects only the echo prefix, not real LLM behaviour. + ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) + IF ${has_anthropic} + ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 + ELSE + ${actor}= Set Variable openai/gpt-4o + END + Set Suite Variable ${LLM_ACTOR} ${actor} + +Create Sample Project Fixture + [Documentation] Create a temp git repo with a multi-module Python project, + ... register it as a resource and create a project. Returns + ... the project name. + ${repo}= Create Temp Git Repo wf09-explore-${RUN_SUFFIX} + Create File ${repo}${/}src${/}auth.py ${AUTH_PY} + Create File ${repo}${/}src${/}routes.py ${ROUTES_PY} + Create File ${repo}${/}src${/}models.py ${MODELS_PY} + Create File ${repo}${/}README.md ${README_CONTENT} + ${git_add}= Run Process git add . cwd=${repo} timeout=30s on_timeout=kill + Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr} + ${git_commit}= Run Process git commit -m Add sample project files cwd=${repo} timeout=30s on_timeout=kill + Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr} + # Register repo as a resource + ${res_name}= Set Variable wf09-res-${RUN_SUFFIX} + ${res_result}= Run CleverAgents Command + ... resource add git-checkout ${res_name} --path ${repo} + Should Be Equal As Integers ${res_result.rc} 0 + Should Not Contain ${res_result.stdout}${res_result.stderr} Traceback + Should Not Contain ${res_result.stdout}${res_result.stderr} INTERNAL + Output Should Contain ${res_result} ${res_name} + # Create a project linked to the resource + ${proj_name}= Set Variable wf09-proj-${RUN_SUFFIX} + ${proj_result}= Run CleverAgents Command + ... project create --resource ${res_name} ${proj_name} + Should Be Equal As Integers ${proj_result.rc} 0 + Should Not Contain ${proj_result.stdout}${proj_result.stderr} Traceback + Should Not Contain ${proj_result.stdout}${proj_result.stderr} INTERNAL + Output Should Contain ${proj_result} ${proj_name} + RETURN ${proj_name} + +Create Session With Actor + [Documentation] Create a session with an LLM actor via ``session create``. + ... Returns the session ID. + ... + ... Note: ``session create`` does not accept a ``--project`` + ... parameter. The session is actor-scoped; the project + ... fixture is registered separately so the LLM can discover + ... it via resource/project queries during the conversation. + ${create_result}= Run CleverAgents Command + ... session create --actor ${LLM_ACTOR} --format json + Should Not Be Empty ${create_result.stdout} + ... msg=session create produced no output + Should Not Contain ${create_result.stdout}${create_result.stderr} Traceback + Should Not Contain ${create_result.stdout}${create_result.stderr} INTERNAL + Output Should Contain ${create_result} session_id + ${session_id}= Safe Parse Json Field ${create_result.stdout} session_id + Should Not Be Empty ${session_id} + ... msg=Could not extract session_id from session create output + RETURN ${session_id} + +Verify Session Tell Response + [Documentation] Send a message to the session via ``session tell`` and verify + ... that it produces a non-empty response with content relevance. + ... + ... ``session tell`` is currently stubbed (M3): the assistant + ... echoes ``Acknowledged: ``. The content check + ... validates this echo pattern. Update when real LLM + ... integration lands. + [Arguments] ${session_id} ${message} ${timeout}=120s + ${tell_result}= Run CleverAgents Command + ... session tell --session ${session_id} ${message} + ... timeout=${timeout} + Should Not Be Empty ${tell_result.stdout} + ... msg=session tell produced no response for: ${message} + Should Not Contain ${tell_result.stdout}${tell_result.stderr} Traceback + Should Not Contain ${tell_result.stdout}${tell_result.stderr} INTERNAL + # Content relevance: stubbed session tell echoes "Acknowledged: ". + Output Should Contain ${tell_result} Acknowledged + RETURN ${tell_result} + +Verify Session History + [Documentation] Verify that ``session show`` returns output containing + ... the session ID and evidence of accumulated messages. + ... Parses JSON output and asserts ``message_count >= 6`` + ... (3 user turns + 3 assistant responses) and that + ... ``recent_messages`` is populated. + [Arguments] ${session_id} + ${show_result}= Run CleverAgents Command + ... session show ${session_id} --format json + Should Not Contain ${show_result.stdout}${show_result.stderr} Traceback + Should Not Contain ${show_result.stdout}${show_result.stderr} INTERNAL + Output Should Contain ${show_result} ${session_id} + # Parse JSON and verify message accumulation (AC #4). + # ``session show --format json`` exposes ``message_count`` (total) and + # ``recent_messages`` (last 5). Use Extract JSON From Stdout (raw_decode) + # to handle potential non-JSON preamble from the CLI. + ${show_json}= Extract JSON From Stdout ${show_result.stdout} + ${msg_count}= Evaluate int($show_json.get('message_count', 0)) + Should Be True ${msg_count} >= 6 + ... msg=Expected message_count >= 6 (3 user turns + 3 assistant replies), found ${msg_count} + # Also verify recent_messages is populated (proves messages are persisted). + # After 3 user + 3 assistant turns the recent window (last 5) should + # contain at least 3 entries. + ${recent}= Evaluate $show_json.get('recent_messages', []) + ${recent_count}= Evaluate len($recent) if isinstance($recent, list) else 0 + Should Be True ${recent_count} >= 3 + ... msg=Expected at least 3 recent_messages after 3 turns, found ${recent_count} + RETURN ${show_result} + +Verify Session Export + [Documentation] Export the session to JSON and validate the exported file + ... has the expected structure (session_id matching the input, + ... non-empty messages list with valid message objects). + [Arguments] ${session_id} + ${export_path}= Set Variable ${SUITE_HOME}${/}wf09_export.json + # Remove any stale export from a previous partial run to avoid false passes. + Remove File ${export_path} + # Spec canonical form: flag before positional arg. + ${export_result}= Run CleverAgents Command + ... session export --output ${export_path} ${session_id} + Should Not Contain ${export_result.stdout}${export_result.stderr} Traceback + Should Not Contain ${export_result.stdout}${export_result.stderr} INTERNAL + File Should Exist ${export_path} msg=session export did not create file + ${export_content}= Get File ${export_path} + Should Not Be Empty ${export_content} msg=Export file is empty + TRY + ${export_json}= Evaluate json.loads($export_content) json + EXCEPT AS ${err} + Fail session export produced invalid JSON: ${err} + END + Dictionary Should Contain Key ${export_json} session_id + Dictionary Should Contain Key ${export_json} messages + # Verify exported session_id matches the actual session. + ${exported_sid}= Evaluate str($export_json.get('session_id', '')) + Should Be Equal As Strings ${exported_sid} ${session_id} + ... msg=Exported session_id does not match the actual session + # Verify messages list is non-empty (not just structurally present). + ${msg_list}= Evaluate $export_json.get('messages', []) + Should Not Be Empty ${msg_list} msg=Exported messages list should not be empty + # Spot-check first message has the expected structure (role field). + ${first_msg}= Evaluate $msg_list[0] if $msg_list else {} + Dictionary Should Contain Key ${first_msg} role + ... msg=Each exported message should have a 'role' field + +WF09 Test Teardown + [Documentation] Log diagnostic context on failure for debugging. + ... Captures session state so CI failures in this LLM-dependent + ... test have actionable data. + ${session_id}= Get Variable Value ${WF09_SESSION_ID} ${EMPTY} + IF '${session_id}' != '' + ${status} ${result}= Run Keyword And Ignore Error + ... Run CleverAgents Command session show ${session_id} --format json expected_rc=None timeout=30s + IF '${status}' == 'PASS' + Log Teardown session show: ${result.stdout} WARN + END + END + +*** Test Cases *** +Workflow 09 — Session-Driven Interactive Exploration + [Documentation] End-to-end workflow: create a session, explore a codebase + ... interactively via session tell, request action creation, + ... review history, and export. + [Timeout] 20 minutes + [Teardown] WF09 Test Teardown + Skip If No LLM Keys + # Initialise test variable for teardown access. + Set Test Variable ${WF09_SESSION_ID} ${EMPTY} + + # ------------------------------------------------------------------ + # Step 1: Create a temp git repo with a sample project fixture + # ------------------------------------------------------------------ + # proj_name exists for LLM discovery; session create is actor-scoped, + # not project-scoped, so the variable is not referenced again directly. + ${proj_name}= Create Sample Project Fixture + + # ------------------------------------------------------------------ + # Step 2: Create a session with an actor + # ------------------------------------------------------------------ + ${session_id}= Create Session With Actor + Set Test Variable ${WF09_SESSION_ID} ${session_id} + + # ------------------------------------------------------------------ + # Step 3: First conversational query — ask about modules + # ``session tell`` is currently stubbed (M3) — echoes an + # acknowledgement rather than calling a real LLM. + # ------------------------------------------------------------------ + ${tell1_result}= Verify Session Tell Response ${session_id} + ... What are the main modules in this project? + + # ------------------------------------------------------------------ + # Step 4: Second conversational query — ask about auth module + # ------------------------------------------------------------------ + ${tell2_result}= Verify Session Tell Response ${session_id} + ... Can you explain how the auth module works? + + # ------------------------------------------------------------------ + # Step 5: Third conversational query — request action creation + # (per spec Example 9, Step 2, third session tell interaction) + # Use 300s timeout: action creation involves LLM tool-calling which + # is significantly slower than simple conversational replies. This + # timeout is forward-compatible for when real LLM integration lands; + # while ``session tell`` remains stubbed the command returns + # instantly so the timeout has no practical effect. + # ------------------------------------------------------------------ + ${tell3_result}= Verify Session Tell Response ${session_id} + ... Create an action called local/wf09-refresh-${RUN_SUFFIX} that automates dependency lock refresh + ... timeout=300s + + # ------------------------------------------------------------------ + # Step 5b: Verify action creation (AC #5) + # The LLM was asked to create an action; verify it exists. Since LLM + # behavior is non-deterministic, use a soft check: warn if the action + # was not created rather than hard-failing. + # ------------------------------------------------------------------ + ${action_check}= Run CleverAgents Command + ... action show local/wf09-refresh-${RUN_SUFFIX} --format json + ... expected_rc=None + IF ${action_check.rc} != 0 + Log AC #5 note: LLM did not create action local/wf09-refresh-${RUN_SUFFIX} in this run WARN + ELSE + Should Not Contain ${action_check.stdout}${action_check.stderr} Traceback + Should Not Contain ${action_check.stdout}${action_check.stderr} INTERNAL + Output Should Contain ${action_check} wf09-refresh + END + + # ------------------------------------------------------------------ + # Step 6: Verify session has accumulated history + # ------------------------------------------------------------------ + ${show_result}= Verify Session History ${session_id} + + # ------------------------------------------------------------------ + # Step 7: Export session to JSON and validate structure + # ------------------------------------------------------------------ + Verify Session Export ${session_id} + + Log Workflow 09 completed: session ${session_id}