|
|
|
@@ -0,0 +1,365 @@
|
|
|
|
|
*** Settings ***
|
|
|
|
|
Documentation E2E Workflow Example 9 — Session-Driven Interactive Exploration.
|
|
|
|
|
...
|
|
|
|
|
... A new developer uses sessions to interactively explore an
|
|
|
|
|
... unfamiliar codebase: creates a session, sends conversational
|
|
|
|
|
... queries via ``session tell``, asks the AI to create an action
|
|
|
|
|
... from the conversation, reviews accumulated history via
|
|
|
|
|
... ``session show``, and exports the conversation to JSON via
|
|
|
|
|
... ``session export``.
|
|
|
|
|
...
|
|
|
|
|
... **Zero mocking** — exercises the real CleverAgents CLI with
|
|
|
|
|
... real LLM API keys. LLM-dependent steps use
|
|
|
|
|
... ``Skip If No LLM Keys`` for graceful degradation in
|
|
|
|
|
... environments without API keys.
|
|
|
|
|
...
|
|
|
|
|
... **Note:** ``session tell`` is currently stubbed at the CLI
|
|
|
|
|
... layer (M3 milestone) — the assistant echoes an
|
|
|
|
|
... acknowledgement rather than invoking a real LLM. AC #5
|
|
|
|
|
... (action creation) therefore always takes the WARN path.
|
|
|
|
|
... Assertions validate the echo response pattern until real
|
|
|
|
|
... LLM integration lands.
|
|
|
|
|
Library Collections
|
|
|
|
|
Resource common_e2e.resource
|
|
|
|
|
Suite Setup WF09 Suite Setup
|
|
|
|
|
Suite Teardown E2E Suite Teardown
|
|
|
|
|
Force Tags E2E
|
|
|
|
|
|
|
|
|
|
*** Variables ***
|
|
|
|
|
${AUTH_PY} SEPARATOR=\n
|
|
|
|
|
... """Simple authentication module."""
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... def authenticate(username: str, password: str) -> bool:
|
|
|
|
|
... """Authenticate a user with username and password."""
|
|
|
|
|
... # Placeholder implementation
|
|
|
|
|
... return username == "test-user" and password == "test-password"
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... def create_token(user_id: str) -> str:
|
|
|
|
|
... """Create a JWT token for authenticated user."""
|
|
|
|
|
... return f"token-{user_id}"
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... def verify_token(token: str) -> bool:
|
|
|
|
|
... """Verify a JWT token is valid."""
|
|
|
|
|
... return token.startswith("token-")
|
|
|
|
|
|
|
|
|
|
${ROUTES_PY} SEPARATOR=\n
|
|
|
|
|
... """HTTP route definitions."""
|
|
|
|
|
...
|
|
|
|
|
... from auth import authenticate, verify_token
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... ROUTES = {
|
|
|
|
|
... "/login": "handle_login",
|
|
|
|
|
... "/dashboard": "handle_dashboard",
|
|
|
|
|
... "/api/users": "handle_users",
|
|
|
|
|
... }
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... def handle_login(request):
|
|
|
|
|
... """Handle login requests."""
|
|
|
|
|
... return {"status": "ok"}
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... def handle_dashboard(request):
|
|
|
|
|
... """Handle dashboard requests."""
|
|
|
|
|
... if not verify_token(request.get("token", "")):
|
|
|
|
|
... return {"status": "unauthorized"}
|
|
|
|
|
... return {"status": "ok", "data": "dashboard"}
|
|
|
|
|
|
|
|
|
|
${MODELS_PY} SEPARATOR=\n
|
|
|
|
|
... """Data models for the application."""
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... class User:
|
|
|
|
|
... """User model with basic fields."""
|
|
|
|
|
...
|
|
|
|
|
... def __init__(self, user_id: str, username: str, email: str):
|
|
|
|
|
... self.user_id = user_id
|
|
|
|
|
... self.username = username
|
|
|
|
|
... self.email = email
|
|
|
|
|
...
|
|
|
|
|
... def to_dict(self):
|
|
|
|
|
... return {"id": self.user_id, "username": self.username, "email": self.email}
|
|
|
|
|
...
|
|
|
|
|
...
|
|
|
|
|
... class Session:
|
|
|
|
|
... """Session model for tracking user sessions."""
|
|
|
|
|
...
|
|
|
|
|
... def __init__(self, session_id: str, user: User):
|
|
|
|
|
... self.session_id = session_id
|
|
|
|
|
... self.user = user
|
|
|
|
|
|
|
|
|
|
${README_CONTENT} SEPARATOR=\n
|
|
|
|
|
... # Sample Web Application
|
|
|
|
|
...
|
|
|
|
|
... A simple web application with authentication, routing, and data models.
|
|
|
|
|
...
|
|
|
|
|
... ## Modules
|
|
|
|
|
...
|
|
|
|
|
... - **auth** — authentication and token management
|
|
|
|
|
... - **routes** — HTTP route definitions and handlers
|
|
|
|
|
... - **models** — data models (User, Session)
|
|
|
|
|
...
|
|
|
|
|
... ## Getting Started
|
|
|
|
|
...
|
|
|
|
|
... Install dependencies and run the application.
|
|
|
|
|
|
|
|
|
|
*** Keywords ***
|
|
|
|
|
WF09 Suite Setup
|
|
|
|
|
[Documentation] E2E Suite Setup plus unique suffix generation and actor selection.
|
|
|
|
|
E2E Suite Setup
|
|
|
|
|
# Initialise the database so session/config commands work in all tests.
|
|
|
|
|
${init}= Run CleverAgents Command init --force --yes
|
|
|
|
|
Should Be Equal As Integers ${init.rc} 0
|
|
|
|
|
Should Not Contain ${init.stdout}${init.stderr} Traceback
|
|
|
|
|
Should Not Contain ${init.stdout}${init.stderr} INTERNAL
|
|
|
|
|
# Generate a unique suffix for resource/project names to avoid UNIQUE
|
|
|
|
|
# constraint collisions on repeated or parallel CI runs.
|
|
|
|
|
${suffix}= Evaluate __import__('uuid').uuid4().hex[:12]
|
|
|
|
|
Set Suite Variable ${RUN_SUFFIX} ${suffix}
|
|
|
|
|
# Pick an actor that matches the available API key.
|
|
|
|
|
# Prefer Anthropic for session-based exploration: Claude models tend to
|
|
|
|
|
# produce richer conversational replies that improve session-history
|
|
|
|
|
# assertions. WF04/WF05 prefer OpenAI to balance provider coverage.
|
|
|
|
|
# Unlike WF04/WF05 which check both providers, WF09 only checks
|
|
|
|
|
# Anthropic and falls through to OpenAI unconditionally — this is
|
|
|
|
|
# intentional: ``session tell`` is currently stubbed (M3), so the
|
|
|
|
|
# actor choice affects only the echo prefix, not real LLM behaviour.
|
|
|
|
|
${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
|
|
|
|
|
IF ${has_anthropic}
|
|
|
|
|
${actor}= Set Variable anthropic/claude-sonnet-4-20250514
|
|
|
|
|
ELSE
|
|
|
|
|
${actor}= Set Variable openai/gpt-4o
|
|
|
|
|
END
|
|
|
|
|
Set Suite Variable ${LLM_ACTOR} ${actor}
|
|
|
|
|
|
|
|
|
|
Create Sample Project Fixture
|
|
|
|
|
[Documentation] Create a temp git repo with a multi-module Python project,
|
|
|
|
|
... register it as a resource and create a project. Returns
|
|
|
|
|
... the project name.
|
|
|
|
|
${repo}= Create Temp Git Repo wf09-explore-${RUN_SUFFIX}
|
|
|
|
|
Create File ${repo}${/}src${/}auth.py ${AUTH_PY}
|
|
|
|
|
Create File ${repo}${/}src${/}routes.py ${ROUTES_PY}
|
|
|
|
|
Create File ${repo}${/}src${/}models.py ${MODELS_PY}
|
|
|
|
|
Create File ${repo}${/}README.md ${README_CONTENT}
|
|
|
|
|
${git_add}= Run Process git add . cwd=${repo} timeout=30s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr}
|
|
|
|
|
${git_commit}= Run Process git commit -m Add sample project files cwd=${repo} timeout=30s on_timeout=kill
|
|
|
|
|
Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr}
|
|
|
|
|
# Register repo as a resource
|
|
|
|
|
${res_name}= Set Variable wf09-res-${RUN_SUFFIX}
|
|
|
|
|
${res_result}= Run CleverAgents Command
|
|
|
|
|
... resource add git-checkout ${res_name} --path ${repo}
|
|
|
|
|
Should Be Equal As Integers ${res_result.rc} 0
|
|
|
|
|
Should Not Contain ${res_result.stdout}${res_result.stderr} Traceback
|
|
|
|
|
Should Not Contain ${res_result.stdout}${res_result.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${res_result} ${res_name}
|
|
|
|
|
# Create a project linked to the resource
|
|
|
|
|
${proj_name}= Set Variable wf09-proj-${RUN_SUFFIX}
|
|
|
|
|
${proj_result}= Run CleverAgents Command
|
|
|
|
|
... project create --resource ${res_name} ${proj_name}
|
|
|
|
|
Should Be Equal As Integers ${proj_result.rc} 0
|
|
|
|
|
Should Not Contain ${proj_result.stdout}${proj_result.stderr} Traceback
|
|
|
|
|
Should Not Contain ${proj_result.stdout}${proj_result.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${proj_result} ${proj_name}
|
|
|
|
|
RETURN ${proj_name}
|
|
|
|
|
|
|
|
|
|
Create Session With Actor
|
|
|
|
|
[Documentation] Create a session with an LLM actor via ``session create``.
|
|
|
|
|
... Returns the session ID.
|
|
|
|
|
...
|
|
|
|
|
... Note: ``session create`` does not accept a ``--project``
|
|
|
|
|
... parameter. The session is actor-scoped; the project
|
|
|
|
|
... fixture is registered separately so the LLM can discover
|
|
|
|
|
... it via resource/project queries during the conversation.
|
|
|
|
|
${create_result}= Run CleverAgents Command
|
|
|
|
|
... session create --actor ${LLM_ACTOR} --format json
|
|
|
|
|
Should Not Be Empty ${create_result.stdout}
|
|
|
|
|
... msg=session create produced no output
|
|
|
|
|
Should Not Contain ${create_result.stdout}${create_result.stderr} Traceback
|
|
|
|
|
Should Not Contain ${create_result.stdout}${create_result.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${create_result} session_id
|
|
|
|
|
${session_id}= Safe Parse Json Field ${create_result.stdout} session_id
|
|
|
|
|
Should Not Be Empty ${session_id}
|
|
|
|
|
... msg=Could not extract session_id from session create output
|
|
|
|
|
RETURN ${session_id}
|
|
|
|
|
|
|
|
|
|
Verify Session Tell Response
|
|
|
|
|
[Documentation] Send a message to the session via ``session tell`` and verify
|
|
|
|
|
... that it produces a non-empty response with content relevance.
|
|
|
|
|
...
|
|
|
|
|
... ``session tell`` is currently stubbed (M3): the assistant
|
|
|
|
|
... echoes ``Acknowledged: <prompt>``. The content check
|
|
|
|
|
... validates this echo pattern. Update when real LLM
|
|
|
|
|
... integration lands.
|
|
|
|
|
[Arguments] ${session_id} ${message} ${timeout}=120s
|
|
|
|
|
${tell_result}= Run CleverAgents Command
|
|
|
|
|
... session tell --session ${session_id} ${message}
|
|
|
|
|
... timeout=${timeout}
|
|
|
|
|
Should Not Be Empty ${tell_result.stdout}
|
|
|
|
|
... msg=session tell produced no response for: ${message}
|
|
|
|
|
Should Not Contain ${tell_result.stdout}${tell_result.stderr} Traceback
|
|
|
|
|
Should Not Contain ${tell_result.stdout}${tell_result.stderr} INTERNAL
|
|
|
|
|
# Content relevance: stubbed session tell echoes "Acknowledged: <prompt>".
|
|
|
|
|
Output Should Contain ${tell_result} Acknowledged
|
|
|
|
|
RETURN ${tell_result}
|
|
|
|
|
|
|
|
|
|
Verify Session History
|
|
|
|
|
[Documentation] Verify that ``session show`` returns output containing
|
|
|
|
|
... the session ID and evidence of accumulated messages.
|
|
|
|
|
... Parses JSON output and asserts ``message_count >= 6``
|
|
|
|
|
... (3 user turns + 3 assistant responses) and that
|
|
|
|
|
... ``recent_messages`` is populated.
|
|
|
|
|
[Arguments] ${session_id}
|
|
|
|
|
${show_result}= Run CleverAgents Command
|
|
|
|
|
... session show ${session_id} --format json
|
|
|
|
|
Should Not Contain ${show_result.stdout}${show_result.stderr} Traceback
|
|
|
|
|
Should Not Contain ${show_result.stdout}${show_result.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${show_result} ${session_id}
|
|
|
|
|
# Parse JSON and verify message accumulation (AC #4).
|
|
|
|
|
# ``session show --format json`` exposes ``message_count`` (total) and
|
|
|
|
|
# ``recent_messages`` (last 5). Use Extract JSON From Stdout (raw_decode)
|
|
|
|
|
# to handle potential non-JSON preamble from the CLI.
|
|
|
|
|
${show_json}= Extract JSON From Stdout ${show_result.stdout}
|
|
|
|
|
${msg_count}= Evaluate int($show_json.get('message_count', 0))
|
|
|
|
|
Should Be True ${msg_count} >= 6
|
|
|
|
|
... msg=Expected message_count >= 6 (3 user turns + 3 assistant replies), found ${msg_count}
|
|
|
|
|
# Also verify recent_messages is populated (proves messages are persisted).
|
|
|
|
|
# After 3 user + 3 assistant turns the recent window (last 5) should
|
|
|
|
|
# contain at least 3 entries.
|
|
|
|
|
${recent}= Evaluate $show_json.get('recent_messages', [])
|
|
|
|
|
${recent_count}= Evaluate len($recent) if isinstance($recent, list) else 0
|
|
|
|
|
Should Be True ${recent_count} >= 3
|
|
|
|
|
... msg=Expected at least 3 recent_messages after 3 turns, found ${recent_count}
|
|
|
|
|
RETURN ${show_result}
|
|
|
|
|
|
|
|
|
|
Verify Session Export
|
|
|
|
|
[Documentation] Export the session to JSON and validate the exported file
|
|
|
|
|
... has the expected structure (session_id matching the input,
|
|
|
|
|
... non-empty messages list with valid message objects).
|
|
|
|
|
[Arguments] ${session_id}
|
|
|
|
|
${export_path}= Set Variable ${SUITE_HOME}${/}wf09_export.json
|
|
|
|
|
# Remove any stale export from a previous partial run to avoid false passes.
|
|
|
|
|
Remove File ${export_path}
|
|
|
|
|
# Spec canonical form: flag before positional arg.
|
|
|
|
|
${export_result}= Run CleverAgents Command
|
|
|
|
|
... session export --output ${export_path} ${session_id}
|
|
|
|
|
Should Not Contain ${export_result.stdout}${export_result.stderr} Traceback
|
|
|
|
|
Should Not Contain ${export_result.stdout}${export_result.stderr} INTERNAL
|
|
|
|
|
File Should Exist ${export_path} msg=session export did not create file
|
|
|
|
|
${export_content}= Get File ${export_path}
|
|
|
|
|
Should Not Be Empty ${export_content} msg=Export file is empty
|
|
|
|
|
TRY
|
|
|
|
|
${export_json}= Evaluate json.loads($export_content) json
|
|
|
|
|
EXCEPT AS ${err}
|
|
|
|
|
Fail session export produced invalid JSON: ${err}
|
|
|
|
|
END
|
|
|
|
|
Dictionary Should Contain Key ${export_json} session_id
|
|
|
|
|
Dictionary Should Contain Key ${export_json} messages
|
|
|
|
|
# Verify exported session_id matches the actual session.
|
|
|
|
|
${exported_sid}= Evaluate str($export_json.get('session_id', ''))
|
|
|
|
|
Should Be Equal As Strings ${exported_sid} ${session_id}
|
|
|
|
|
... msg=Exported session_id does not match the actual session
|
|
|
|
|
# Verify messages list is non-empty (not just structurally present).
|
|
|
|
|
${msg_list}= Evaluate $export_json.get('messages', [])
|
|
|
|
|
Should Not Be Empty ${msg_list} msg=Exported messages list should not be empty
|
|
|
|
|
# Spot-check first message has the expected structure (role field).
|
|
|
|
|
${first_msg}= Evaluate $msg_list[0] if $msg_list else {}
|
|
|
|
|
Dictionary Should Contain Key ${first_msg} role
|
|
|
|
|
... msg=Each exported message should have a 'role' field
|
|
|
|
|
|
|
|
|
|
WF09 Test Teardown
|
|
|
|
|
[Documentation] Log diagnostic context on failure for debugging.
|
|
|
|
|
... Captures session state so CI failures in this LLM-dependent
|
|
|
|
|
... test have actionable data.
|
|
|
|
|
${session_id}= Get Variable Value ${WF09_SESSION_ID} ${EMPTY}
|
|
|
|
|
IF '${session_id}' != ''
|
|
|
|
|
${status} ${result}= Run Keyword And Ignore Error
|
|
|
|
|
... Run CleverAgents Command session show ${session_id} --format json expected_rc=None timeout=30s
|
|
|
|
|
IF '${status}' == 'PASS'
|
|
|
|
|
Log Teardown session show: ${result.stdout} WARN
|
|
|
|
|
END
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
*** Test Cases ***
|
|
|
|
|
Workflow 09 — Session-Driven Interactive Exploration
|
|
|
|
|
[Documentation] End-to-end workflow: create a session, explore a codebase
|
|
|
|
|
... interactively via session tell, request action creation,
|
|
|
|
|
... review history, and export.
|
|
|
|
|
[Timeout] 20 minutes
|
|
|
|
|
[Teardown] WF09 Test Teardown
|
|
|
|
|
Skip If No LLM Keys
|
|
|
|
|
# Initialise test variable for teardown access.
|
|
|
|
|
Set Test Variable ${WF09_SESSION_ID} ${EMPTY}
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 1: Create a temp git repo with a sample project fixture
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# proj_name exists for LLM discovery; session create is actor-scoped,
|
|
|
|
|
# not project-scoped, so the variable is not referenced again directly.
|
|
|
|
|
${proj_name}= Create Sample Project Fixture
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 2: Create a session with an actor
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
${session_id}= Create Session With Actor
|
|
|
|
|
Set Test Variable ${WF09_SESSION_ID} ${session_id}
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 3: First conversational query — ask about modules
|
|
|
|
|
# ``session tell`` is currently stubbed (M3) — echoes an
|
|
|
|
|
# acknowledgement rather than calling a real LLM.
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
${tell1_result}= Verify Session Tell Response ${session_id}
|
|
|
|
|
... What are the main modules in this project?
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 4: Second conversational query — ask about auth module
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
${tell2_result}= Verify Session Tell Response ${session_id}
|
|
|
|
|
... Can you explain how the auth module works?
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 5: Third conversational query — request action creation
|
|
|
|
|
# (per spec Example 9, Step 2, third session tell interaction)
|
|
|
|
|
# Use 300s timeout: action creation involves LLM tool-calling which
|
|
|
|
|
# is significantly slower than simple conversational replies. This
|
|
|
|
|
# timeout is forward-compatible for when real LLM integration lands;
|
|
|
|
|
# while ``session tell`` remains stubbed the command returns
|
|
|
|
|
# instantly so the timeout has no practical effect.
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
${tell3_result}= Verify Session Tell Response ${session_id}
|
|
|
|
|
... Create an action called local/wf09-refresh-${RUN_SUFFIX} that automates dependency lock refresh
|
|
|
|
|
... timeout=300s
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 5b: Verify action creation (AC #5)
|
|
|
|
|
# The LLM was asked to create an action; verify it exists. Since LLM
|
|
|
|
|
# behavior is non-deterministic, use a soft check: warn if the action
|
|
|
|
|
# was not created rather than hard-failing.
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
${action_check}= Run CleverAgents Command
|
|
|
|
|
... action show local/wf09-refresh-${RUN_SUFFIX} --format json
|
|
|
|
|
... expected_rc=None
|
|
|
|
|
IF ${action_check.rc} != 0
|
|
|
|
|
Log AC #5 note: LLM did not create action local/wf09-refresh-${RUN_SUFFIX} in this run WARN
|
|
|
|
|
ELSE
|
|
|
|
|
Should Not Contain ${action_check.stdout}${action_check.stderr} Traceback
|
|
|
|
|
Should Not Contain ${action_check.stdout}${action_check.stderr} INTERNAL
|
|
|
|
|
Output Should Contain ${action_check} wf09-refresh
|
|
|
|
|
END
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 6: Verify session has accumulated history
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
${show_result}= Verify Session History ${session_id}
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
# Step 7: Export session to JSON and validate structure
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
Verify Session Export ${session_id}
|
|
|
|
|
|
|
|
|
|
Log Workflow 09 completed: session ${session_id}
|
freemo