diff --git a/CHANGELOG.md b/CHANGELOG.md index 707a2dec7..16177ee7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Added integration test for Workflow Example 9: session-driven interactive + exploration. Exercises `session create`, `tell`, `show`, `list`, and + `export` via Robot Framework with mocked LLM providers. (#773) - Added TDD bug-capture tests for bug #1076 — `use_action()` does not propagate `automation_profile` to Plan. Three Behave BDD scenarios (`@tdd_bug @tdd_bug_1076 @tdd_expected_fail`) verify the full precedence @@ -158,6 +161,7 @@ - Added tool-level execution environment preferences with NONE, REQUIRED, PREFERRED, and SPECIFIC modes. ToolRunner routes tool execution based on preference mode with caller-override precedence. (#879) + - Added TDD bug-capture tests for #969 — `plan correct` expects `decision_id` but M3 acceptance test passes `plan_id`. Behave BDD scenarios (revert and append modes) and Robot Framework integration tests verify that diff --git a/features/lsp_client.feature b/features/lsp_client.feature new file mode 100644 index 000000000..6a7e4d66e --- /dev/null +++ b/features/lsp_client.feature @@ -0,0 +1,36 @@ +@lsp @client +Feature: LSP Client + As a developer + I want a robust LSP client + So that I can communicate with language servers + + Scenario: Initialize handshake success + Given an LSP client with a mock transport + When I initialize the client with workspace "/tmp/test-ws" + Then the client should send an "initialize" request + And the client should send an "initialized" notification + And the client should be initialized + + Scenario: Shutdown sequence + Given an initialized LSP client + When I shutdown the client + Then the client should send a "shutdown" request + And the client should send an "exit" notification + And the client state should be uninitialized + + Scenario: Text document notifications + Given an initialized LSP client + When I open a document "file:///test.py" with content "print('hello')" + Then the client should send a "textDocument/didOpen" notification + When I close the document "file:///test.py" + Then the client should send a "textDocument/didClose" notification + + Scenario: Diagnostics handling + Given an initialized LSP client + When the server publishes diagnostics for "file:///test.py" + Then the client should have diagnostics for "file:///test.py" + + Scenario: Completion request + Given an initialized LSP client + When I request completions for "file:///test.py" at line 1 character 5 + Then the client should send a "textDocument/completion" request diff --git a/features/steps/lsp_client_steps.py b/features/steps/lsp_client_steps.py new file mode 100644 index 000000000..7614c4827 --- /dev/null +++ b/features/steps/lsp_client_steps.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from behave import given, then, when + +from cleveragents.lsp.client import LspClient +from cleveragents.lsp.transport import StdioTransport + + +class MockTransport(StdioTransport): + def __init__(self): + self.sent_messages = [] + self.responses = [] + self._is_alive = True + + @property + def is_alive(self) -> bool: + return self._is_alive + + def start(self) -> None: + pass + + def stop(self, timeout=None) -> None: + pass + + def send_message(self, body: dict) -> None: + self.sent_messages.append(body) + + def read_message(self, timeout=None) -> dict | None: + if self.responses: + return self.responses.pop(0) + return None + + def queue_response(self, response: dict): + self.responses.append(response) + + +@given("an LSP client with a mock transport") +def step_client_with_mock_transport(context): + context.transport = MockTransport() + # Mock __init__ of StdioTransport is bypassed by our subclass, + # but LspClient checks it. + context.client = LspClient(context.transport, server_name="test-server") + + +@when('I initialize the client with workspace "{workspace}"') +def step_initialize_client(context, workspace): + # Queue the response for the initialize request + # ID will be 1 because it's the first request + context.transport.queue_response( + {"jsonrpc": "2.0", "id": 1, "result": {"capabilities": {"textDocumentSync": 1}}} + ) + context.client.initialize(workspace) + + +@then('the client should send an "initialize" request') +def step_check_initialize_request(context): + msg = context.transport.sent_messages[0] + assert msg["method"] == "initialize" + + +@then('the client should send an "initialized" notification') +def step_check_initialized_notification(context): + msg = context.transport.sent_messages[1] + assert msg["method"] == "initialized" + + +@then("the client should be initialized") +def step_check_initialized(context): + assert context.client.is_initialized + + +@given("an initialized LSP client") +def step_initialized_client(context): + step_client_with_mock_transport(context) + step_initialize_client(context, "/tmp/ws") + context.transport.sent_messages.clear() # Clear init messages + # Reset request ID counter if needed, or track IDs. + # LspClient._request_id will be 1. Next request is 2. + + +@when("I shutdown the client") +def step_shutdown_client(context): + # Next ID is 2 + context.transport.queue_response({"jsonrpc": "2.0", "id": 2, "result": None}) + context.client.shutdown() + + +@then('the client should send a "shutdown" request') +def step_check_shutdown_request(context): + msg = next(m for m in context.transport.sent_messages if m["method"] == "shutdown") + assert msg + + +@then('the client should send an "exit" notification') +def step_check_exit_notification(context): + msg = next(m for m in context.transport.sent_messages if m["method"] == "exit") + assert msg + + +@then("the client state should be uninitialized") +def step_check_not_initialized(context): + assert not context.client.is_initialized + + +@when('I open a document "{uri}" with content "{content}"') +def step_open_document(context, uri, content): + context.client.did_open(uri, "python", 1, content) + + +@then('the client should send a "textDocument/didOpen" notification') +def step_check_did_open(context): + msg = context.transport.sent_messages[-1] + assert msg["method"] == "textDocument/didOpen" + + +@when('I close the document "{uri}"') +def step_close_document(context, uri): + context.client.did_close(uri) + + +@then('the client should send a "textDocument/didClose" notification') +def step_check_did_close(context): + msg = context.transport.sent_messages[-1] + assert msg["method"] == "textDocument/didClose" + + +@when('the server publishes diagnostics for "{uri}"') +def step_server_publishes_diagnostics(context, uri): + notification = { + "jsonrpc": "2.0", + "method": "textDocument/publishDiagnostics", + "params": {"uri": uri, "diagnostics": [{"message": "error", "range": {}}]}, + } + context.client._handle_notification(notification) + + +@then('the client should have diagnostics for "{uri}"') +def step_check_diagnostics(context, uri): + diags = context.client.get_diagnostics(uri) + assert len(diags) > 0 + + +@when('I request completions for "{uri}" at line {line:d} character {char:d}') +def step_request_completions(context, uri, line, char): + # Next ID is 2 (after init) + context.transport.queue_response({"jsonrpc": "2.0", "id": 2, "result": []}) + context.client.get_completions(uri, line, char) + + +@then('the client should send a "textDocument/completion" request') +def step_check_completion_request(context): + msg = context.transport.sent_messages[-1] + assert msg["method"] == "textDocument/completion" diff --git a/robot/helper_wf09_session_exploration.py b/robot/helper_wf09_session_exploration.py new file mode 100644 index 000000000..039f1d363 --- /dev/null +++ b/robot/helper_wf09_session_exploration.py @@ -0,0 +1,473 @@ +"""Helper script for wf09_session_exploration.robot E2E tests. + +Workflow Example 9: Session-Driven Interactive Exploration. +Exercises session-based conversational interaction with ``session create``, +``session tell``, ``session show``, ``session list``, and ``session export`` +using mocked LLM providers. + +Each subcommand is self-contained, sets up its own workspace, and prints +a sentinel on success. Exit code 0 = pass, 1 = failure. + +Usage:: + + python robot/helper_wf09_session_exploration.py +""" + +from __future__ import annotations + +import json +import os +import sys +from collections.abc import Callable +from pathlib import Path + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) +_ROBOT = str(Path(__file__).resolve().parent) +if _ROBOT not in sys.path: + sys.path.insert(0, _ROBOT) + +from helper_e2e_common import ( # noqa: E402 + cleanup_workspace, + fail, + run_cli, + setup_workspace, + write_yaml, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _init_workspace(prefix: str) -> str: + """Create a workspace and run ``agents init --yes``.""" + workspace = setup_workspace(prefix=prefix) + result = run_cli("init", "--yes", workspace=workspace) + if result.returncode != 0: + cleanup_workspace(workspace) + fail( + f"agents init --yes failed (rc={result.returncode})\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + return workspace + + +def _extract_session_id(stdout: str) -> str: + """Extract a session ULID from ``session create --format plain`` output. + + The plain output contains a line like ``session_id: 01HXYZ...``. + Also handles debug log lines that prefix the output. + """ + for line in stdout.splitlines(): + stripped = line.strip() + if stripped.lower().startswith("session_id:"): + sid = stripped.split(":", 1)[1].strip() + if len(sid) != 26 or not sid.isalnum(): + fail(f"Extracted session ID is not a valid ULID: {sid!r}") + return sid + fail(f"Could not extract session ID from output:\n{stdout}") + # unreachable, but keeps the type checker happy + return "" # pragma: no cover + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + + +def create_session() -> None: + """Create a new session and verify the ID is returned.""" + workspace = _init_workspace("wf09_create_") + try: + result = run_cli( + "session", + "create", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail( + f"session create rc={result.returncode}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + sid = _extract_session_id(result.stdout) + if not sid: + fail("session create returned empty ID") + print(f"Session created: {sid}") + print("wf09-create-session-ok") + finally: + cleanup_workspace(workspace) + + +def session_tell_basic() -> None: + """Create a session, send a message via ``session tell``, verify response.""" + workspace = _init_workspace("wf09_tell_") + try: + # Create session + result = run_cli( + "session", + "create", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session create rc={result.returncode}\nstderr: {result.stderr}") + sid = _extract_session_id(result.stdout) + + # Send a message + result = run_cli( + "session", + "tell", + "--session", + sid, + "Hello, can you analyze this codebase?", + workspace=workspace, + ) + if result.returncode != 0: + fail( + f"session tell rc={result.returncode}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + # The stubbed tell should produce some output (acknowledgement) + if "Traceback" in result.stderr or "INTERNAL" in result.stderr: + fail(f"session tell produced errors:\nstderr: {result.stderr}") + if not result.stdout.strip(): + fail("session tell produced empty stdout") + print(f"Tell response length: {len(result.stdout)} chars") + print("wf09-session-tell-ok") + finally: + cleanup_workspace(workspace) + + +def session_show_details() -> None: + """Create session, send message, then ``session show`` to verify metadata.""" + workspace = _init_workspace("wf09_show_") + try: + # Create session + result = run_cli( + "session", + "create", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session create rc={result.returncode}\nstderr: {result.stderr}") + sid = _extract_session_id(result.stdout) + + # Send a message + result = run_cli( + "session", + "tell", + "--session", + sid, + "Explore the architecture", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session tell rc={result.returncode}\nstderr: {result.stderr}") + + # Show session details + result = run_cli( + "session", + "show", + sid, + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail( + f"session show rc={result.returncode}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + if "Traceback" in result.stderr or "INTERNAL" in result.stderr: + fail(f"session show produced errors:\nstderr: {result.stderr}") + # The show output should mention the session ID + if sid not in result.stdout: + fail(f"session show output does not contain session ID {sid}") + print(f"Session show for {sid}: OK") + print("wf09-session-show-ok") + finally: + cleanup_workspace(workspace) + + +def session_history_accumulation() -> None: + """Send multiple messages and verify history accumulates.""" + workspace = _init_workspace("wf09_history_") + try: + # Create session + result = run_cli( + "session", + "create", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session create rc={result.returncode}\nstderr: {result.stderr}") + sid = _extract_session_id(result.stdout) + + # Send three messages + messages = [ + "What modules exist in this codebase?", + "Tell me about the CLI layer", + "How does the DI container work?", + ] + for msg in messages: + result = run_cli( + "session", + "tell", + "--session", + sid, + msg, + workspace=workspace, + ) + if result.returncode != 0: + fail( + f"session tell '{msg}' rc={result.returncode}\n" + f"stderr: {result.stderr}" + ) + + # Show session to verify accumulated history + result = run_cli( + "session", + "show", + sid, + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session show rc={result.returncode}\nstderr: {result.stderr}") + if "Traceback" in result.stderr or "INTERNAL" in result.stderr: + fail(f"session show errors:\nstderr: {result.stderr}") + # Verify the output references message count or contains messages + # In plain format, the show command should list messages + stdout_lower = result.stdout.lower() + if sid.lower() not in stdout_lower: + fail(f"session show does not mention session ID {sid}") + if "message_count" not in stdout_lower: + fail( + f"session show does not contain 'message_count' after 3 tells.\n" + f"stdout: {result.stdout}" + ) + print(f"History accumulation for {sid}: 3 messages sent") + print("wf09-session-history-ok") + finally: + cleanup_workspace(workspace) + + +def session_export_json() -> None: + """Create session, send message, export to JSON, verify structure.""" + workspace = _init_workspace("wf09_export_") + try: + # Create session + result = run_cli( + "session", + "create", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session create rc={result.returncode}\nstderr: {result.stderr}") + sid = _extract_session_id(result.stdout) + + # Send a message to have content to export + result = run_cli( + "session", + "tell", + "--session", + sid, + "Summarize this project", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session tell rc={result.returncode}\nstderr: {result.stderr}") + + # Export session to file + export_path = os.path.join(workspace, "session_export.json") + result = run_cli( + "session", + "export", + sid, + "--output", + export_path, + workspace=workspace, + ) + if result.returncode != 0: + fail( + f"session export rc={result.returncode}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + + # Verify JSON file exists and has valid structure + if not os.path.isfile(export_path): + fail(f"Export file not created at {export_path}") + with open(export_path, encoding="utf-8") as fh: + data = json.load(fh) + if not isinstance(data, dict): + fail(f"Exported JSON is not a dict: {type(data)}") + # Check for expected keys + if "session_id" not in data: + fail(f"Exported JSON missing 'session_id' key. Keys: {list(data.keys())}") + if "messages" not in data: + fail(f"Exported JSON missing 'messages' key. Keys: {list(data.keys())}") + if "schema_version" not in data: + fail( + f"Exported JSON missing 'schema_version' key. Keys: {list(data.keys())}" + ) + msg_count = len(data["messages"]) + if msg_count < 2: + fail(f"Expected at least 2 messages (user + assistant), got {msg_count}") + print(f"Exported session {sid}: {msg_count} message(s)") + print("wf09-session-export-ok") + finally: + cleanup_workspace(workspace) + + +def session_list_shows_created() -> None: + """Create a session and verify ``session list`` shows it.""" + workspace = _init_workspace("wf09_list_") + try: + # Create session + result = run_cli( + "session", + "create", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session create rc={result.returncode}\nstderr: {result.stderr}") + sid = _extract_session_id(result.stdout) + + # List sessions + result = run_cli( + "session", + "list", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail( + f"session list rc={result.returncode}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + if "Traceback" in result.stderr or "INTERNAL" in result.stderr: + fail(f"session list errors:\nstderr: {result.stderr}") + # The list output should contain the session ID + if sid not in result.stdout: + fail( + f"session list output does not contain session ID {sid}\n" + f"stdout: {result.stdout}" + ) + print("Session list includes created session: OK") + print("wf09-session-list-ok") + finally: + cleanup_workspace(workspace) + + +def action_from_conversation() -> None: + """Demonstrate action creation after a session conversation. + + With mock AI, the assistant echoes an acknowledgement rather than + generating an action YAML. This test verifies that after a + conversational exploration session the user can create an action + via the standard ``action create --config`` path. + """ + workspace = _init_workspace("wf09_action_") + try: + # Create session and converse + result = run_cli( + "session", + "create", + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session create rc={result.returncode}\nstderr: {result.stderr}") + sid = _extract_session_id(result.stdout) + + result = run_cli( + "session", + "tell", + "--session", + sid, + "I want to fix a bug in the parser module. Create an action for me.", + workspace=workspace, + ) + if result.returncode != 0: + fail(f"session tell rc={result.returncode}\nstderr: {result.stderr}") + + # In a real workflow, the AI would output action YAML. + # With mock AI, we simulate the user creating an action from + # the conversation's guidance. + action_yaml = write_yaml( + "name: local/fix-parser-bug\n" + "description: Fix the parser bug from session\n" + "automation_profile: review\n" + "strategy_actor: openai/gpt-4\n" + "execution_actor: openai/gpt-4\n" + "definition_of_done: Parser bug is fixed and tests pass\n" + ) + result = run_cli( + "action", + "create", + "--config", + action_yaml, + workspace=workspace, + ) + if result.returncode != 0: + fail( + f"action create rc={result.returncode}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + if "Traceback" in result.stderr or "INTERNAL" in result.stderr: + fail(f"action create errors:\nstderr: {result.stderr}") + print(f"Action created from session {sid} conversation") + print("wf09-action-from-conversation-ok") + finally: + cleanup_workspace(workspace) + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +_COMMANDS: dict[str, Callable[[], None]] = { + "create-session": create_session, + "session-tell": session_tell_basic, + "session-show": session_show_details, + "session-history": session_history_accumulation, + "session-export": session_export_json, + "session-list": session_list_shows_created, + "action-from-conversation": action_from_conversation, +} + + +def main() -> int: + """Dispatch to the requested subcommand.""" + if len(sys.argv) < 2: + print( + f"Usage: helper_wf09_session_exploration.py <{'|'.join(_COMMANDS)}>", + ) + return 1 + command = sys.argv[1] + handler = _COMMANDS.get(command) + if handler is None: + print(f"Unknown command: {command}") + return 1 + handler() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/robot/wf09_session_exploration.robot b/robot/wf09_session_exploration.robot new file mode 100644 index 000000000..8de16798c --- /dev/null +++ b/robot/wf09_session_exploration.robot @@ -0,0 +1,71 @@ +*** Settings *** +Documentation WF09: Session-driven interactive exploration workflow E2E tests. +... Exercises session create, tell, show, list, and export using +... real CLI subprocesses with mocked LLM providers. +Resource ${CURDIR}/common.resource +Library Process +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment +Force Tags integration session workflow wf09 + +*** Variables *** +${HELPER} ${CURDIR}/helper_wf09_session_exploration.py +${TIMEOUT} 120s + +*** Test Cases *** +WF09 Create Exploration Session + [Documentation] Create a new session and verify a ULID is returned. + ${result}= Run Process ${PYTHON} ${HELPER} create-session cwd=${SUITE_HOME} timeout=${TIMEOUT} on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf09-create-session-ok + +WF09 Session Tell Basic + [Documentation] Create a session and send a message via session tell. + ${result}= Run Process ${PYTHON} ${HELPER} session-tell cwd=${SUITE_HOME} timeout=${TIMEOUT} on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf09-session-tell-ok + +WF09 Session Show Details + [Documentation] Create session, send message, and verify session show metadata. + ${result}= Run Process ${PYTHON} ${HELPER} session-show cwd=${SUITE_HOME} timeout=${TIMEOUT} on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf09-session-show-ok + +WF09 Session History Accumulation + [Documentation] Send multiple messages and verify history accumulates. + ${result}= Run Process ${PYTHON} ${HELPER} session-history cwd=${SUITE_HOME} timeout=${TIMEOUT} on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf09-session-history-ok + +WF09 Session Export JSON Structure + [Documentation] Export a session to JSON and verify the file structure. + ${result}= Run Process ${PYTHON} ${HELPER} session-export cwd=${SUITE_HOME} timeout=${TIMEOUT} on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf09-session-export-ok + +WF09 Session List Includes Created + [Documentation] Create a session and verify session list shows it. + ${result}= Run Process ${PYTHON} ${HELPER} session-list cwd=${SUITE_HOME} timeout=${TIMEOUT} on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf09-session-list-ok + +WF09 Action From Session Conversation + [Documentation] After a session conversation, create an action to show + ... the exploration-to-action workflow path. + ${result}= Run Process ${PYTHON} ${HELPER} action-from-conversation cwd=${SUITE_HOME} timeout=${TIMEOUT} on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf09-action-from-conversation-ok