diff --git a/noxfile.py b/noxfile.py index 72e47daf4..6b054ecd3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -580,6 +580,11 @@ def integration_tests(session: nox.Session): pabot_args, robot_args = _split_pabot_args(session.posargs) parallel_args = _pabot_parallel_args(pabot_args) + # Conditionally exclude LLM-dependent tests when API keys are absent. + llm_exclude_args: list[str] = [] + if not os.environ.get("ANTHROPIC_API_KEY"): + llm_exclude_args = ["--exclude", "llm-required"] + session.run( "pabot", *parallel_args, @@ -604,6 +609,7 @@ def integration_tests(session: nox.Session): "code_blocks", "--exclude", "wip", + *llm_exclude_args, "--listener", "robot/tdd_expected_fail_listener.py", *robot_args, diff --git a/robot/_test_utils.py b/robot/_test_utils.py new file mode 100644 index 000000000..51b86af5f --- /dev/null +++ b/robot/_test_utils.py @@ -0,0 +1,46 @@ +"""Robot Framework test utilities — local to the ``robot/`` directory. + +Contains pure utility functions that do not depend on any test framework +or mock library. Test doubles (mocks, stubs, fakes) do NOT belong here; +integration tests should use real implementations per CONTRIBUTING.md. +""" + +from __future__ import annotations + +import json +from typing import Any + +# --------------------------------------------------------------------------- +# LSP response parsing +# --------------------------------------------------------------------------- + + +def parse_lsp_responses(raw: bytes) -> list[dict[str, Any]]: + """Parse Content-Length framed JSON-RPC responses from raw bytes. + + Args: + raw: The raw bytes containing one or more Content-Length + framed JSON-RPC response messages. + + Returns: + List of parsed JSON-RPC response dicts. + """ + responses: list[dict[str, Any]] = [] + offset = 0 + while offset < len(raw): + header_end = raw.find(b"\r\n\r\n", offset) + if header_end == -1: + break + header_section = raw[offset:header_end].decode("utf-8", errors="replace") + content_length: int | None = None + for line in header_section.split("\r\n"): + if line.lower().startswith("content-length:"): + content_length = int(line.split(":", 1)[1].strip()) + if content_length is None: + break + body_start = header_end + 4 + body_end = body_start + content_length + body = raw[body_start:body_end] + responses.append(json.loads(body)) + offset = body_end + return responses diff --git a/robot/context_analysis_agent.robot b/robot/context_analysis_agent.robot index 797a10c72..af8676b5c 100644 --- a/robot/context_analysis_agent.robot +++ b/robot/context_analysis_agent.robot @@ -25,12 +25,13 @@ Context Analysis Agent Module Can Be Imported Context Analysis Agent Can Be Instantiated With Default Parameters [Documentation] Create ContextAnalysisAgent with defaults + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.context_analysis import ContextAnalysisAgent - ... from langchain_community.llms import FakeListLLM - ... agent = ContextAnalysisAgent(llm=FakeListLLM(responses=['test']*3)) + ... from langchain_anthropic import ChatAnthropic + ... agent = ContextAnalysisAgent(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... assert agent is not None ... assert agent.chunk_size == 2000 ... assert agent.chunk_overlap == 200 @@ -42,12 +43,13 @@ Context Analysis Agent Can Be Instantiated With Default Parameters Context Analysis Agent Can Be Instantiated With Custom Chunk Settings [Documentation] Create ContextAnalysisAgent with custom chunk_size and overlap + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.context_analysis import ContextAnalysisAgent - ... from langchain_community.llms import FakeListLLM - ... agent = ContextAnalysisAgent(llm=FakeListLLM(responses=['test']*3), chunk_size=1000, chunk_overlap=100) + ... from langchain_anthropic import ChatAnthropic + ... agent = ContextAnalysisAgent(llm=ChatAnthropic(model='claude-3-haiku-20240307'), chunk_size=1000, chunk_overlap=100) ... assert agent.chunk_size == 1000 ... assert agent.chunk_overlap == 100 ... print('Chunk size: ' + str(agent.chunk_size) + ', overlap: ' + str(agent.chunk_overlap)) diff --git a/robot/database_integration.robot b/robot/database_integration.robot index 67e25f914..0f39a11d0 100644 --- a/robot/database_integration.robot +++ b/robot/database_integration.robot @@ -112,6 +112,7 @@ Unit Of Work Transaction Rollback Service Layer Uses Repositories [Documentation] Test that services properly use repositories + [Tags] llm-required Create Temporary Project Directory Initialize Project With Service service-project ${project}= Get Current Project From Service @@ -127,6 +128,7 @@ Service Layer Uses Repositories End To End Database Workflow [Documentation] Test complete workflow using database + [Tags] llm-required Create Temporary Project Directory # Run the complete workflow in one script @@ -138,14 +140,14 @@ End To End Database Workflow ... from cleveragents.application.services.context_service import ContextService ... from cleveragents.infrastructure.database.unit_of_work import UnitOfWork ... from cleveragents.config.settings import Settings - ... from features.mocks.mock_ai_provider import MockAIProvider + ... from cleveragents.providers.registry import get_provider_registry ... from pathlib import Path - ... os.environ["CLEVERAGENTS_TESTING_USE_MOCK_AI"] = "true" ... settings = Settings() ... uow = UnitOfWork('sqlite:///${TEMP_DIR}/test.db') - ... mock_ai = MockAIProvider() + ... registry = get_provider_registry(settings) + ... ai_provider = registry.create_ai_provider(provider_type='anthropic', model_id='claude-3-haiku-20240307') ... project_service = ProjectService(settings, uow) - ... plan_service = PlanService(settings, uow, mock_ai) + ... plan_service = PlanService(settings, uow, ai_provider) ... plan_service.actor_service.ensure_default_mock_actor(force=True) ... context_service = ContextService(settings, uow) ... # Initialize project @@ -548,14 +550,15 @@ Create Plan With Service ... from cleveragents.application.services.project_service import ProjectService ... from cleveragents.infrastructure.database.unit_of_work import UnitOfWork ... from cleveragents.config.settings import Settings - ... from features.mocks.mock_ai_provider import MockAIProvider + ... from cleveragents.providers.registry import get_provider_registry ... import os ... os.chdir('${TEMP_DIR}') ... settings = Settings() ... uow = UnitOfWork('sqlite:///${TEMP_DIR}/test.db') - ... mock_ai = MockAIProvider() + ... registry = get_provider_registry(settings) + ... ai_provider = registry.create_ai_provider(provider_type='anthropic', model_id='claude-3-haiku-20240307') ... project_service = ProjectService(settings, uow) - ... plan_service = PlanService(settings, uow, mock_ai) + ... plan_service = PlanService(settings, uow, ai_provider) ... project = project_service.get_current_project() ... plan = plan_service.create_plan(project, '${prompt}') ... print('Plan created') diff --git a/robot/fixtures/mcp_echo_server.py b/robot/fixtures/mcp_echo_server.py new file mode 100644 index 000000000..18f924031 --- /dev/null +++ b/robot/fixtures/mcp_echo_server.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Minimal MCP server for integration testing. + +Speaks the Model Context Protocol over stdio using newline-delimited +JSON-RPC (NDJSON). Provides two tools: + + * ``echo`` — returns its ``message`` argument unchanged. + * ``add`` — returns the sum of ``a`` and ``b``. + +Usage:: + + python robot/fixtures/mcp_echo_server.py + +The server reads JSON-RPC requests from stdin (one per line) and writes +JSON-RPC responses to stdout (one per line). It exits when stdin is +closed or when it receives an unrecoverable error. +""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +# -- Tool definitions exposed via tools/list --------------------------------- + +TOOLS: list[dict[str, Any]] = [ + { + "name": "echo", + "description": "Echoes the input message back.", + "inputSchema": { + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "Message to echo.", + }, + }, + "required": ["message"], + }, + }, + { + "name": "add", + "description": "Adds two numbers and returns the sum.", + "inputSchema": { + "type": "object", + "properties": { + "a": {"type": "number", "description": "First number."}, + "b": {"type": "number", "description": "Second number."}, + }, + "required": ["a", "b"], + }, + }, +] + + +# -- Tool invocation handlers ------------------------------------------------ + + +def _invoke_echo(arguments: dict[str, Any]) -> dict[str, Any]: + return {"content": [{"type": "text", "text": arguments.get("message", "")}]} + + +def _invoke_add(arguments: dict[str, Any]) -> dict[str, Any]: + a = arguments.get("a", 0) + b = arguments.get("b", 0) + return {"content": [{"type": "text", "text": str(a + b)}]} + + +TOOL_HANDLERS: dict[str, Any] = { + "echo": _invoke_echo, + "add": _invoke_add, +} + + +# -- JSON-RPC helpers -------------------------------------------------------- + + +def _respond(request_id: int | str | None, result: dict[str, Any]) -> None: + """Write a JSON-RPC success response.""" + msg = {"jsonrpc": "2.0", "id": request_id, "result": result} + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def _error( + request_id: int | str | None, + code: int, + message: str, +) -> None: + """Write a JSON-RPC error response.""" + msg = { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": code, "message": message}, + } + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +# -- Request routing --------------------------------------------------------- + + +def _handle_initialize( + request_id: int | str | None, + _params: dict[str, Any], +) -> None: + _respond( + request_id, + { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {"listChanged": True}, + }, + "serverInfo": { + "name": "mcp-echo-server", + "version": "1.0.0", + }, + }, + ) + + +def _handle_tools_list( + request_id: int | str | None, + _params: dict[str, Any], +) -> None: + _respond(request_id, {"tools": TOOLS}) + + +def _handle_tools_call( + request_id: int | str | None, + params: dict[str, Any], +) -> None: + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + + handler = TOOL_HANDLERS.get(tool_name) + if handler is None: + _error(request_id, -32602, f"Unknown tool: {tool_name}") + return + + try: + result = handler(arguments) + _respond(request_id, result) + except Exception as exc: + _respond(request_id, {"isError": True, "error": str(exc)}) + + +HANDLERS: dict[str, Any] = { + "initialize": _handle_initialize, + "tools/list": _handle_tools_list, + "tools/call": _handle_tools_call, +} + + +# -- Main loop --------------------------------------------------------------- + + +def main() -> None: + """Read JSON-RPC requests from stdin and dispatch responses.""" + for line in sys.stdin: + line = line.strip() + if not line: + continue + + try: + request = json.loads(line) + except json.JSONDecodeError: + continue + + # Notifications (no id) are acknowledged silently. + request_id = request.get("id") + method = request.get("method", "") + + if request_id is None: + # This is a notification — no response needed. + continue + + handler = HANDLERS.get(method) + if handler is None: + _error(request_id, -32601, f"Method not found: {method}") + continue + + handler(request_id, request.get("params", {})) + + +if __name__ == "__main__": + main() diff --git a/robot/helper_context_analysis.py b/robot/helper_context_analysis.py index 117ec09d2..83f12292f 100644 --- a/robot/helper_context_analysis.py +++ b/robot/helper_context_analysis.py @@ -15,7 +15,7 @@ from typing import Any src_dir = Path(__file__).parent.parent / "src" sys.path.insert(0, str(src_dir)) -from langchain_community.llms import FakeListLLM # noqa: E402 +from langchain_anthropic import ChatAnthropic # noqa: E402 from cleveragents.agents.context_analysis import ( # noqa: E402 ContextAnalysisAgent, @@ -23,18 +23,15 @@ from cleveragents.agents.context_analysis import ( # noqa: E402 ) +def _create_llm() -> ChatAnthropic: + """Create a real ChatAnthropic LLM instance for integration tests.""" + return ChatAnthropic(model="claude-3-haiku-20240307") + + def test_nodes() -> None: """Test that the workflow graph contains all expected nodes.""" try: - agent = ContextAnalysisAgent( - llm=FakeListLLM( - responses=[ - "Dependencies: ['os']", - "Relevance: High", - "Summary: test", - ] - ) - ) + agent = ContextAnalysisAgent(llm=_create_llm()) # Get the nodes from the graph graph = agent.graph @@ -66,15 +63,7 @@ def test_nodes() -> None: def test_load_files() -> None: """Test file loading functionality.""" try: - agent = ContextAnalysisAgent( - llm=FakeListLLM( - responses=[ - "Dependencies: ['os']", - "Relevance: High", - "Summary: test", - ] - ) - ) + agent = ContextAnalysisAgent(llm=_create_llm()) # Create a temporary test file with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: @@ -117,15 +106,7 @@ def test_load_files() -> None: def test_missing_file() -> None: """Test error handling for missing files.""" try: - agent = ContextAnalysisAgent( - llm=FakeListLLM( - responses=[ - "Dependencies: ['os']", - "Relevance: High", - "Summary: test", - ] - ) - ) + agent = ContextAnalysisAgent(llm=_create_llm()) # Create state with non-existent file state: ContextAnalysisState = { @@ -163,15 +144,7 @@ def test_missing_file() -> None: def test_invoke() -> None: """Test complete workflow execution via invoke.""" try: - agent = ContextAnalysisAgent( - llm=FakeListLLM( - responses=[ - "Dependencies: ['os']", - "Relevance: High", - "Summary: test", - ] - ) - ) + agent = ContextAnalysisAgent(llm=_create_llm()) # Create a temporary test file with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: @@ -237,15 +210,7 @@ def test_invoke() -> None: def test_streaming() -> None: """Test streaming workflow execution.""" try: - agent = ContextAnalysisAgent( - llm=FakeListLLM( - responses=[ - "Dependencies: ['os']", - "Relevance: High", - "Summary: test", - ] - ) - ) + agent = ContextAnalysisAgent(llm=_create_llm()) # Create a temporary test file with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: diff --git a/robot/helper_lsp_stub.py b/robot/helper_lsp_stub.py index c7f98035f..76db0ef34 100644 --- a/robot/helper_lsp_stub.py +++ b/robot/helper_lsp_stub.py @@ -29,14 +29,12 @@ if src_dir not in sys.path: from cleveragents.lsp.server import LspServer # noqa: E402 -# Import the shared response-parsing utility from the test mocks -# package so that parsing logic is defined in a single place. -# Same sys.path rationale as above — needed for standalone execution. -features_dir = str(Path(__file__).resolve().parents[1]) -if features_dir not in sys.path: - sys.path.insert(0, features_dir) +# Robot-local stubs (avoid cross-framework imports from features/mocks/) +robot_dir = str(Path(__file__).resolve().parent) +if robot_dir not in sys.path: + sys.path.insert(0, robot_dir) -from features.mocks.lsp_transport_mock import parse_lsp_responses # noqa: E402 +from _test_utils import parse_lsp_responses # noqa: E402 def _encode_message(msg: dict) -> bytes: diff --git a/robot/helper_mcp_adapter.py b/robot/helper_mcp_adapter.py index c2dd6804f..e557155bd 100644 --- a/robot/helper_mcp_adapter.py +++ b/robot/helper_mcp_adapter.py @@ -1,5 +1,8 @@ """Robot Framework helper for MCP Tool Adapter smoke tests. +Uses a real ``StdioMCPTransport`` connecting to the Python-based +``mcp_echo_server.py`` fixture instead of mock transports. + Usage: python robot/helper_mcp_adapter.py create-adapter python robot/helper_mcp_adapter.py discover-tools @@ -12,22 +15,34 @@ from __future__ import annotations import sys from pathlib import Path -from typing import Any -_ROOT = str(Path(__file__).resolve().parents[1]) _SRC = str(Path(__file__).resolve().parents[1] / "src") -for _p in (_SRC, _ROOT): - if _p not in sys.path: - sys.path.insert(0, _p) - -from features.mocks.mock_mcp_transport import MockMCPTransport # noqa: E402 +if _SRC not in sys.path: + sys.path.insert(0, _SRC) from cleveragents.mcp.adapter import MCPServerConfig, MCPToolAdapter # noqa: E402 +from cleveragents.mcp.stdio_transport import StdioMCPTransport # noqa: E402 from cleveragents.tool.registry import ToolRegistry # noqa: E402 +# Path to the MCP echo test server fixture. +_FIXTURE_SERVER = str( + Path(__file__).resolve().parent / "fixtures" / "mcp_echo_server.py" +) -def _mock_tool(name: str) -> dict[str, Any]: - return {"name": name, "description": f"Mock {name}", "inputSchema": {}} + +def _make_transport() -> StdioMCPTransport: + """Create a real stdio transport with a short timeout for tests.""" + return StdioMCPTransport(response_timeout=10.0) + + +def _make_config(name: str = "test") -> MCPServerConfig: + """Create a config pointing to the MCP echo fixture server.""" + return MCPServerConfig( + name=name, + transport="stdio", + command=sys.executable, + args=[_FIXTURE_SERVER], + ) def main() -> int: @@ -59,49 +74,49 @@ def _create_adapter() -> int: def _discover_tools() -> int: - tools = [ - _mock_tool("create_issue"), - _mock_tool("list_repos"), - _mock_tool("search"), - ] - config = MCPServerConfig(name="test", transport="stdio", command="echo") - transport = MockMCPTransport(tools=tools) + config = _make_config() + transport = _make_transport() adapter = MCPToolAdapter(config=config, transport=transport) - adapter.connect() - discovered = adapter.discover_tools() - print(f"discovered: {len(discovered)}") - for i, t in enumerate(discovered): - print(f"tool-{i}: {t.name}") - return 0 + try: + adapter.connect() + discovered = adapter.discover_tools() + print(f"discovered: {len(discovered)}") + for i, t in enumerate(discovered): + print(f"tool-{i}: {t.name}") + return 0 + finally: + adapter.disconnect() def _invoke_tool() -> int: - tools = [_mock_tool("create_issue")] - config = MCPServerConfig(name="test", transport="stdio", command="echo") - transport = MockMCPTransport( - tools=tools, invoke_results={"create_issue": {"id": 42}} - ) + config = _make_config() + transport = _make_transport() adapter = MCPToolAdapter(config=config, transport=transport) - adapter.connect() - adapter.discover_tools() - result = adapter.invoke("create_issue", {"title": "Bug"}) - print(f"success: {result.success}") - print(f"has-data: {bool(result.data)}") - return 0 + try: + adapter.connect() + adapter.discover_tools() + result = adapter.invoke("add", {"a": 3, "b": 4}) + print(f"success: {result.success}") + print(f"has-data: {bool(result.data)}") + return 0 + finally: + adapter.disconnect() def _register_tools() -> int: - tools = [_mock_tool("tool_a"), _mock_tool("tool_b")] - config = MCPServerConfig(name="test", transport="stdio", command="echo") - transport = MockMCPTransport(tools=tools) + config = _make_config() + transport = _make_transport() adapter = MCPToolAdapter(config=config, transport=transport) - adapter.connect() - registry = ToolRegistry() - names = adapter.register_tools(registry, namespace="mcp-test") - print(f"registered: {len(names)}") - if names and "/" in names[0]: - print(f"prefix: {names[0].split('/')[0]}/") - return 0 + try: + adapter.connect() + registry = ToolRegistry() + names = adapter.register_tools(registry, namespace="mcp-test") + print(f"registered: {len(names)}") + if names and "/" in names[0]: + print(f"prefix: {names[0].split('/')[0]}/") + return 0 + finally: + adapter.disconnect() def _reject_no_command() -> int: diff --git a/robot/helper_plan_generation.py b/robot/helper_plan_generation.py index 69ba17997..1abdff6ec 100644 --- a/robot/helper_plan_generation.py +++ b/robot/helper_plan_generation.py @@ -12,7 +12,7 @@ SRC_DIR = PROJECT_ROOT / "src" if str(SRC_DIR) not in sys.path: sys.path.insert(0, str(SRC_DIR)) -from langchain_community.llms import FakeListLLM # noqa: E402 +from langchain_anthropic import ChatAnthropic # noqa: E402 from cleveragents.agents.plan_generation import PlanGenerationGraph # noqa: E402 from cleveragents.domain.models.core import Context # noqa: E402 @@ -21,7 +21,8 @@ from cleveragents.domain.models.core import Context # noqa: E402 def run_context_summary() -> None: """Generate plan context metadata to validate Robot tests.""" - graph = PlanGenerationGraph(llm=FakeListLLM(responses=["test response"] * 3)) + llm = ChatAnthropic(model="claude-3-haiku-20240307") + graph = PlanGenerationGraph(llm=llm) with tempfile.NamedTemporaryFile(delete=False, suffix=".py") as tmp: tmp.write(b"def main():\n return True\n") tmp_path = Path(tmp.name) diff --git a/robot/helper_skill_refresh.py b/robot/helper_skill_refresh.py index f2a391202..b7b7a64b8 100644 --- a/robot/helper_skill_refresh.py +++ b/robot/helper_skill_refresh.py @@ -5,6 +5,9 @@ Exercises ``SkillRegistry.refresh()``, ``SkillRegistry.refresh_all()``, process so Robot Framework can verify outputs without importing Python objects directly. +Uses a real ``StdioMCPTransport`` connecting to the Python-based +``mcp_echo_server.py`` fixture instead of mock transports. + Usage:: python robot/helper_skill_refresh.py refresh-single-ok @@ -26,22 +29,24 @@ import time from pathlib import Path from typing import Any -_ROOT = str(Path(__file__).resolve().parents[1]) _SRC = str(Path(__file__).resolve().parents[1] / "src") -for _p in (_SRC, _ROOT): - if _p not in sys.path: - sys.path.insert(0, _p) - -from features.mocks.mock_mcp_transport import MockMCPTransport # noqa: E402 +if _SRC not in sys.path: + sys.path.insert(0, _SRC) from cleveragents.domain.models.core.skill import Skill # noqa: E402 from cleveragents.mcp.adapter import MCPServerConfig, MCPToolAdapter # noqa: E402 from cleveragents.mcp.refresh_hook import MCPRefreshHook # noqa: E402 +from cleveragents.mcp.stdio_transport import StdioMCPTransport # noqa: E402 from cleveragents.skills.protocol import SkillDefinition, SkillMetadata # noqa: E402 from cleveragents.skills.refresh import SkillRefreshResult # noqa: E402 from cleveragents.skills.registry import SkillRegistry # noqa: E402 from cleveragents.tool.registry import ToolRegistry # noqa: E402 +# Path to the MCP echo test server fixture. +_FIXTURE_SERVER = str( + Path(__file__).resolve().parent / "fixtures" / "mcp_echo_server.py" +) + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -64,8 +69,19 @@ def _make_adapter( server_name: str = "test-server", tools: list[dict[str, Any]] | None = None, ) -> MCPToolAdapter: - config = MCPServerConfig(name=server_name, transport="stdio", command="echo") - transport = MockMCPTransport(tools=tools or []) + """Create an adapter connected to the real MCP echo fixture server. + + The ``tools`` parameter is accepted for API compatibility but is + ignored — the real server determines its own tool list. + """ + _ = tools # Not used with real transport. + config = MCPServerConfig( + name=server_name, + transport="stdio", + command=sys.executable, + args=[_FIXTURE_SERVER], + ) + transport = StdioMCPTransport(response_timeout=10.0) adapter = MCPToolAdapter(config=config, transport=transport) adapter.connect() return adapter @@ -208,83 +224,92 @@ def _result_merge() -> int: def _hook_wiring() -> int: - """MCPRefreshHook wires notification → refresh_all() correctly.""" + """MCPRefreshHook wires notification -> refresh_all() correctly.""" adapter = _make_adapter("hook-server") - skill_reg = SkillRegistry() - skill_reg.register(_make_skill("ns/hook-skill")) + try: + skill_reg = SkillRegistry() + skill_reg.register(_make_skill("ns/hook-skill")) - # Use zero debounce for deterministic test - hook = MCPRefreshHook( - adapter=adapter, skill_registry=skill_reg, debounce_seconds=0.0 - ) + # Use zero debounce for deterministic test + hook = MCPRefreshHook( + adapter=adapter, skill_registry=skill_reg, debounce_seconds=0.0 + ) - # Dispatch the notification - adapter.dispatch_notification("notifications/tools/list_changed") + # Dispatch the notification + adapter.dispatch_notification("notifications/tools/list_changed") - # Wait briefly for the daemon timer to fire - deadline = time.monotonic() + 3.0 - while hook.refresh_count == 0 and time.monotonic() < deadline: - time.sleep(0.05) + # Wait briefly for the daemon timer to fire + deadline = time.monotonic() + 3.0 + while hook.refresh_count == 0 and time.monotonic() < deadline: + time.sleep(0.05) - print(f"refresh-count: {hook.refresh_count}") - print(f"refreshed-at-least-once: {hook.refresh_count >= 1}") - hook.cancel() - print("hook-wiring-ok") - return 0 + print(f"refresh-count: {hook.refresh_count}") + print(f"refreshed-at-least-once: {hook.refresh_count >= 1}") + hook.cancel() + print("hook-wiring-ok") + return 0 + finally: + adapter.disconnect() def _hook_debounce() -> int: """Multiple notifications within debounce window collapse to one refresh.""" adapter = _make_adapter("debounce-server") - skill_reg = SkillRegistry() - skill_reg.register(_make_skill("ns/debounce-skill")) + try: + skill_reg = SkillRegistry() + skill_reg.register(_make_skill("ns/debounce-skill")) - hook = MCPRefreshHook( - adapter=adapter, skill_registry=skill_reg, debounce_seconds=0.1 - ) + hook = MCPRefreshHook( + adapter=adapter, skill_registry=skill_reg, debounce_seconds=0.1 + ) - # Fire 5 notifications in rapid succession - for _ in range(5): - adapter.dispatch_notification("notifications/tools/list_changed") - time.sleep(0.01) + # Fire 5 notifications in rapid succession + for _ in range(5): + adapter.dispatch_notification("notifications/tools/list_changed") + time.sleep(0.01) - # Wait for the single coalesced refresh to complete - deadline = time.monotonic() + 3.0 - while hook.refresh_count == 0 and time.monotonic() < deadline: - time.sleep(0.05) + # Wait for the single coalesced refresh to complete + deadline = time.monotonic() + 3.0 + while hook.refresh_count == 0 and time.monotonic() < deadline: + time.sleep(0.05) - # Allow a moment for any spurious second fire - time.sleep(0.3) + # Allow a moment for any spurious second fire + time.sleep(0.3) - print(f"refresh-count: {hook.refresh_count}") - print(f"debounced-to-one: {hook.refresh_count == 1}") - hook.cancel() - print("hook-debounce-ok") - return 0 + print(f"refresh-count: {hook.refresh_count}") + print(f"debounced-to-one: {hook.refresh_count == 1}") + hook.cancel() + print("hook-debounce-ok") + return 0 + finally: + adapter.disconnect() def _hook_cancel() -> int: """MCPRefreshHook.cancel() prevents the pending refresh from firing.""" adapter = _make_adapter("cancel-server") - skill_reg = SkillRegistry() - skill_reg.register(_make_skill("ns/cancel-skill")) + try: + skill_reg = SkillRegistry() + skill_reg.register(_make_skill("ns/cancel-skill")) - # Use a generous debounce so we can cancel before it fires - hook = MCPRefreshHook( - adapter=adapter, skill_registry=skill_reg, debounce_seconds=2.0 - ) - adapter.dispatch_notification("notifications/tools/list_changed") + # Use a generous debounce so we can cancel before it fires + hook = MCPRefreshHook( + adapter=adapter, skill_registry=skill_reg, debounce_seconds=2.0 + ) + adapter.dispatch_notification("notifications/tools/list_changed") - # Cancel before the timer fires - hook.cancel() + # Cancel before the timer fires + hook.cancel() - # Wait longer than the debounce — refresh should NOT have occurred - time.sleep(0.2) + # Wait longer than the debounce — refresh should NOT have occurred + time.sleep(0.2) - print(f"refresh-count-after-cancel: {hook.refresh_count}") - print(f"no-refresh-after-cancel: {hook.refresh_count == 0}") - print("hook-cancel-ok") - return 0 + print(f"refresh-count-after-cancel: {hook.refresh_count}") + print(f"no-refresh-after-cancel: {hook.refresh_count == 0}") + print("hook-cancel-ok") + return 0 + finally: + adapter.disconnect() # --------------------------------------------------------------------------- diff --git a/robot/helper_uko_indexer.py b/robot/helper_uko_indexer.py index 66e2a28f2..4ecb16231 100644 --- a/robot/helper_uko_indexer.py +++ b/robot/helper_uko_indexer.py @@ -4,13 +4,18 @@ Provides a CLI-style interface for Robot to invoke UKOIndexer creation, index lifecycle operations, graceful degradation, and provenance tracking. Exit code 0 = success, 1 = failure. +Uses a real ``LocationContentReader`` backed by temporary files instead +of an in-memory mock content reader. + Usage: python robot/helper_uko_indexer.py """ from __future__ import annotations +import shutil import sys +import tempfile from collections.abc import Callable from pathlib import Path @@ -44,20 +49,47 @@ from cleveragents.domain.models.acms.provenance import ( # noqa: E402 from cleveragents.domain.models.acms.python_analyzer import ( # noqa: E402 PythonAnalyzer, ) -from cleveragents.domain.models.core.resource import Resource # noqa: E402 - -# Ensure features/ is importable for shared mocks -_FEATURES = str(Path(__file__).resolve().parents[1]) -if _FEATURES not in sys.path: - sys.path.insert(0, _FEATURES) - -from features.mocks.uko_indexer_mocks import InMemoryContentReader # noqa: E402 +from cleveragents.domain.models.core.resource import PhysVirt, Resource # noqa: E402 ULID_1 = "01HQ8ZDRX50000000000000001" ULID_2 = "01HQ8ZDRX50000000000000002" PROJECT = "local/test" +class _TempContentManager: + """Manages temporary files for content used by ``LocationContentReader``. + + Writes content strings to real files in a temporary directory so the + ``LocationContentReader`` (which reads from the filesystem) can + resolve them. + """ + + def __init__(self) -> None: + self._tmpdir = Path(tempfile.mkdtemp(prefix="uko_test_")) + self._files: dict[str, Path] = {} + self._reader = LocationContentReader(base_dir=self._tmpdir) + + @property + def reader(self) -> LocationContentReader: + return self._reader + + def set_content(self, resource_id: str, content: str) -> str: + """Write *content* to a temp file and return the file path.""" + fname = f"{resource_id}.py" + fpath = self._tmpdir / fname + fpath.write_text(content, encoding="utf-8") + self._files[resource_id] = fpath + return str(fpath) + + def get_path(self, resource_id: str) -> str: + """Return the path for a previously written resource.""" + return str(self._files[resource_id]) + + def cleanup(self) -> None: + """Remove the temporary directory and all contents.""" + shutil.rmtree(self._tmpdir, ignore_errors=True) + + def _make_resource( resource_id: str, location: str = "src/example.py", @@ -67,7 +99,7 @@ def _make_resource( resource_id=resource_id, resource_type_name="git-checkout", location=location, - classification="physical", + classification=PhysVirt.PHYSICAL, ) @@ -77,7 +109,7 @@ def _make_indexer( vector: bool = True, ) -> tuple[ UKOIndexer, - InMemoryContentReader, + _TempContentManager, InMemoryTextIndexBackend | None, InMemoryVectorIndexBackend | None, InMemoryGraphIndexBackend, @@ -85,7 +117,7 @@ def _make_indexer( """Create a fully-wired UKOIndexer with in-memory backends.""" registry = AnalyzerRegistry() registry.register(PythonAnalyzer()) - reader = InMemoryContentReader() + content_mgr = _TempContentManager() graph = InMemoryGraphIndexBackend() text_be = InMemoryTextIndexBackend() if text else None vector_be = InMemoryVectorIndexBackend() if vector else None @@ -94,11 +126,11 @@ def _make_indexer( graph_backend=graph, text_backend=text_be, vector_backend=vector_be, - content_reader=reader, + content_reader=content_mgr.reader, ) return ( indexer, - reader, + content_mgr, text_be, vector_be, graph, @@ -107,11 +139,11 @@ def _make_indexer( def cmd_index_resource() -> int: """Test basic index_resource pipeline.""" + indexer, content_mgr, _text, _vec, _graph = _make_indexer() try: - indexer, reader, _text, _vec, _graph = _make_indexer() - resource = _make_resource(ULID_1) code = '"""Module doc."""\nclass Foo:\n """Class doc."""\n pass\n' - reader.set_content(ULID_1, code) + location = content_mgr.set_content(ULID_1, code) + resource = _make_resource(ULID_1, location=location) result = indexer.index_resource(resource, project=PROJECT) assert result.resource_id == ULID_1 @@ -127,14 +159,16 @@ def cmd_index_resource() -> int: except Exception as exc: print(f"uko-indexer-index-resource-fail: {exc}") return 1 + finally: + content_mgr.cleanup() def cmd_remove_resource() -> int: """Test remove_resource cleanup including backend state.""" + indexer, content_mgr, text_be, vec_be, graph_be = _make_indexer() try: - indexer, reader, text_be, vec_be, graph_be = _make_indexer() - resource = _make_resource(ULID_1) - reader.set_content(ULID_1, "x = 1\n") + location = content_mgr.set_content(ULID_1, "x = 1\n") + resource = _make_resource(ULID_1, location=location) indexer.index_resource(resource, project=PROJECT) assert indexer.indexed_resource_count == 1 @@ -156,19 +190,21 @@ def cmd_remove_resource() -> int: except Exception as exc: print(f"uko-indexer-remove-resource-fail: {exc}") return 1 + finally: + content_mgr.cleanup() def cmd_reindex_resource() -> int: """Test reindex_resource (remove + re-index).""" + indexer, content_mgr, _text, _vec, _graph = _make_indexer() try: - indexer, reader, _text, _vec, _graph = _make_indexer() - resource = _make_resource(ULID_1) - reader.set_content(ULID_1, "x = 1\n") + location = content_mgr.set_content(ULID_1, "x = 1\n") + resource = _make_resource(ULID_1, location=location) result1 = indexer.index_resource(resource, project=PROJECT) assert result1.triple_count > 0 # Update content and reindex - reader.set_content( + content_mgr.set_content( ULID_1, '"""Reindexed."""\nclass Bar:\n pass\n', ) @@ -182,14 +218,16 @@ def cmd_reindex_resource() -> int: except Exception as exc: print(f"uko-indexer-reindex-resource-fail: {exc}") return 1 + finally: + content_mgr.cleanup() def cmd_graceful_degradation() -> int: """Test graceful degradation without text/vector backends.""" + indexer, content_mgr, _, _, _graph = _make_indexer(text=False, vector=False) try: - indexer, reader, _, _, _graph = _make_indexer(text=False, vector=False) - resource = _make_resource(ULID_1) - reader.set_content(ULID_1, "y = 2\n") + location = content_mgr.set_content(ULID_1, "y = 2\n") + resource = _make_resource(ULID_1, location=location) result = indexer.index_resource(resource, project=PROJECT) assert result.triple_count > 0 @@ -204,6 +242,8 @@ def cmd_graceful_degradation() -> int: except Exception as exc: print(f"uko-indexer-graceful-degradation-fail: {exc}") return 1 + finally: + content_mgr.cleanup() def cmd_provenance() -> int: @@ -218,12 +258,15 @@ def cmd_provenance() -> int: assert prov.valid_from is not None # Verify provenance is attached during indexing via IndexResult - indexer, reader, _, _, _graph = _make_indexer() - resource = _make_resource(ULID_1) - reader.set_content(ULID_1, "z = 3\n") - result = indexer.index_resource(resource, project=PROJECT) - assert result.triple_count > 0 - assert result.resource_id == ULID_1 + indexer, content_mgr, _, _, _graph = _make_indexer() + try: + location = content_mgr.set_content(ULID_1, "z = 3\n") + resource = _make_resource(ULID_1, location=location) + result = indexer.index_resource(resource, project=PROJECT) + assert result.triple_count > 0 + assert result.resource_id == ULID_1 + finally: + content_mgr.cleanup() print("uko-indexer-provenance-ok") return 0 @@ -283,10 +326,10 @@ def cmd_index_backends() -> int: def cmd_validation() -> int: """Test input validation guards.""" + indexer, content_mgr, _, _, _ = _make_indexer() try: - indexer, reader, _, _, _ = _make_indexer() - resource = _make_resource(ULID_1) - reader.set_content(ULID_1, "a = 1\n") + location = content_mgr.set_content(ULID_1, "a = 1\n") + resource = _make_resource(ULID_1, location=location) # Empty project try: @@ -328,12 +371,15 @@ def cmd_validation() -> int: except Exception as exc: print(f"uko-indexer-validation-fail: {exc}") return 1 + finally: + content_mgr.cleanup() def cmd_protocol_compliance() -> int: """Verify protocol compliance for key types.""" + content_mgr = _TempContentManager() try: - reader = InMemoryContentReader() + reader = content_mgr.reader assert isinstance(reader, ContentReader) hook = DefaultLifecycleHook() @@ -356,6 +402,8 @@ def cmd_protocol_compliance() -> int: except Exception as exc: print(f"uko-indexer-protocol-compliance-fail: {exc}") return 1 + finally: + content_mgr.cleanup() def cmd_file_watching() -> int: diff --git a/robot/mcp_adapter.robot b/robot/mcp_adapter.robot index 60855f68e..6a229f3e6 100644 --- a/robot/mcp_adapter.robot +++ b/robot/mcp_adapter.robot @@ -19,12 +19,12 @@ Create MCP Adapter From Config Should Contain ${result.stdout} connected: False Discover MCP Tools - [Documentation] Connect and discover tools from a mock MCP server + [Documentation] Connect and discover tools from a real MCP server ${result}= Run Process ${PYTHON} ${HELPER} discover-tools cwd=${WORKSPACE} Log ${result.stdout} Should Be Equal As Integers ${result.rc} 0 - Should Contain ${result.stdout} discovered: 3 - Should Contain ${result.stdout} tool-0: create_issue + Should Contain ${result.stdout} discovered: 2 + Should Contain ${result.stdout} tool-0: echo Invoke MCP Tool [Documentation] Invoke a discovered tool and verify result diff --git a/robot/plan_generation_graph.robot b/robot/plan_generation_graph.robot index 70df08711..922cdf888 100644 --- a/robot/plan_generation_graph.robot +++ b/robot/plan_generation_graph.robot @@ -25,12 +25,13 @@ Plan Generation Graph Module Can Be Imported Plan Generation Graph Can Be Instantiated With Default Parameters [Documentation] Create PlanGenerationGraph with defaults + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... assert graph is not None ... assert graph.max_retries == 3 ... assert graph.llm is not None @@ -41,12 +42,13 @@ Plan Generation Graph Can Be Instantiated With Default Parameters Plan Generation Graph Can Be Instantiated With Custom Max Retries [Documentation] Create PlanGenerationGraph with custom max_retries + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=5) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307'), max_retries=5) ... assert graph.max_retries == 5 ... print('Max retries: ' + str(graph.max_retries)) ${result}= Run Process ${PYTHON} -c ${script} shell=True @@ -55,12 +57,13 @@ Plan Generation Graph Can Be Instantiated With Custom Max Retries Plan Generation Graph Creates Prompt Templates [Documentation] Verify prompt templates are created + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... assert hasattr(graph, 'analyze_prompt') ... assert hasattr(graph, 'generate_prompt') ... assert hasattr(graph, 'validate_prompt') @@ -73,12 +76,13 @@ Plan Generation Graph Creates Prompt Templates Plan Generation Graph Builds Workflow With Correct Nodes [Documentation] Verify workflow graph has correct nodes + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... nodes = graph.graph.nodes ... assert 'load_context' in nodes ... assert 'analyze_requirements' in nodes @@ -109,12 +113,13 @@ LangGraph Graphs Package Exports Workflow Classes Format Context Summary With No Files Returns Appropriate Message [Documentation] Test _format_context_summary with empty list + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... summary = graph._format_context_summary([]) ... assert summary == 'No context files provided' ... print('Empty context handled correctly') @@ -124,13 +129,14 @@ Format Context Summary With No Files Returns Appropriate Message Format Context Summary With Multiple Files [Documentation] Test _format_context_summary with multiple Context objects + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from cleveragents.domain.models.core import Context - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... contexts = [ ... Context(plan_id=1, path='file1.py', content='# File 1 content'), ... Context(plan_id=1, path='file2.py', content='# File 2 content'), @@ -146,13 +152,14 @@ Format Context Summary With Multiple Files Format Context Summary Limits To Five Files [Documentation] Test that _format_context_summary limits to first 5 files + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from cleveragents.domain.models.core import Context - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... contexts = [Context(plan_id=1, path=f'file{i}.py', content='content') for i in range(8)] ... summary = graph._format_context_summary(contexts) ... assert 'file0.py' in summary @@ -165,13 +172,14 @@ Format Context Summary Limits To Five Files Load Context Node Initializes State [Documentation] Test _load_context node execution + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from cleveragents.domain.models.core import Project, Plan - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... state = {'project': None, 'plan': None, 'contexts': []} ... result = graph._load_context(state) ... assert result['retry_count'] == 0 @@ -194,12 +202,13 @@ Load Context Node Generates Summary With Sample Contexts Should Retry Returns Retry When Validation Fails And Retries Available [Documentation] Test _should_retry returns "retry" appropriately + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=3) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307'), max_retries=3) ... state = { ... 'validation_result': {'status': 'FAIL'}, ... 'retry_count': 0 @@ -214,12 +223,13 @@ Should Retry Returns Retry When Validation Fails And Retries Available Should Retry Returns End When Validation Passes [Documentation] Test _should_retry returns "end" when validation succeeds + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=3) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307'), max_retries=3) ... state = { ... 'validation_result': {'status': 'PASS'}, ... 'retry_count': 0 @@ -233,12 +243,13 @@ Should Retry Returns End When Validation Passes Should Retry Returns End When Max Retries Reached [Documentation] Test _should_retry returns "end" when max retries reached + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=3) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307'), max_retries=3) ... state = { ... 'validation_result': {'status': 'FAIL'}, ... 'retry_count': 3 @@ -277,12 +288,13 @@ Plan Generation State TypedDict Has Correct Structure Validate Node Fails When No Changes Provided [Documentation] Test _validate with no changes + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... state = {'generated_changes': []} ... result = graph._validate(state) ... assert result['validation_result']['status'] == 'FAIL' @@ -294,12 +306,13 @@ Validate Node Fails When No Changes Provided Generate Plan Handles Missing Requirements [Documentation] Test _generate_plan with no requirements + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... from langchain_anthropic import ChatAnthropic + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... state = {'analyzed_requirements': {}} ... result = graph._generate_plan(state) ... assert result['generated_changes'] == [] @@ -311,14 +324,15 @@ Generate Plan Handles Missing Requirements Generate Plan Infers Test File Name From Prompt [Documentation] Test file name inference for test-related prompts + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from cleveragents.domain.models.core import Project, Plan - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... state = { ... 'project': Project(id=1, name='test', path=Path('/tmp/test_project')), ... 'plan': Plan(id=1, project_id=1, name='Unit Test Plan', prompt='Create unit tests'), @@ -340,14 +354,15 @@ Generate Plan Infers Test File Name From Prompt Generate Plan Infers Error Handler File Name From Prompt [Documentation] Test file name inference for error/exception prompts + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from cleveragents.domain.models.core import Project, Plan - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... state = { ... 'project': Project(id=1, name='test', path=Path('/tmp/test_project')), ... 'plan': Plan(id=1, project_id=1, name='Error Handling Plan', prompt='Add error handling'), @@ -368,14 +383,15 @@ Generate Plan Infers Error Handler File Name From Prompt Workflow Invoke Method Returns Complete State [Documentation] Test that invoke() returns complete workflow state + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from cleveragents.domain.models.core import Project, Plan, Context - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... project = Project(id=1, name='test_project', path=Path('/tmp/test_project')) ... plan = Plan(id=1, project_id=1, name='Logging Plan', prompt='Add logging') ... contexts = [Context(plan_id=plan.id, path='app.py', content='def main(): pass')] @@ -398,14 +414,15 @@ Workflow Invoke Method Returns Complete State Workflow Stream Method Yields Events [Documentation] Test that stream() yields workflow events + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... from pathlib import Path ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from cleveragents.domain.models.core import Project, Plan, Context - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... project = Project(id=1, name='test_project', path=Path('/tmp/test_project')) ... plan = Plan(id=1, project_id=1, name='Feature Plan', prompt='Add feature') ... contexts = [Context(plan_id=plan.id, path='app.py', content='# app')] @@ -420,13 +437,14 @@ Workflow Stream Method Yields Events Graph Has Checkpointer For State Persistence [Documentation] Verify checkpointer is configured + [Tags] llm-required ${script}= Catenate SEPARATOR=\n ... import sys ... sys.path.insert(0, '${SRC_DIR}') ... from cleveragents.agents.plan_generation import PlanGenerationGraph - ... from langchain_community.llms import FakeListLLM + ... from langchain_anthropic import ChatAnthropic ... from langgraph.checkpoint.memory import MemorySaver - ... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3)) + ... graph = PlanGenerationGraph(llm=ChatAnthropic(model='claude-3-haiku-20240307')) ... assert graph.checkpointer is not None ... assert isinstance(graph.checkpointer, MemorySaver) ... assert graph.app is not None diff --git a/src/cleveragents/application/services/plan_service.py b/src/cleveragents/application/services/plan_service.py index 478129af1..868c238ff 100644 --- a/src/cleveragents/application/services/plan_service.py +++ b/src/cleveragents/application/services/plan_service.py @@ -406,10 +406,16 @@ class PlanService: model_name = model_value or provider_name provider_instance_mutable = cast(Any, provider_instance) - if hasattr(provider_instance_mutable, "name"): - provider_instance_mutable.name = provider_name - if hasattr(provider_instance_mutable, "model_id"): - provider_instance_mutable.model_id = model_name + try: + if hasattr(provider_instance_mutable, "name"): + provider_instance_mutable.name = provider_name + except AttributeError: + pass # Read-only property on real provider implementations + try: + if hasattr(provider_instance_mutable, "model_id"): + provider_instance_mutable.model_id = model_name + except AttributeError: + pass # Read-only property on real provider implementations selection_metadata = self._provider_selection_metadata( actor=actor, diff --git a/src/cleveragents/mcp/__init__.py b/src/cleveragents/mcp/__init__.py index fa3550a17..3d2438096 100644 --- a/src/cleveragents/mcp/__init__.py +++ b/src/cleveragents/mcp/__init__.py @@ -12,6 +12,7 @@ from cleveragents.mcp.adapter import ( MCPToolResult, ) from cleveragents.mcp.refresh_hook import MCPRefreshHook +from cleveragents.mcp.stdio_transport import StdioMCPTransport __all__ = [ "MCPRefreshHook", @@ -19,4 +20,5 @@ __all__ = [ "MCPToolAdapter", "MCPToolDescriptor", "MCPToolResult", + "StdioMCPTransport", ] diff --git a/src/cleveragents/mcp/adapter.py b/src/cleveragents/mcp/adapter.py index 2b0905512..7bbf39f2c 100644 --- a/src/cleveragents/mcp/adapter.py +++ b/src/cleveragents/mcp/adapter.py @@ -455,9 +455,19 @@ class MCPToolAdapter: duration_ms=elapsed, ) + raw_content = result.get("content", result) + # MCP servers return content as a list of content blocks; + # normalise to a dict so MCPToolResult.data is always a dict. + if isinstance(raw_content, list): + data: dict[str, Any] = {"content": raw_content} + elif isinstance(raw_content, dict): + data = raw_content + else: + data = {"result": raw_content} + return MCPToolResult( success=True, - data=result.get("content", result), + data=data, duration_ms=elapsed, ) diff --git a/src/cleveragents/mcp/stdio_transport.py b/src/cleveragents/mcp/stdio_transport.py new file mode 100644 index 000000000..31b320a74 --- /dev/null +++ b/src/cleveragents/mcp/stdio_transport.py @@ -0,0 +1,314 @@ +"""Stdio-based MCP transport for communicating with MCP servers via subprocess. + +Implements the ``MCPTransport`` interface by spawning a child process +and exchanging newline-delimited JSON-RPC messages over stdin/stdout. + +Operations: + | Method | Description | + |--------------|---------------------------------------------------| + | ``connect`` | Spawn subprocess, perform MCP initialize handshake | + | ``call`` | Send JSON-RPC request, receive response | + | ``close`` | Terminate the subprocess gracefully | +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import select +import subprocess +import threading +from typing import Any + +from cleveragents.mcp.adapter import MCPServerConfig, MCPTransport + +logger = logging.getLogger(__name__) + +#: Default timeout in seconds for reading a JSON-RPC response. +DEFAULT_RESPONSE_TIMEOUT: float = 30.0 + +#: MCP protocol version for the initialize handshake. +MCP_PROTOCOL_VERSION = "2024-11-05" + + +class StdioMCPTransport(MCPTransport): + """MCP transport that communicates with a server via stdio subprocess. + + Spawns the server as a child process using the command and args from + ``MCPServerConfig``. Messages are exchanged as newline-delimited + JSON-RPC (NDJSON) over the subprocess's stdin and stdout streams. + + Parameters + ---------- + response_timeout: + Maximum seconds to wait for a JSON-RPC response before raising + ``TimeoutError``. Defaults to 30 s. + """ + + def __init__( + self, + *, + response_timeout: float = DEFAULT_RESPONSE_TIMEOUT, + ) -> None: + self._process: subprocess.Popen[bytes] | None = None + self._response_timeout = response_timeout + self._request_id = 0 + self._lock = threading.Lock() + self._read_lock = threading.Lock() + + def _next_id(self) -> int: + """Generate the next JSON-RPC request ID (thread-safe).""" + with self._lock: + self._request_id += 1 + return self._request_id + + def connect(self, config: MCPServerConfig) -> dict[str, Any]: + """Spawn the MCP server subprocess and perform the initialize handshake. + + Parameters + ---------- + config: + Server configuration with ``command`` and ``args``. + + Returns + ------- + dict[str, Any] + Server capabilities from the initialize response. + + Raises + ------ + ConnectionError + If the subprocess cannot be started or the handshake fails. + """ + if not config.command: + msg = f"MCPServerConfig '{config.name}': stdio transport requires 'command'" + raise ConnectionError(msg) + + cmd = [config.command, *config.args] + env = config.env or None + + try: + self._process = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + except (OSError, FileNotFoundError) as exc: + msg = f"Failed to spawn MCP server process: {exc}" + raise ConnectionError(msg) from exc + + # Perform the MCP initialize handshake. + try: + result = self._send_request( + "initialize", + { + "protocolVersion": MCP_PROTOCOL_VERSION, + "capabilities": {}, + "clientInfo": { + "name": "cleveragents", + "version": "1.0.0", + }, + }, + ) + except Exception as exc: + self._terminate_process() + msg = f"MCP initialize handshake failed: {exc}" + raise ConnectionError(msg) from exc + + # Send the initialized notification (no response expected). + self._send_notification("notifications/initialized", {}) + + capabilities = result.get("capabilities", {}) + logger.info( + "MCP stdio transport connected to '%s' (PID %d)", + config.name, + self._process.pid, + ) + return {"capabilities": capabilities} + + def call(self, method: str, params: dict[str, Any]) -> dict[str, Any]: + """Send a JSON-RPC request and return the result. + + Parameters + ---------- + method: + The JSON-RPC method name (e.g. ``tools/list``, ``tools/call``). + params: + Method parameters. + + Returns + ------- + dict[str, Any] + The ``result`` field from the JSON-RPC response. + + Raises + ------ + RuntimeError + If the transport is not connected. + TimeoutError + If no response is received within the timeout. + """ + if self._process is None or self._process.poll() is not None: + msg = "MCP transport is not connected" + raise RuntimeError(msg) + + return self._send_request(method, params) + + def close(self) -> None: + """Terminate the MCP server subprocess gracefully.""" + self._terminate_process() + + def _send_request( + self, + method: str, + params: dict[str, Any], + ) -> dict[str, Any]: + """Send a JSON-RPC request and read the response. + + Returns the ``result`` field from the response. + """ + request_id = self._next_id() + message = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + self._write_message(message) + return self._read_response(request_id) + + def _send_notification( + self, + method: str, + params: dict[str, Any], + ) -> None: + """Send a JSON-RPC notification (no response expected).""" + message = { + "jsonrpc": "2.0", + "method": method, + "params": params, + } + self._write_message(message) + + def _write_message(self, message: dict[str, Any]) -> None: + """Write a JSON-RPC message to the subprocess stdin.""" + proc = self._process + if proc is None or proc.stdin is None: + msg = "MCP transport stdin is not available" + raise RuntimeError(msg) + + data = json.dumps(message) + "\n" + try: + proc.stdin.write(data.encode("utf-8")) + proc.stdin.flush() + except (BrokenPipeError, OSError) as exc: + msg = f"Failed to write to MCP server stdin: {exc}" + raise RuntimeError(msg) from exc + + def _read_response(self, expected_id: int) -> dict[str, Any]: + """Read the next JSON-RPC response matching the expected ID. + + Skips notifications (messages without an ``id`` field) until the + response with the matching ``id`` is found. + """ + proc = self._process + if proc is None or proc.stdout is None: + msg = "MCP transport stdout is not available" + raise RuntimeError(msg) + + with self._read_lock: + deadline_remaining = self._response_timeout + while deadline_remaining > 0: + # Use select to implement timeout on stdout reads. + ready, _, _ = select.select( + [proc.stdout], [], [], min(deadline_remaining, 1.0) + ) + if not ready: + deadline_remaining -= 1.0 + # Check if process is still alive. + if proc.poll() is not None: + stderr_out = "" + if proc.stderr: + stderr_out = proc.stderr.read().decode( + "utf-8", errors="replace" + ) + msg = ( + f"MCP server process exited unexpectedly " + f"(code {proc.returncode}): {stderr_out}" + ) + raise RuntimeError(msg) + continue + + line = proc.stdout.readline() + if not line: + msg = "MCP server closed stdout unexpectedly" + raise RuntimeError(msg) + + line_str = line.decode("utf-8", errors="replace").strip() + if not line_str: + continue + + try: + response = json.loads(line_str) + except json.JSONDecodeError: + logger.debug("Ignoring non-JSON line from MCP server: %s", line_str) + continue + + # Skip notifications (no id field). + if "id" not in response: + logger.debug( + "Received MCP notification: %s", response.get("method") + ) + continue + + if response.get("id") != expected_id: + logger.warning( + "Unexpected response ID %s (expected %s)", + response.get("id"), + expected_id, + ) + continue + + # Check for JSON-RPC error. + if "error" in response: + error = response["error"] + msg = f"MCP server error: {error.get('message', error)}" + raise RuntimeError(msg) + + return response.get("result", {}) + + msg = ( + f"Timed out waiting for MCP response to request " + f"{expected_id} after {self._response_timeout}s" + ) + raise TimeoutError(msg) + + def _terminate_process(self) -> None: + """Terminate the subprocess, attempting graceful shutdown first.""" + proc = self._process + if proc is None: + return + + try: + if proc.poll() is None: + # Try to close stdin to signal the server to exit. + if proc.stdin: + with contextlib.suppress(OSError): + proc.stdin.close() + try: + proc.wait(timeout=5.0) + except subprocess.TimeoutExpired: + proc.terminate() + try: + proc.wait(timeout=3.0) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=2.0) + except Exception: + logger.warning("Error during MCP process cleanup", exc_info=True) + finally: + self._process = None + logger.info("MCP stdio transport process terminated")