"""Library for Skill package loading integration tests (ADR-2034). Writes a real Skill package YAML file to a real temp directory and resolves it through the real registry/package-resolution machinery (``LocalPackageStore``, ``PackageContentResolver``, ``SkillLoader``), wires it into a real ``LLMAgent`` via ``AgentFactory``, and drives a conversation through a mocked chat model — only the LLM API call itself is mocked, so no real network LLM calls happen here, matching every other ``*.robot`` suite in this project (``ToolCallingTestLib.py``, ``TokenBudgetTestLib.py``). Skill resolution, validation, config injection, and tool dispatch are all exercised for real, with no mocks anywhere in that path. """ from __future__ import annotations import asyncio import tempfile from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch from langchain_core.messages import AIMessage, SystemMessage, ToolMessage from cleveractors.agents.base import Agent from cleveractors.agents.factory import AgentFactory from cleveractors.agents.skill_resolution import SkillReferenceResolver from cleveractors.agents.skills import SkillLoader from cleveractors.core.exceptions import AgentCreationError from cleveractors.registry import LocalPackageStore from cleveractors.templates.renderer import TemplateRenderer class SkillLoadingTestLib: """Keywords for integration-style tests of Skill package loading.""" def __init__(self) -> None: self._tmp_dir: str | None = None self._factory: AgentFactory | None = None self._agent: Agent | None = None self._create_error: Exception | None = None self._mock_ainvoke_calls: list[dict[str, Any]] = [] self._patches: list[Any] = [] self._result: str | None = None def _teardown_patches(self) -> None: for p in self._patches: p.stop() self._patches.clear() def write_real_skill_package( self, filename: str = "pdf-processing.yaml", name: str = "pdf-processing", ) -> None: """Write a real, valid Skill package YAML file to a real temp directory.""" self._tmp_dir = tempfile.mkdtemp() file_path = Path(self._tmp_dir) / filename file_path.write_text( "skill: true\n" f"name: {name}\n" "description: Extract text and tables from PDFs.\n" "instructions: |\n" " Run scripts/extract.py to extract text.\n" "resources:\n" " references/REFERENCE.md:\n" " encoding: utf-8\n" " content: |\n" " # Reference\n" " Full CLI surface documented here.\n" ) def create_agent_factory_with_local_skill_store(self) -> None: """Create a real AgentFactory wired to resolve local: skill refs for real.""" assert self._tmp_dir is not None, "write_real_skill_package must run first" local_store = LocalPackageStore(base_dir=self._tmp_dir) skill_loader = SkillLoader( resolver=SkillReferenceResolver(local_store=local_store) ) self._factory = AgentFactory( config={}, template_renderer=TemplateRenderer(), skill_loader=skill_loader, ) def _install_mock_chat_model(self) -> None: self._teardown_patches() self._mock_ainvoke_calls = [] def _build_mock_model(*args: Any, **kwargs: Any) -> Any: mock_model = MagicMock() mock_model.temperature = 0.7 async def _mock_ainvoke(messages: Any, **invoke_kwargs: Any) -> AIMessage: self._mock_ainvoke_calls.append( {"kwargs": invoke_kwargs, "messages": list(messages)} ) call_count = len(self._mock_ainvoke_calls) if call_count == 1: return AIMessage( content="", usage_metadata={ "input_tokens": 40, "output_tokens": 10, "total_tokens": 50, }, tool_calls=[ { "id": "call_skill_001", "name": "skill", "args": {"skill_name": "pdf-processing"}, } ], ) return AIMessage( content="Final answer using skill instructions", usage_metadata={ "input_tokens": 20, "output_tokens": 8, "total_tokens": 28, }, ) mock_model.ainvoke = _mock_ainvoke return mock_model patcher = patch( "cleveractors.agents.llm.build_chat_model", side_effect=_build_mock_model ) patcher.start() self._patches.append(patcher) def create_llm_agent_with_skill( self, filename: str = "pdf-processing.yaml" ) -> None: """Create a real LLMAgent (via AgentFactory) configured with skills:.""" self._install_mock_chat_model() assert self._factory is not None, ( "create_agent_factory_with_local_skill_store must run first" ) try: self._agent = self._factory._create_agent_instance( # noqa: SLF001 "pdf_assistant", "llm", { "provider": "openai", "api_key": "mock-key", "model": "gpt-3.5-turbo", "system_prompt": "You help users work with PDF documents.", "skills": [f"local:{filename}"], }, ) self._create_error = None except AgentCreationError as exc: self._agent = None self._create_error = exc def create_llm_agent_with_missing_skill(self) -> None: """Attempt to create an LLMAgent referencing a skill file that does not exist.""" self._install_mock_chat_model() assert self._factory is not None, ( "create_agent_factory_with_local_skill_store must run first" ) try: self._agent = self._factory._create_agent_instance( # noqa: SLF001 "broken_assistant", "llm", { "provider": "openai", "api_key": "mock-key", "skills": ["local:does-not-exist.yaml"], }, ) self._create_error = None except AgentCreationError as exc: self._agent = None self._create_error = exc def process_message_with_agent(self, message: str) -> None: assert self._agent is not None, "No agent was created" try: self._result = asyncio.run(self._agent.process_message(message)) finally: self._teardown_patches() def agent_should_have_been_created(self) -> None: assert self._create_error is None, ( f"Agent creation failed: {self._create_error}" ) assert self._agent is not None def agent_creation_should_have_failed_with_agent_creation_error(self) -> None: assert self._create_error is not None, ( "Expected AgentCreationError but agent was created" ) assert isinstance(self._create_error, AgentCreationError), ( f"Expected AgentCreationError, got {type(self._create_error).__name__}" ) def agent_metadata_skills_loaded_should_equal(self, expected: int) -> None: assert self._agent is not None metadata = self._agent.get_metadata() actual = metadata.get("skills_loaded") assert actual == int(expected), f"skills_loaded = {actual}, expected {expected}" def agent_capabilities_should_include(self, capability: str) -> None: assert self._agent is not None capabilities = self._agent.get_capabilities() assert capability in capabilities, f"{capability!r} not in {capabilities!r}" def system_prompt_sent_to_model_should_contain(self, text: str) -> None: assert self._mock_ainvoke_calls, "chat_model.ainvoke was never called" first_call_messages = self._mock_ainvoke_calls[0]["messages"] system_texts = [ str(m.content) for m in first_call_messages if isinstance(m, SystemMessage) ] assert any(text in t for t in system_texts), ( f"{text!r} not found in system messages: {system_texts}" ) def tool_message_sent_to_model_should_contain(self, text: str) -> None: assert len(self._mock_ainvoke_calls) >= 2, ( "Expected at least 2 ainvoke calls (tool round + final answer)" ) second_call_messages = self._mock_ainvoke_calls[1]["messages"] tool_texts = [ str(m.content) for m in second_call_messages if isinstance(m, ToolMessage) ] assert any(text in t for t in tool_texts), ( f"{text!r} not found in tool messages sent back to the model: {tool_texts}" ) def model_should_have_been_offered_the_skill_tool(self) -> None: for call in self._mock_ainvoke_calls: for tool in call.get("kwargs", {}).get("tools") or []: if tool.get("function", {}).get("name") == "skill": return raise AssertionError("model was never invoked with the skill tool declared") def result_should_contain(self, text: str) -> None: assert self._result is not None assert text in self._result, f"Expected {text!r} in result: {self._result!r}"