cleveractors-core/robot/SkillLoadingTestLib.py

"""Library for Skill package loading integration tests (ADR-2034).

Writes a real Skill package YAML file to a real temp directory and
resolves it through the real registry/package-resolution machinery
(``LocalPackageStore``, ``PackageContentResolver``, ``SkillLoader``), wires
it into a real ``LLMAgent`` via ``AgentFactory``, and drives a conversation
through a mocked chat model — only the LLM API call itself is mocked, so no
real network LLM calls happen here, matching every other ``*.robot`` suite
in this project (``ToolCallingTestLib.py``, ``TokenBudgetTestLib.py``).
Skill resolution, validation, config injection, and tool dispatch are all
exercised for real, with no mocks anywhere in that path.
"""

from __future__ import annotations

import asyncio
import tempfile
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, patch

from langchain_core.messages import AIMessage, SystemMessage, ToolMessage

from cleveractors.agents.base import Agent
from cleveractors.agents.factory import AgentFactory
from cleveractors.agents.skill_resolution import SkillReferenceResolver
from cleveractors.agents.skills import SkillLoader
from cleveractors.core.exceptions import AgentCreationError
from cleveractors.registry import LocalPackageStore
from cleveractors.templates.renderer import TemplateRenderer


class SkillLoadingTestLib:
    """Keywords for integration-style tests of Skill package loading."""

    def __init__(self) -> None:
        self._tmp_dir: str | None = None
        self._factory: AgentFactory | None = None
        self._agent: Agent | None = None
        self._create_error: Exception | None = None
        self._mock_ainvoke_calls: list[dict[str, Any]] = []
        self._patches: list[Any] = []
        self._result: str | None = None

    def _teardown_patches(self) -> None:
        for p in self._patches:
            p.stop()
        self._patches.clear()

    def write_real_skill_package(
        self,
        filename: str = "pdf-processing.yaml",
        name: str = "pdf-processing",
    ) -> None:
        """Write a real, valid Skill package YAML file to a real temp directory."""
        self._tmp_dir = tempfile.mkdtemp()
        file_path = Path(self._tmp_dir) / filename
        file_path.write_text(
            "skill: true\n"
            f"name: {name}\n"
            "description: Extract text and tables from PDFs.\n"
            "instructions: |\n"
            "  Run scripts/extract.py <file.pdf> to extract text.\n"
            "resources:\n"
            "  references/REFERENCE.md:\n"
            "    encoding: utf-8\n"
            "    content: |\n"
            "      # Reference\n"
            "      Full CLI surface documented here.\n"
        )

    def create_agent_factory_with_local_skill_store(self) -> None:
        """Create a real AgentFactory wired to resolve local: skill refs for real."""
        assert self._tmp_dir is not None, "write_real_skill_package must run first"
        local_store = LocalPackageStore(base_dir=self._tmp_dir)
        skill_loader = SkillLoader(
            resolver=SkillReferenceResolver(local_store=local_store)
        )
        self._factory = AgentFactory(
            config={},
            template_renderer=TemplateRenderer(),
            skill_loader=skill_loader,
        )

    def _install_mock_chat_model(self) -> None:
        self._teardown_patches()
        self._mock_ainvoke_calls = []

        def _build_mock_model(*args: Any, **kwargs: Any) -> Any:
            mock_model = MagicMock()
            mock_model.temperature = 0.7

            async def _mock_ainvoke(messages: Any, **invoke_kwargs: Any) -> AIMessage:
                self._mock_ainvoke_calls.append(
                    {"kwargs": invoke_kwargs, "messages": list(messages)}
                )
                call_count = len(self._mock_ainvoke_calls)
                if call_count == 1:
                    return AIMessage(
                        content="",
                        usage_metadata={
                            "input_tokens": 40,
                            "output_tokens": 10,
                            "total_tokens": 50,
                        },
                        tool_calls=[
                            {
                                "id": "call_skill_001",
                                "name": "skill",
                                "args": {"skill_name": "pdf-processing"},
                            }
                        ],
                    )
                return AIMessage(
                    content="Final answer using skill instructions",
                    usage_metadata={
                        "input_tokens": 20,
                        "output_tokens": 8,
                        "total_tokens": 28,
                    },
                )

            mock_model.ainvoke = _mock_ainvoke
            return mock_model

        patcher = patch(
            "cleveractors.agents.llm.build_chat_model", side_effect=_build_mock_model
        )
        patcher.start()
        self._patches.append(patcher)

    def create_llm_agent_with_skill(
        self, filename: str = "pdf-processing.yaml"
    ) -> None:
        """Create a real LLMAgent (via AgentFactory) configured with skills:."""
        self._install_mock_chat_model()
        assert self._factory is not None, (
            "create_agent_factory_with_local_skill_store must run first"
        )
        try:
            self._agent = self._factory._create_agent_instance(  # noqa: SLF001
                "pdf_assistant",
                "llm",
                {
                    "provider": "openai",
                    "api_key": "mock-key",
                    "model": "gpt-3.5-turbo",
                    "system_prompt": "You help users work with PDF documents.",
                    "skills": [f"local:{filename}"],
                },
            )
            self._create_error = None
        except AgentCreationError as exc:
            self._agent = None
            self._create_error = exc

    def create_llm_agent_with_missing_skill(self) -> None:
        """Attempt to create an LLMAgent referencing a skill file that does not exist."""
        self._install_mock_chat_model()
        assert self._factory is not None, (
            "create_agent_factory_with_local_skill_store must run first"
        )
        try:
            self._agent = self._factory._create_agent_instance(  # noqa: SLF001
                "broken_assistant",
                "llm",
                {
                    "provider": "openai",
                    "api_key": "mock-key",
                    "skills": ["local:does-not-exist.yaml"],
                },
            )
            self._create_error = None
        except AgentCreationError as exc:
            self._agent = None
            self._create_error = exc

    def process_message_with_agent(self, message: str) -> None:
        assert self._agent is not None, "No agent was created"
        try:
            self._result = asyncio.run(self._agent.process_message(message))
        finally:
            self._teardown_patches()

    def agent_should_have_been_created(self) -> None:
        assert self._create_error is None, (
            f"Agent creation failed: {self._create_error}"
        )
        assert self._agent is not None

    def agent_creation_should_have_failed_with_agent_creation_error(self) -> None:
        assert self._create_error is not None, (
            "Expected AgentCreationError but agent was created"
        )
        assert isinstance(self._create_error, AgentCreationError), (
            f"Expected AgentCreationError, got {type(self._create_error).__name__}"
        )

    def agent_metadata_skills_loaded_should_equal(self, expected: int) -> None:
        assert self._agent is not None
        metadata = self._agent.get_metadata()
        actual = metadata.get("skills_loaded")
        assert actual == int(expected), f"skills_loaded = {actual}, expected {expected}"

    def agent_capabilities_should_include(self, capability: str) -> None:
        assert self._agent is not None
        capabilities = self._agent.get_capabilities()
        assert capability in capabilities, f"{capability!r} not in {capabilities!r}"

    def system_prompt_sent_to_model_should_contain(self, text: str) -> None:
        assert self._mock_ainvoke_calls, "chat_model.ainvoke was never called"
        first_call_messages = self._mock_ainvoke_calls[0]["messages"]
        system_texts = [
            str(m.content) for m in first_call_messages if isinstance(m, SystemMessage)
        ]
        assert any(text in t for t in system_texts), (
            f"{text!r} not found in system messages: {system_texts}"
        )

    def tool_message_sent_to_model_should_contain(self, text: str) -> None:
        assert len(self._mock_ainvoke_calls) >= 2, (
            "Expected at least 2 ainvoke calls (tool round + final answer)"
        )
        second_call_messages = self._mock_ainvoke_calls[1]["messages"]
        tool_texts = [
            str(m.content) for m in second_call_messages if isinstance(m, ToolMessage)
        ]
        assert any(text in t for t in tool_texts), (
            f"{text!r} not found in tool messages sent back to the model: {tool_texts}"
        )

    def model_should_have_been_offered_the_skill_tool(self) -> None:
        for call in self._mock_ainvoke_calls:
            for tool in call.get("kwargs", {}).get("tools") or []:
                if tool.get("function", {}).get("name") == "skill":
                    return
        raise AssertionError("model was never invoked with the skill tool declared")

    def result_should_contain(self, text: str) -> None:
        assert self._result is not None
        assert text in self._result, f"Expected {text!r} in result: {self._result!r}"