cleveragents-core/benchmarks/skill_flatten_bench.py

"""ASV benchmarks for Skill flattening and capability summaries.

Measures the performance of:
- FlattenSimple: Flatten a skill with 5 tool refs
- FlattenDeep: Flatten a skill with 3 levels of includes
- FlattenWide: Flatten a skill including 10 other skills
- ComputeSummary: Compute capability summary for 20 inline tools
"""

from __future__ import annotations

import sys
from pathlib import Path

try:
    from cleveragents.domain.models.core.skill import (
        Skill,
        SkillInclude,
        SkillInlineTool,
        SkillResolver,
    )
    from cleveragents.domain.models.core.tool import (
        ToolCapability,
        ToolSource,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.domain.models.core.skill import (
        Skill,
        SkillInclude,
        SkillInlineTool,
        SkillResolver,
    )
    from cleveragents.domain.models.core.tool import (
        ToolCapability,
        ToolSource,
    )


class FlattenSimpleSuite:
    """Benchmark flattening a skill with 5 named tool refs."""

    def setup(self) -> None:
        """Prepare skill and resolver."""
        self.skill = Skill(
            name="bench/simple",
            description="Simple benchmark skill",
            tool_refs=[f"bench/tool-{i}" for i in range(5)],
        )
        self.resolver = SkillResolver()

    def time_flatten_simple(self) -> None:
        """Benchmark resolve_tools for a skill with 5 refs."""
        self.resolver.resolve_tools(self.skill, {})


class FlattenDeepSuite:
    """Benchmark flattening a skill with 3 levels of includes."""

    def setup(self) -> None:
        """Build a 3-level include chain."""
        self.base = Skill(
            name="bench/deep-base",
            description="Base",
            tool_refs=["bench/base-tool"],
        )
        self.mid = Skill(
            name="bench/deep-mid",
            description="Mid",
            tool_refs=["bench/mid-tool"],
            includes=[SkillInclude(name="bench/deep-base")],
        )
        self.top = Skill(
            name="bench/deep-top",
            description="Top",
            tool_refs=["bench/top-tool"],
            includes=[SkillInclude(name="bench/deep-mid")],
        )
        self.registry = {
            "bench/deep-base": self.base,
            "bench/deep-mid": self.mid,
            "bench/deep-top": self.top,
        }
        self.resolver = SkillResolver()

    def time_flatten_deep(self) -> None:
        """Benchmark resolve_tools for a 3-level include chain."""
        self.resolver.resolve_tools(self.top, self.registry)


class FlattenWideSuite:
    """Benchmark flattening a skill including 10 other skills."""

    def setup(self) -> None:
        """Build a skill with 10 child includes."""
        self.registry: dict[str, Skill] = {}
        includes: list[SkillInclude] = []
        for i in range(10):
            child_name = f"bench/wide-child-{i}"
            self.registry[child_name] = Skill(
                name=child_name,
                description=f"Child {i}",
                tool_refs=[f"bench/wide-tool-{i}"],
            )
            includes.append(SkillInclude(name=child_name))
        self.top = Skill(
            name="bench/wide-top",
            description="Wide skill",
            includes=includes,
        )
        self.registry["bench/wide-top"] = self.top
        self.resolver = SkillResolver()

    def time_flatten_wide(self) -> None:
        """Benchmark resolve_tools for a wide include fan-out."""
        self.resolver.resolve_tools(self.top, self.registry)


class ComputeSummarySuite:
    """Benchmark compute_capability_summary for 20 inline tools."""

    def setup(self) -> None:
        """Create a skill with 20 inline tools of varying capabilities."""
        inlines: list[SkillInlineTool] = []
        for i in range(20):
            if i % 3 == 0:
                cap = ToolCapability(read_only=True)
            elif i % 3 == 1:
                cap = ToolCapability(writes=True, side_effects=["fs"])
            else:
                cap = ToolCapability(checkpointable=True)
            inlines.append(
                SkillInlineTool(
                    description=f"Tool {i}",
                    source=ToolSource.CUSTOM,
                    code=f"return {i}",
                    capability=cap,
                )
            )
        self.skill = Skill(
            name="bench/summary",
            description="Summary benchmark",
            anonymous_tools=inlines,
        )
        self.resolver = SkillResolver()
        self.resolved = self.resolver.resolve_tools(self.skill, {})

    def time_compute_summary(self) -> None:
        """Benchmark compute_capability_summary with 20 tools."""
        self.resolver.compute_capability_summary(self.skill, self.resolved)