"""ASV benchmarks for Skill flattening and capability summaries. Measures the performance of: - FlattenSimple: Flatten a skill with 5 tool refs - FlattenDeep: Flatten a skill with 3 levels of includes - FlattenWide: Flatten a skill including 10 other skills - ComputeSummary: Compute capability summary for 20 inline tools """ from __future__ import annotations import sys from pathlib import Path try: from cleveragents.domain.models.core.skill import ( Skill, SkillInclude, SkillInlineTool, SkillResolver, ) from cleveragents.domain.models.core.tool import ( ToolCapability, ToolSource, ) except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.domain.models.core.skill import ( Skill, SkillInclude, SkillInlineTool, SkillResolver, ) from cleveragents.domain.models.core.tool import ( ToolCapability, ToolSource, ) class FlattenSimpleSuite: """Benchmark flattening a skill with 5 named tool refs.""" def setup(self) -> None: """Prepare skill and resolver.""" self.skill = Skill( name="bench/simple", description="Simple benchmark skill", tool_refs=[f"bench/tool-{i}" for i in range(5)], ) self.resolver = SkillResolver() def time_flatten_simple(self) -> None: """Benchmark resolve_tools for a skill with 5 refs.""" self.resolver.resolve_tools(self.skill, {}) class FlattenDeepSuite: """Benchmark flattening a skill with 3 levels of includes.""" def setup(self) -> None: """Build a 3-level include chain.""" self.base = Skill( name="bench/deep-base", description="Base", tool_refs=["bench/base-tool"], ) self.mid = Skill( name="bench/deep-mid", description="Mid", tool_refs=["bench/mid-tool"], includes=[SkillInclude(name="bench/deep-base")], ) self.top = Skill( name="bench/deep-top", description="Top", tool_refs=["bench/top-tool"], includes=[SkillInclude(name="bench/deep-mid")], ) self.registry = { "bench/deep-base": self.base, "bench/deep-mid": self.mid, "bench/deep-top": self.top, } self.resolver = SkillResolver() def time_flatten_deep(self) -> None: """Benchmark resolve_tools for a 3-level include chain.""" self.resolver.resolve_tools(self.top, self.registry) class FlattenWideSuite: """Benchmark flattening a skill including 10 other skills.""" def setup(self) -> None: """Build a skill with 10 child includes.""" self.registry: dict[str, Skill] = {} includes: list[SkillInclude] = [] for i in range(10): child_name = f"bench/wide-child-{i}" self.registry[child_name] = Skill( name=child_name, description=f"Child {i}", tool_refs=[f"bench/wide-tool-{i}"], ) includes.append(SkillInclude(name=child_name)) self.top = Skill( name="bench/wide-top", description="Wide skill", includes=includes, ) self.registry["bench/wide-top"] = self.top self.resolver = SkillResolver() def time_flatten_wide(self) -> None: """Benchmark resolve_tools for a wide include fan-out.""" self.resolver.resolve_tools(self.top, self.registry) class ComputeSummarySuite: """Benchmark compute_capability_summary for 20 inline tools.""" def setup(self) -> None: """Create a skill with 20 inline tools of varying capabilities.""" inlines: list[SkillInlineTool] = [] for i in range(20): if i % 3 == 0: cap = ToolCapability(read_only=True) elif i % 3 == 1: cap = ToolCapability(writes=True, side_effects=["fs"]) else: cap = ToolCapability(checkpointable=True) inlines.append( SkillInlineTool( description=f"Tool {i}", source=ToolSource.CUSTOM, code=f"return {i}", capability=cap, ) ) self.skill = Skill( name="bench/summary", description="Summary benchmark", anonymous_tools=inlines, ) self.resolver = SkillResolver() self.resolved = self.resolver.resolve_tools(self.skill, {}) def time_compute_summary(self) -> None: """Benchmark compute_capability_summary with 20 tools.""" self.resolver.compute_capability_summary(self.skill, self.resolved)