forked from cleveragents/cleveragents-core
151 lines
4.8 KiB
Python
151 lines
4.8 KiB
Python
"""ASV benchmarks for Skill flattening and capability summaries.
|
|
|
|
Measures the performance of:
|
|
- FlattenSimple: Flatten a skill with 5 tool refs
|
|
- FlattenDeep: Flatten a skill with 3 levels of includes
|
|
- FlattenWide: Flatten a skill including 10 other skills
|
|
- ComputeSummary: Compute capability summary for 20 inline tools
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.domain.models.core.skill import (
|
|
Skill,
|
|
SkillInclude,
|
|
SkillInlineTool,
|
|
SkillResolver,
|
|
)
|
|
from cleveragents.domain.models.core.tool import (
|
|
ToolCapability,
|
|
ToolSource,
|
|
)
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.domain.models.core.skill import (
|
|
Skill,
|
|
SkillInclude,
|
|
SkillInlineTool,
|
|
SkillResolver,
|
|
)
|
|
from cleveragents.domain.models.core.tool import (
|
|
ToolCapability,
|
|
ToolSource,
|
|
)
|
|
|
|
|
|
class FlattenSimpleSuite:
|
|
"""Benchmark flattening a skill with 5 named tool refs."""
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare skill and resolver."""
|
|
self.skill = Skill(
|
|
name="bench/simple",
|
|
description="Simple benchmark skill",
|
|
tool_refs=[f"bench/tool-{i}" for i in range(5)],
|
|
)
|
|
self.resolver = SkillResolver()
|
|
|
|
def time_flatten_simple(self) -> None:
|
|
"""Benchmark resolve_tools for a skill with 5 refs."""
|
|
self.resolver.resolve_tools(self.skill, {})
|
|
|
|
|
|
class FlattenDeepSuite:
|
|
"""Benchmark flattening a skill with 3 levels of includes."""
|
|
|
|
def setup(self) -> None:
|
|
"""Build a 3-level include chain."""
|
|
self.base = Skill(
|
|
name="bench/deep-base",
|
|
description="Base",
|
|
tool_refs=["bench/base-tool"],
|
|
)
|
|
self.mid = Skill(
|
|
name="bench/deep-mid",
|
|
description="Mid",
|
|
tool_refs=["bench/mid-tool"],
|
|
includes=[SkillInclude(name="bench/deep-base")],
|
|
)
|
|
self.top = Skill(
|
|
name="bench/deep-top",
|
|
description="Top",
|
|
tool_refs=["bench/top-tool"],
|
|
includes=[SkillInclude(name="bench/deep-mid")],
|
|
)
|
|
self.registry = {
|
|
"bench/deep-base": self.base,
|
|
"bench/deep-mid": self.mid,
|
|
"bench/deep-top": self.top,
|
|
}
|
|
self.resolver = SkillResolver()
|
|
|
|
def time_flatten_deep(self) -> None:
|
|
"""Benchmark resolve_tools for a 3-level include chain."""
|
|
self.resolver.resolve_tools(self.top, self.registry)
|
|
|
|
|
|
class FlattenWideSuite:
|
|
"""Benchmark flattening a skill including 10 other skills."""
|
|
|
|
def setup(self) -> None:
|
|
"""Build a skill with 10 child includes."""
|
|
self.registry: dict[str, Skill] = {}
|
|
includes: list[SkillInclude] = []
|
|
for i in range(10):
|
|
child_name = f"bench/wide-child-{i}"
|
|
self.registry[child_name] = Skill(
|
|
name=child_name,
|
|
description=f"Child {i}",
|
|
tool_refs=[f"bench/wide-tool-{i}"],
|
|
)
|
|
includes.append(SkillInclude(name=child_name))
|
|
self.top = Skill(
|
|
name="bench/wide-top",
|
|
description="Wide skill",
|
|
includes=includes,
|
|
)
|
|
self.registry["bench/wide-top"] = self.top
|
|
self.resolver = SkillResolver()
|
|
|
|
def time_flatten_wide(self) -> None:
|
|
"""Benchmark resolve_tools for a wide include fan-out."""
|
|
self.resolver.resolve_tools(self.top, self.registry)
|
|
|
|
|
|
class ComputeSummarySuite:
|
|
"""Benchmark compute_capability_summary for 20 inline tools."""
|
|
|
|
def setup(self) -> None:
|
|
"""Create a skill with 20 inline tools of varying capabilities."""
|
|
inlines: list[SkillInlineTool] = []
|
|
for i in range(20):
|
|
if i % 3 == 0:
|
|
cap = ToolCapability(read_only=True)
|
|
elif i % 3 == 1:
|
|
cap = ToolCapability(writes=True, side_effects=["fs"])
|
|
else:
|
|
cap = ToolCapability(checkpointable=True)
|
|
inlines.append(
|
|
SkillInlineTool(
|
|
description=f"Tool {i}",
|
|
source=ToolSource.CUSTOM,
|
|
code=f"return {i}",
|
|
capability=cap,
|
|
)
|
|
)
|
|
self.skill = Skill(
|
|
name="bench/summary",
|
|
description="Summary benchmark",
|
|
anonymous_tools=inlines,
|
|
)
|
|
self.resolver = SkillResolver()
|
|
self.resolved = self.resolver.resolve_tools(self.skill, {})
|
|
|
|
def time_compute_summary(self) -> None:
|
|
"""Benchmark compute_capability_summary with 20 tools."""
|
|
self.resolver.compute_capability_summary(self.skill, self.resolved)
|