feat(skill): add agent skills loader
Implemented AgentSkillSpec loader that parses SKILL.md frontmatter and progressive disclosure sections (discover/activate/deactivate) into structured SkillStep objects with stable 1-based ordering. Mapped Agent Skills to AgentSkillToolDescriptor with namespaced naming (namespace/short_name), source="agent_skill", read-only defaults, and AgentSkillResourceSlot bindings for scripts/, references/, and assets/ directories. All resource slots are unconditionally read_only. Added explicit validation for missing frontmatter fields (name, description) and invalid namespace format with actionable error messages. Added docs/reference/agent_skills.md covering folder layout, SKILL.md parsing rules, progressive disclosure model, and tool mapping. Added Behave scenarios covering valid/invalid SKILL.md parsing, namespaced naming, step ordering, missing frontmatter errors, progressive disclosure lifecycle, tool mapping, and resource binding slots. Added Robot Framework integration tests using the deploy-to-staging example skill folder (robot/agent_skills_loader.robot). Added ASV benchmarks for parsing throughput, folder load, progressive disclosure lifecycle, and resource listing (benchmarks/agent_skills_loader_bench.py). ISSUES CLOSED: #160
This commit is contained in:
@@ -0,0 +1,219 @@
|
||||
"""ASV benchmarks for Agent Skills Standard loader parsing throughput.
|
||||
|
||||
Measures the performance of:
|
||||
- SKILL.md string parsing (frontmatter + body)
|
||||
- Agent skills folder loading (from disk)
|
||||
- Progressive disclosure lifecycle (discover → activate → deactivate)
|
||||
- Tool descriptor conversion
|
||||
- Resource listing from multi-file folders
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.skills.agent_skills_loader import ( # noqa: E402
|
||||
AgentSkillLoader,
|
||||
AgentSkillSpec,
|
||||
)
|
||||
|
||||
_MINIMAL_SKILL_MD = """\
|
||||
---
|
||||
name: local/bench-minimal
|
||||
description: Minimal benchmark skill.
|
||||
---
|
||||
Step 1: Do this.
|
||||
"""
|
||||
|
||||
_FULL_SKILL_MD = """\
|
||||
---
|
||||
name: local/bench-full
|
||||
description: Full benchmark skill with all optional fields.
|
||||
version: 2.1.0
|
||||
steps:
|
||||
- Verify the environment is ready
|
||||
- Check that all required tools are available
|
||||
- Read the configuration from the reference files
|
||||
- Execute the main workflow script
|
||||
- Monitor progress and capture output
|
||||
- Handle any errors gracefully and log them
|
||||
- Release any acquired resources
|
||||
- Write the result summary to the output
|
||||
- Notify downstream systems if required
|
||||
compatibility:
|
||||
min_agent_version: "3.0.0"
|
||||
metadata:
|
||||
author: benchmark
|
||||
tags: [bench, performance]
|
||||
priority: high
|
||||
allowed-tools:
|
||||
- local/read-file
|
||||
- local/shell
|
||||
- local/git-status
|
||||
- local/search-files
|
||||
---
|
||||
# Full Benchmark Skill
|
||||
|
||||
This skill is used to benchmark the Agent Skills loader performance.
|
||||
|
||||
## Phase 1 — Preparation
|
||||
|
||||
1. Verify the environment is ready.
|
||||
2. Check that all required tools are available.
|
||||
3. Read the configuration from the reference files.
|
||||
|
||||
## Phase 2 — Execution
|
||||
|
||||
4. Execute the main workflow script.
|
||||
5. Monitor progress and capture output.
|
||||
6. Handle any errors gracefully and log them.
|
||||
|
||||
## Phase 3 — Cleanup
|
||||
|
||||
7. Release any acquired resources.
|
||||
8. Write the result summary to the output.
|
||||
9. Notify downstream systems if required.
|
||||
"""
|
||||
|
||||
|
||||
class AgentSkillSpecParsingSuite:
|
||||
"""Benchmark AgentSkillSpec string parsing throughput."""
|
||||
|
||||
def time_parse_minimal(self) -> None:
|
||||
"""Benchmark parsing a minimal SKILL.md string."""
|
||||
AgentSkillSpec.from_string(_MINIMAL_SKILL_MD)
|
||||
|
||||
def time_parse_full(self) -> None:
|
||||
"""Benchmark parsing a fully-populated SKILL.md string."""
|
||||
AgentSkillSpec.from_string(_FULL_SKILL_MD)
|
||||
|
||||
def time_parse_structured_steps(self) -> None:
|
||||
"""Benchmark parsing structured steps from frontmatter."""
|
||||
spec = AgentSkillSpec.from_string(_FULL_SKILL_MD)
|
||||
_ = spec.steps
|
||||
|
||||
|
||||
class AgentSkillFolderLoadSuite:
|
||||
"""Benchmark AgentSkillLoader.from_folder() throughput."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Create a temporary skill folder for folder-load benchmarks."""
|
||||
self._tmp = Path(tempfile.mkdtemp(prefix="asv_agent_skill_"))
|
||||
|
||||
# Write SKILL.md
|
||||
(self._tmp / "SKILL.md").write_text(_FULL_SKILL_MD)
|
||||
|
||||
# Write scripts/
|
||||
scripts = self._tmp / "scripts"
|
||||
scripts.mkdir()
|
||||
for i in range(5):
|
||||
(scripts / f"script_{i}.py").write_text(f"# Script {i}\nprint({i})\n")
|
||||
|
||||
# Write references/
|
||||
refs = self._tmp / "references"
|
||||
refs.mkdir()
|
||||
for i in range(3):
|
||||
(refs / f"ref_{i}.md").write_text(f"# Reference {i}\nContent {i}.\n")
|
||||
|
||||
# Write assets/
|
||||
assets = self._tmp / "assets"
|
||||
assets.mkdir()
|
||||
(assets / "config.json").write_text('{"key": "value"}\n')
|
||||
|
||||
def teardown(self) -> None:
|
||||
"""Remove the temporary skill folder."""
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(self._tmp, ignore_errors=True)
|
||||
|
||||
def time_load_from_folder(self) -> None:
|
||||
"""Benchmark loading a fully-populated agent skills folder."""
|
||||
AgentSkillLoader.from_folder(self._tmp)
|
||||
|
||||
|
||||
class AgentSkillLifecycleSuite:
|
||||
"""Benchmark the full progressive disclosure lifecycle."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Create a temporary skill folder for lifecycle benchmarks."""
|
||||
self._tmp = Path(tempfile.mkdtemp(prefix="asv_agent_skill_lc_"))
|
||||
(self._tmp / "SKILL.md").write_text(_FULL_SKILL_MD)
|
||||
scripts = self._tmp / "scripts"
|
||||
scripts.mkdir()
|
||||
(scripts / "run.py").write_text("print('run')\n")
|
||||
|
||||
def teardown(self) -> None:
|
||||
"""Remove the temporary skill folder."""
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(self._tmp, ignore_errors=True)
|
||||
|
||||
def time_discover(self) -> None:
|
||||
"""Benchmark discover() — Tier 1 metadata-only."""
|
||||
loader = AgentSkillLoader.from_folder(self._tmp)
|
||||
loader.discover()
|
||||
|
||||
def time_activate(self) -> None:
|
||||
"""Benchmark discover() + activate() — Tier 1 + Tier 2."""
|
||||
loader = AgentSkillLoader.from_folder(self._tmp)
|
||||
loader.discover()
|
||||
loader.activate()
|
||||
|
||||
def time_full_lifecycle(self) -> None:
|
||||
"""Benchmark discover → activate → list_resources → deactivate."""
|
||||
loader = AgentSkillLoader.from_folder(self._tmp)
|
||||
loader.discover()
|
||||
loader.activate()
|
||||
loader.list_resources()
|
||||
loader.deactivate()
|
||||
|
||||
def time_to_tool_descriptor(self) -> None:
|
||||
"""Benchmark conversion to AgentSkillToolDescriptor."""
|
||||
loader = AgentSkillLoader.from_folder(self._tmp)
|
||||
loader.to_tool_descriptor()
|
||||
|
||||
|
||||
class AgentSkillResourceListSuite:
|
||||
"""Benchmark list_resources() with varying resource counts."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Create a temporary folder with many resource files."""
|
||||
self._tmp = Path(tempfile.mkdtemp(prefix="asv_agent_skill_res_"))
|
||||
(self._tmp / "SKILL.md").write_text(_MINIMAL_SKILL_MD)
|
||||
|
||||
scripts = self._tmp / "scripts"
|
||||
scripts.mkdir()
|
||||
for i in range(20):
|
||||
(scripts / f"script_{i:02d}.py").write_text(f"# {i}\n")
|
||||
|
||||
refs = self._tmp / "references"
|
||||
refs.mkdir()
|
||||
for i in range(10):
|
||||
(refs / f"ref_{i:02d}.md").write_text(f"# {i}\n")
|
||||
|
||||
assets = self._tmp / "assets"
|
||||
assets.mkdir()
|
||||
for i in range(5):
|
||||
(assets / f"asset_{i:02d}.json").write_text(f'{{"id": {i}}}\n')
|
||||
|
||||
def teardown(self) -> None:
|
||||
"""Remove the temporary folder."""
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(self._tmp, ignore_errors=True)
|
||||
|
||||
def time_list_resources_35_files(self) -> None:
|
||||
"""Benchmark list_resources() with 35 resource files."""
|
||||
loader = AgentSkillLoader.from_folder(self._tmp)
|
||||
loader.list_resources()
|
||||
Reference in New Issue
Block a user