aditya
6d7b759f25
CI / coverage (pull_request) Has been skipped
CI / docker (pull_request) Has been skipped
CI / lint (pull_request) Failing after 11s
CI / typecheck (pull_request) Successful in 28s
CI / security (pull_request) Successful in 25s
CI / quality (pull_request) Successful in 17s
CI / integration_tests (pull_request) Successful in 5m29s
CI / build (pull_request) Successful in 16s
CI / unit_tests (pull_request) Successful in 16m9s
133 lines
3.3 KiB
Python
133 lines
3.3 KiB
Python
"""ASV benchmarks for Skill YAML schema validation throughput.
|
|
|
|
Measures the performance of:
|
|
- YAML string parsing + schema validation
|
|
- YAML file loading + schema validation
|
|
- Key normalization overhead
|
|
- model_dump() serialization
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.skills.schema import SkillConfigSchema # noqa: E402
|
|
|
|
_MINIMAL_YAML = """\
|
|
name: local/bench-skill
|
|
"""
|
|
|
|
_FULL_YAML = """\
|
|
name: local/bench-full
|
|
description: "Full benchmark skill"
|
|
tools:
|
|
- name: builtin/read_file
|
|
- name: builtin/write_file
|
|
writes: true
|
|
inline_tools:
|
|
- name: bench_tool
|
|
description: "Benchmark inline tool"
|
|
source: custom
|
|
code: |
|
|
def run(input_data):
|
|
return {"ok": True}
|
|
input_schema:
|
|
type: object
|
|
properties:
|
|
param:
|
|
type: string
|
|
writes: false
|
|
checkpointable: false
|
|
side_effects: []
|
|
includes:
|
|
- name: local/file-reader
|
|
mcp_servers:
|
|
- name: github
|
|
transport: stdio
|
|
command: npx
|
|
args: ["-y", "@modelcontextprotocol/server-github"]
|
|
env:
|
|
TOKEN: "abc"
|
|
tool_filter:
|
|
include: [create_issue]
|
|
agent_skill_folders:
|
|
- path: ./skills/deploy
|
|
name: deploy
|
|
"""
|
|
|
|
_CAMEL_CASE_YAML = """\
|
|
name: local/bench-camel
|
|
inlineTools:
|
|
- name: camel_tool
|
|
source: custom
|
|
code: |
|
|
def run(input_data):
|
|
return {}
|
|
mcpServers:
|
|
- name: srv
|
|
transport: stdio
|
|
command: echo
|
|
"""
|
|
|
|
|
|
class SkillSchemaValidationSuite:
|
|
"""Benchmark SkillConfigSchema.from_yaml() throughput."""
|
|
|
|
def time_validate_minimal(self) -> None:
|
|
"""Benchmark minimal YAML validation."""
|
|
SkillConfigSchema.from_yaml(_MINIMAL_YAML)
|
|
|
|
def time_validate_full(self) -> None:
|
|
"""Benchmark fully-populated YAML validation."""
|
|
SkillConfigSchema.from_yaml(_FULL_YAML)
|
|
|
|
def time_validate_camel_case_normalization(self) -> None:
|
|
"""Benchmark YAML with camelCase key normalization."""
|
|
SkillConfigSchema.from_yaml(_CAMEL_CASE_YAML)
|
|
|
|
|
|
class SkillSchemaFileLoadSuite:
|
|
"""Benchmark SkillConfigSchema.from_yaml_file() throughput."""
|
|
|
|
def setup(self) -> None:
|
|
"""Write a temporary YAML file for file-load benchmarks."""
|
|
fd, self._path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(_FULL_YAML)
|
|
|
|
def teardown(self) -> None:
|
|
"""Remove the temporary file."""
|
|
Path(self._path).unlink(missing_ok=True)
|
|
|
|
def time_load_from_file(self) -> None:
|
|
"""Benchmark loading and validating a YAML file."""
|
|
SkillConfigSchema.from_yaml_file(self._path)
|
|
|
|
|
|
class SkillSchemaSerializationSuite:
|
|
"""Benchmark serialization of a validated SkillConfigSchema."""
|
|
|
|
def setup(self) -> None:
|
|
"""Parse YAML once for serialization benchmarks."""
|
|
self._config = SkillConfigSchema.from_yaml(_FULL_YAML)
|
|
|
|
def time_model_dump(self) -> None:
|
|
"""Benchmark model_dump() serialization."""
|
|
self._config.model_dump()
|
|
|
|
def time_model_dump_json(self) -> None:
|
|
"""Benchmark model_dump_json() JSON serialization."""
|
|
self._config.model_dump_json()
|