diff --git a/benchmarks/skill_schema_bench.py b/benchmarks/skill_schema_bench.py new file mode 100644 index 000000000..588f3a71e --- /dev/null +++ b/benchmarks/skill_schema_bench.py @@ -0,0 +1,133 @@ +"""ASV benchmarks for Skill YAML schema validation throughput. + +Measures the performance of: +- YAML string parsing + schema validation +- YAML file loading + schema validation +- Key normalization overhead +- model_dump() serialization +""" + +from __future__ import annotations + +import importlib +import os +import sys +import tempfile +from pathlib import Path + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.skills.schema import SkillConfigSchema # noqa: E402 + +_MINIMAL_YAML = """\ +name: local/bench-skill +""" + +_FULL_YAML = """\ +name: local/bench-full +description: "Full benchmark skill" +tools: + - name: builtin/read_file + - name: builtin/write_file + writes: true +inline_tools: + - name: bench_tool + description: "Benchmark inline tool" + source: custom + code: | + def run(input_data): + return {"ok": True} + input_schema: + type: object + properties: + param: + type: string + writes: false + checkpointable: false + side_effects: [] +includes: + - name: local/file-reader +mcp_servers: + - name: github + transport: stdio + command: npx + args: ["-y", "@modelcontextprotocol/server-github"] + env: + TOKEN: "abc" + tool_filter: + include: [create_issue] +agent_skill_folders: + - path: ./skills/deploy + name: deploy +""" + +_CAMEL_CASE_YAML = """\ +name: local/bench-camel +inlineTools: + - name: camel_tool + source: custom + code: | + def run(input_data): + return {} +mcpServers: + - name: srv + transport: stdio + command: echo +""" + + +class SkillSchemaValidationSuite: + """Benchmark SkillConfigSchema.from_yaml() throughput.""" + + def time_validate_minimal(self) -> None: + """Benchmark minimal YAML validation.""" + SkillConfigSchema.from_yaml(_MINIMAL_YAML) + + def time_validate_full(self) -> None: + """Benchmark fully-populated YAML validation.""" + SkillConfigSchema.from_yaml(_FULL_YAML) + + def time_validate_camel_case_normalization(self) -> None: + """Benchmark YAML with camelCase key normalization.""" + SkillConfigSchema.from_yaml(_CAMEL_CASE_YAML) + + +class SkillSchemaFileLoadSuite: + """Benchmark SkillConfigSchema.from_yaml_file() throughput.""" + + def setup(self) -> None: + """Write a temporary YAML file for file-load benchmarks.""" + fd, self._path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_FULL_YAML) + + def teardown(self) -> None: + """Remove the temporary file.""" + Path(self._path).unlink(missing_ok=True) + + def time_load_from_file(self) -> None: + """Benchmark loading and validating a YAML file.""" + SkillConfigSchema.from_yaml_file(self._path) + + +class SkillSchemaSerializationSuite: + """Benchmark serialization of a validated SkillConfigSchema.""" + + def setup(self) -> None: + """Parse YAML once for serialization benchmarks.""" + self._config = SkillConfigSchema.from_yaml(_FULL_YAML) + + def time_model_dump(self) -> None: + """Benchmark model_dump() serialization.""" + self._config.model_dump() + + def time_model_dump_json(self) -> None: + """Benchmark model_dump_json() JSON serialization.""" + self._config.model_dump_json() + diff --git a/docs/schema/skill.schema.yaml b/docs/schema/skill.schema.yaml new file mode 100644 index 000000000..843b519f1 --- /dev/null +++ b/docs/schema/skill.schema.yaml @@ -0,0 +1,245 @@ +# ============================================================================ +# CleverAgents Skill Configuration Schema +# ============================================================================ +# +# Formal YAML schema definition for skill configuration files. +# Skills are reusable, namespaced collections of tools registered via: +# agents skill add --config +# +# This schema is based on the formal JSON Schema defined in +# docs/specification.md (Section "Skill Configuration Files → JSON Schema", +# lines 15157-15362). +# +# Schema Version: 1 +# ============================================================================ + +schema_version: "1" + +# ---------------------------------------------------------------------------- +# Top-Level Fields +# ---------------------------------------------------------------------------- + +fields: + # ─── Identity ───────────────────────────────────────────────── + name: + type: string + required: true + pattern: "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$" + description: > + Fully qualified skill name in / format. + Examples: local/file-reader, myorg/devops-toolkit + + description: + type: string + required: false + description: > + Human-readable description of what this skill provides. + + # ─── Tool References ───────────────────────────────────────── + tools: + type: array + required: false + description: > + References to named tools from the Tool Registry. + items: + type: object + required_fields: [name] + additional_properties: false + fields: + name: + type: string + required: true + pattern: "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$" + description: "Fully qualified name of a tool registered in the Tool Registry." + description: + type: string + required: false + description: "Override the tool's registered description within this skill context." + writes: + type: boolean + required: false + description: "Override the tool's writes capability flag." + checkpointable: + type: boolean + required: false + description: "Override the tool's checkpointable capability flag." + + # ─── Inline (Anonymous) Tools ──────────────────────────────── + inline_tools: + type: array + required: false + description: > + Anonymous tool definitions that exist only within this skill. + Not registered in the Tool Registry. + items: + type: object + required_fields: [name, source, code] + additional_properties: false + fields: + name: + type: string + required: true + description: "Tool name, unique within this skill." + description: + type: string + required: false + description: "Human-readable description of the tool's purpose." + source: + type: string + required: true + const: custom + description: "Source type. Always 'custom' for inline tools." + code: + type: string + required: true + description: > + Python code defining the tool's behavior. + Must contain a run(input_data) function. + input_schema: + type: object + required: false + description: "JSON Schema describing the tool's input parameters." + writes: + type: boolean + required: false + default: false + description: "Whether this tool performs write operations." + checkpointable: + type: boolean + required: false + default: false + description: "Whether this tool supports checkpointing." + side_effects: + type: array + required: false + default: [] + items: + type: string + description: > + Descriptions of side effects (e.g., 'network_call', 'schema_mutation'). + + # ─── Included Skills ───────────────────────────────────────── + includes: + type: array + required: false + description: > + Other skills whose tools are merged into this skill. + items: + type: object + required_fields: [name] + additional_properties: false + fields: + name: + type: string + required: true + pattern: "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$" + description: "Fully qualified skill name to include." + description: + type: string + required: false + description: "Override the included skill's description." + + # ─── MCP Server Specifications ─────────────────────────────── + mcp_servers: + type: array + required: false + description: > + MCP server specifications for exposing remote tools. + items: + type: object + required_fields: [name, transport] + additional_properties: false + fields: + name: + type: string + required: true + description: "Identifier for the MCP server." + transport: + type: string + required: true + enum: [stdio, sse, streamable-http] + description: "Transport protocol." + command: + type: string + required: false + description: "Command to start the server (required for stdio transport)." + args: + type: array + required: false + items: + type: string + description: "Command-line arguments for the server command." + env: + type: object + required: false + additional_properties: + type: string + description: "Environment variables to set when starting the server." + url: + type: string + required: false + format: uri + description: "Server URL (required for sse and streamable-http transports)." + headers: + type: object + required: false + additional_properties: + type: string + description: "HTTP headers for remote server connections." + tool_filter: + type: object + required: false + additional_properties: false + fields: + include: + type: array + required: false + items: + type: string + description: "Whitelist of tool names to expose." + exclude: + type: array + required: false + items: + type: string + description: "Blacklist of tool names to hide." + + # ─── Agent Skills Standard Folders ─────────────────────────── + agent_skill_folders: + type: array + required: false + description: > + References to Agent Skills Standard (SKILL.md-based) tool bundles. + items: + type: object + required_fields: [path] + additional_properties: false + fields: + path: + type: string + required: true + description: "Path to the folder containing SKILL.md." + name: + type: string + required: false + description: "Override the skill bundle name." + +# ---------------------------------------------------------------------------- +# Constraints +# ---------------------------------------------------------------------------- + +required: [name] +additional_properties: false + +# ---------------------------------------------------------------------------- +# Key Normalization +# ---------------------------------------------------------------------------- +# The schema loader accepts camelCase keys and normalizes them to snake_case: +# inlineTools -> inline_tools +# mcpServers -> mcp_servers +# agentSkillFolders -> agent_skill_folders +# inputSchema -> input_schema +# sideEffects -> side_effects +# toolFilter -> tool_filter +# Legacy camelCase keys emit a warning but are accepted. + diff --git a/examples/skills/composed.yaml b/examples/skills/composed.yaml new file mode 100644 index 000000000..79d610d51 --- /dev/null +++ b/examples/skills/composed.yaml @@ -0,0 +1,31 @@ +# Composed skill example — includes other skills and adds own tools +# Register: agents skill add --config composed.yaml + +name: local/git-github +description: "Git operations and GitHub integration" + +tools: + - name: builtin/git_status + - name: builtin/git_diff + - name: builtin/git_log + - name: builtin/git_blame + +includes: + - name: local/file-reader + +mcp_servers: + - name: github + transport: stdio + command: npx + args: + - "-y" + - "@modelcontextprotocol/server-github" + env: + GITHUB_PERSONAL_ACCESS_TOKEN: "${GITHUB_TOKEN}" + tool_filter: + include: + - create_issue + - create_pull_request + - list_repos + - get_file_contents + diff --git a/examples/skills/inline-tool.yaml b/examples/skills/inline-tool.yaml new file mode 100644 index 000000000..579074138 --- /dev/null +++ b/examples/skills/inline-tool.yaml @@ -0,0 +1,37 @@ +# Inline-tool skill example — self-contained tools defined inline +# Register: agents skill add --config inline-tool.yaml + +name: local/text-processing +description: "Simple text transformation utilities" + +inline_tools: + - name: word_count + description: "Count words in text" + source: custom + code: | + def run(input_data): + text = input_data.get("text", "") + return {"count": len(text.split())} + input_schema: + type: object + properties: + text: + type: string + description: "Text to count words in" + required: ["text"] + writes: false + + - name: to_uppercase + description: "Convert text to uppercase" + source: custom + code: | + def run(input_data): + return {"result": input_data.get("text", "").upper()} + input_schema: + type: object + properties: + text: + type: string + required: ["text"] + writes: false + diff --git a/examples/skills/mcp-backed.yaml b/examples/skills/mcp-backed.yaml new file mode 100644 index 000000000..fb911db7b --- /dev/null +++ b/examples/skills/mcp-backed.yaml @@ -0,0 +1,23 @@ +# MCP-backed skill example — tools served via MCP server +# Register: agents skill add --config mcp-backed.yaml + +name: local/linear-tracker +description: "Linear issue tracking integration via MCP" + +mcp_servers: + - name: linear + transport: stdio + command: npx + args: ["-y", "@modelcontextprotocol/server-linear"] + env: + LINEAR_API_KEY: "${LINEAR_API_KEY}" + + - name: sentry + transport: sse + url: "https://mcp.sentry.dev/sse" + headers: + Authorization: "Bearer ${SENTRY_TOKEN}" + tool_filter: + exclude: + - delete_project + diff --git a/examples/skills/single-tool.yaml b/examples/skills/single-tool.yaml new file mode 100644 index 000000000..1df881db3 --- /dev/null +++ b/examples/skills/single-tool.yaml @@ -0,0 +1,11 @@ +# Single-tool skill example +# Register: agents skill add --config single-tool.yaml + +name: local/file-reader +description: "Basic file reading operations" + +tools: + - name: builtin/read_file + - name: builtin/list_directory + - name: builtin/search_files + diff --git a/examples/skills/validation-only.yaml b/examples/skills/validation-only.yaml new file mode 100644 index 000000000..1b4b79fd1 --- /dev/null +++ b/examples/skills/validation-only.yaml @@ -0,0 +1,5 @@ +# Minimal skill example — name-only (all other fields optional per spec) +# Register: agents skill add --config validation-only.yaml + +name: local/empty-skill + diff --git a/features/skill_schema.feature b/features/skill_schema.feature new file mode 100644 index 000000000..5704307c6 --- /dev/null +++ b/features/skill_schema.feature @@ -0,0 +1,266 @@ +Feature: Skill YAML schema validation + As a CleverAgents user + I want to define skills via YAML configuration files + So that I can create validated, reusable tool collections + + # ──────────────────────────────────────────────────────────── + # Valid schema scenarios + # ──────────────────────────────────────────────────────────── + + Scenario: Load a minimal valid skill YAML (name only) + Given a skill YAML string with only the name field + When I validate the skill schema + Then the skill schema validation should succeed + And the skill config name should be "local/empty-skill" + + Scenario: Load a skill YAML with tool references + Given a skill YAML string with tool references + When I validate the skill schema + Then the skill schema validation should succeed + And the skill config name should be "local/file-reader" + And the skill config should have 3 tools + + Scenario: Load a skill YAML with all sections populated + Given a skill YAML string with all sections populated + When I validate the skill schema + Then the skill schema validation should succeed + And the skill config name should be "local/full-skill" + And the skill config description should be "A fully populated skill" + And the skill config should have 2 tools + And the skill config should have 1 inline tools + And the skill config should have 1 includes + And the skill config should have 1 mcp servers + And the skill config should have 1 agent skill folders + + Scenario: Load the single-tool example YAML + Given the skill YAML file "examples/skills/single-tool.yaml" + When I validate the skill schema from file + Then the skill schema validation should succeed + + Scenario: Load the composed example YAML + Given the skill YAML file "examples/skills/composed.yaml" + When I validate the skill schema from file + Then the skill schema validation should succeed + + Scenario: Load the inline-tool example YAML + Given the skill YAML file "examples/skills/inline-tool.yaml" + When I validate the skill schema from file + Then the skill schema validation should succeed + + Scenario: Load the validation-only example YAML + Given the skill YAML file "examples/skills/validation-only.yaml" + When I validate the skill schema from file + Then the skill schema validation should succeed + + Scenario: Load the mcp-backed example YAML + Given the skill YAML file "examples/skills/mcp-backed.yaml" + When I validate the skill schema from file + Then the skill schema validation should succeed + + Scenario: Optional fields have correct defaults when omitted + Given a skill YAML string with only the name field + When I validate the skill schema + Then the skill schema validation should succeed + And the skill config tools list should be empty + And the skill config inline_tools list should be empty + And the skill config includes list should be empty + And the skill config mcp_servers list should be empty + And the skill config agent_skill_folders list should be empty + + Scenario: Tool reference with override fields + Given a skill YAML string with tool overrides + When I validate the skill schema + Then the skill schema validation should succeed + And skill tool 0 name should be "builtin/shell_execute" + And skill tool 0 description should be "Execute commands in sandbox" + And skill tool 0 writes should be true + And skill tool 0 checkpointable should be false + + Scenario: Inline tool with all fields + Given a skill YAML string with a full inline tool + When I validate the skill schema + Then the skill schema validation should succeed + And skill inline tool 0 name should be "run_migrations" + And skill inline tool 0 source should be "custom" + And skill inline tool 0 writes should be true + And skill inline tool 0 checkpointable should be true + + Scenario: MCP server with stdio transport + Given a skill YAML string with a stdio MCP server + When I validate the skill schema + Then the skill schema validation should succeed + And skill mcp server 0 name should be "github" + And skill mcp server 0 transport should be "stdio" + And skill mcp server 0 command should be "npx" + + Scenario: MCP server with SSE transport + Given a skill YAML string with an SSE MCP server + When I validate the skill schema + Then the skill schema validation should succeed + And skill mcp server 0 transport should be "sse" + And skill mcp server 0 url should be "https://mcp.example.com/sse" + + Scenario: MCP server with tool filter + Given a skill YAML string with MCP tool filter + When I validate the skill schema + Then the skill schema validation should succeed + And skill mcp server 0 tool filter include should have 2 items + And skill mcp server 0 tool filter exclude should have 1 items + + Scenario: Agent skill folder entries + Given a skill YAML string with agent skill folders + When I validate the skill schema + Then the skill schema validation should succeed + And the skill config should have 2 agent skill folders + And skill agent folder 0 path should be "./skills/deploy" + And skill agent folder 1 name should be "code-review" + + # ──────────────────────────────────────────────────────────── + # Invalid schema scenarios + # ──────────────────────────────────────────────────────────── + + Scenario: Missing name field produces a clear error + Given a skill YAML string missing the name field + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "name" + + Scenario: Invalid namespaced name without slash + Given a skill YAML string with name "invalid-name-no-slash" + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "namespace/name" + + Scenario: Invalid namespaced name with special characters + Given a skill YAML string with name "local/bad name!!" + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "namespace/name" + + Scenario: Invalid tool reference name + Given a skill YAML string with an invalid tool ref name "no-slash-tool" + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "namespace/name" + + Scenario: Inline tool missing source field + Given a skill YAML string with an inline tool missing source + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "source" + + Scenario: Inline tool missing code field + Given a skill YAML string with an inline tool missing code + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "code" + + Scenario: Inline tool with invalid source value + Given a skill YAML string with an inline tool source "plugin" + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "custom" + + Scenario: Invalid include name + Given a skill YAML string with an invalid include name "bad-include" + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "namespace/name" + + Scenario: Invalid MCP transport value + Given a skill YAML string with MCP transport "grpc" + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "transport" + + Scenario: MCP server missing transport field + Given a skill YAML string with MCP server missing transport + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "transport" + + Scenario: Agent skill folder missing path + Given a skill YAML string with agent folder missing path + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "path" + + Scenario: Extra unknown field is rejected + Given a skill YAML string with an unknown top-level field + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "extra" + + Scenario: None YAML string raises ValueError + Given a None skill YAML string + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "None" + + Scenario: Empty YAML string raises ValueError + Given an empty skill YAML string + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "empty" + + Scenario: YAML that is a list instead of a mapping + Given a skill YAML string that is a list + When I validate the skill schema expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "mapping" + + Scenario: None file path raises ValueError + Given a None skill YAML file path + When I validate the skill schema from file expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "None" + + Scenario: Non-existent file raises FileNotFoundError + Given the skill YAML file "examples/skills/nonexistent.yaml" + When I validate the skill schema from file expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "not found" + + Scenario: Directory path raises ValueError + Given the skill YAML directory path "examples/skills" + When I validate the skill schema from file expecting failure + Then the skill schema validation should fail + And the skill schema error should mention "not a file" + + # ──────────────────────────────────────────────────────────── + # Key normalization scenarios + # ──────────────────────────────────────────────────────────── + + Scenario: camelCase keys are normalized to snake_case + Given a skill YAML string with camelCase keys + When I validate the skill schema + Then the skill schema validation should succeed + And the skill config should have 1 inline tools + And the skill config should have 1 mcp servers + + # ──────────────────────────────────────────────────────────── + # Environment variable interpolation + # ──────────────────────────────────────────────────────────── + + Scenario: Environment variable interpolation in YAML values + Given the environment variable "TEST_MCP_CMD" is set to "npx" + And a skill YAML string using env var "${TEST_MCP_CMD}" for mcp command + When I validate the skill schema + Then the skill schema validation should succeed + And skill mcp server 0 command should be "npx" + + # ──────────────────────────────────────────────────────────── + # Round-trip serialization + # ──────────────────────────────────────────────────────────── + + Scenario: Schema model serializes to dict correctly + Given a skill YAML string with all sections populated + When I validate the skill schema + Then the skill schema validation should succeed + And the skill config model_dump should contain key "name" + And the skill config model_dump should contain key "tools" + And the skill config model_dump should contain key "inline_tools" + And the skill config model_dump should contain key "includes" + And the skill config model_dump should contain key "mcp_servers" + And the skill config model_dump should contain key "agent_skill_folders" + diff --git a/features/steps/skill_schema_steps.py b/features/steps/skill_schema_steps.py new file mode 100644 index 000000000..f6fbaeefa --- /dev/null +++ b/features/steps/skill_schema_steps.py @@ -0,0 +1,662 @@ +"""Step definitions for skill YAML schema validation. + +Tests for features/skill_schema.feature — validates the SkillConfigSchema +Pydantic model, YAML loading, key normalization, environment variable +interpolation, and error messages. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from behave import given, then, when # type: ignore[import-untyped] +from behave.runner import Context # type: ignore[import-untyped] +from pydantic import ValidationError + +from cleveragents.skills.schema import SkillConfigSchema + +# ──────────────────────────────────────────────────────────── +# Helpers +# ──────────────────────────────────────────────────────────── + +_NAME_ONLY_YAML = """\ +name: local/empty-skill +""" + +_TOOLS_YAML = """\ +name: local/file-reader +description: "Basic file reading operations" +tools: + - name: builtin/read_file + - name: builtin/list_directory + - name: builtin/search_files +""" + +_FULL_YAML = """\ +name: local/full-skill +description: "A fully populated skill" +tools: + - name: builtin/read_file + - name: builtin/write_file + writes: true +inline_tools: + - name: helper_tool + description: "Inline helper" + source: custom + code: | + def run(input_data): + return {"ok": True} +includes: + - name: local/file-reader +mcp_servers: + - name: github + transport: stdio + command: npx + args: ["-y", "@modelcontextprotocol/server-github"] +agent_skill_folders: + - path: ./skills/deploy +""" + +_TOOL_OVERRIDES_YAML = """\ +name: local/tool-overrides +tools: + - name: builtin/shell_execute + description: "Execute commands in sandbox" + writes: true + checkpointable: false +""" + +_FULL_INLINE_TOOL_YAML = """\ +name: local/inline-full +inline_tools: + - name: run_migrations + description: "Run database migrations" + source: custom + code: | + def run(input_data): + return {"ok": True} + input_schema: + type: object + properties: + direction: + type: string + writes: true + checkpointable: true + side_effects: ["schema_mutation"] +""" + +_STDIO_MCP_YAML = """\ +name: local/mcp-stdio +mcp_servers: + - name: github + transport: stdio + command: npx + args: ["-y", "@modelcontextprotocol/server-github"] +""" + +_SSE_MCP_YAML = """\ +name: local/mcp-sse +mcp_servers: + - name: remote-server + transport: sse + url: "https://mcp.example.com/sse" + headers: + Authorization: "Bearer token" +""" + +_MCP_FILTER_YAML = """\ +name: local/mcp-filter +mcp_servers: + - name: filtered + transport: stdio + command: npx + tool_filter: + include: + - tool_a + - tool_b + exclude: + - tool_c +""" + +_AGENT_FOLDERS_YAML = """\ +name: local/agent-folders +agent_skill_folders: + - path: ./skills/deploy + - path: ./skills/code-review-bundle + name: code-review +""" + +_CAMEL_CASE_YAML = """\ +name: local/camel-skill +inlineTools: + - name: my_tool + source: custom + code: | + def run(input_data): + return {} +mcpServers: + - name: srv + transport: stdio + command: echo +""" + + +# ──────────────────────────────────────────────────────────── +# Given steps +# ──────────────────────────────────────────────────────────── + + +@given("a skill YAML string with only the name field") +def step_given_name_only_yaml(context: Context) -> None: + """Provide a minimal skill YAML with only name.""" + context.skill_yaml_string = _NAME_ONLY_YAML + + +@given("a skill YAML string with tool references") +def step_given_tools_yaml(context: Context) -> None: + """Provide a skill YAML with tool references.""" + context.skill_yaml_string = _TOOLS_YAML + + +@given("a skill YAML string with all sections populated") +def step_given_full_yaml(context: Context) -> None: + """Provide a fully populated skill YAML.""" + context.skill_yaml_string = _FULL_YAML + + +@given('the skill YAML file "{rel_path}"') +def step_given_skill_yaml_file(context: Context, rel_path: str) -> None: + """Set the path to a skill YAML file relative to project root.""" + project_root = Path(__file__).resolve().parents[2] + context.skill_yaml_file = str(project_root / rel_path) + + +@given("a skill YAML string with tool overrides") +def step_given_tool_overrides_yaml(context: Context) -> None: + """Provide a skill YAML with tool reference overrides.""" + context.skill_yaml_string = _TOOL_OVERRIDES_YAML + + +@given("a skill YAML string with a full inline tool") +def step_given_full_inline_tool_yaml(context: Context) -> None: + """Provide a skill YAML with a fully populated inline tool.""" + context.skill_yaml_string = _FULL_INLINE_TOOL_YAML + + +@given("a skill YAML string with a stdio MCP server") +def step_given_stdio_mcp_yaml(context: Context) -> None: + """Provide a skill YAML with a stdio MCP server.""" + context.skill_yaml_string = _STDIO_MCP_YAML + + +@given("a skill YAML string with an SSE MCP server") +def step_given_sse_mcp_yaml(context: Context) -> None: + """Provide a skill YAML with an SSE MCP server.""" + context.skill_yaml_string = _SSE_MCP_YAML + + +@given("a skill YAML string with MCP tool filter") +def step_given_mcp_filter_yaml(context: Context) -> None: + """Provide a skill YAML with MCP tool filter.""" + context.skill_yaml_string = _MCP_FILTER_YAML + + +@given("a skill YAML string with agent skill folders") +def step_given_agent_folders_yaml(context: Context) -> None: + """Provide a skill YAML with agent skill folders.""" + context.skill_yaml_string = _AGENT_FOLDERS_YAML + + +@given("a skill YAML string missing the name field") +def step_given_missing_name_yaml(context: Context) -> None: + """Provide a skill YAML without the name field.""" + context.skill_yaml_string = "description: Missing the name\n" + + +@given('a skill YAML string with name "{name}"') +def step_given_yaml_with_name(context: Context, name: str) -> None: + """Provide a skill YAML with a custom name value.""" + context.skill_yaml_string = f"name: {name}\n" + + +@given('a skill YAML string with an invalid tool ref name "{name}"') +def step_given_invalid_tool_ref(context: Context, name: str) -> None: + """Provide a skill YAML with an invalid tool reference name.""" + context.skill_yaml_string = f"""\ +name: local/bad-ref +tools: + - name: {name} +""" + + +@given("a skill YAML string with an inline tool missing source") +def step_given_inline_missing_source(context: Context) -> None: + """Provide a skill YAML with an inline tool missing the source field.""" + context.skill_yaml_string = """\ +name: local/bad-inline +inline_tools: + - name: my_tool + code: | + def run(input_data): + return {} +""" + + +@given("a skill YAML string with an inline tool missing code") +def step_given_inline_missing_code(context: Context) -> None: + """Provide a skill YAML with an inline tool missing the code field.""" + context.skill_yaml_string = """\ +name: local/bad-inline +inline_tools: + - name: my_tool + source: custom +""" + + +@given('a skill YAML string with an inline tool source "{source}"') +def step_given_inline_invalid_source(context: Context, source: str) -> None: + """Provide a skill YAML with an invalid inline tool source.""" + context.skill_yaml_string = f"""\ +name: local/bad-source +inline_tools: + - name: my_tool + source: {source} + code: | + def run(input_data): + return {{}} +""" + + +@given('a skill YAML string with an invalid include name "{name}"') +def step_given_invalid_include(context: Context, name: str) -> None: + """Provide a skill YAML with an invalid include name.""" + context.skill_yaml_string = f"""\ +name: local/bad-include +includes: + - name: {name} +""" + + +@given('a skill YAML string with MCP transport "{transport}"') +def step_given_invalid_mcp_transport(context: Context, transport: str) -> None: + """Provide a skill YAML with an invalid MCP transport.""" + context.skill_yaml_string = f"""\ +name: local/bad-mcp +mcp_servers: + - name: srv + transport: {transport} +""" + + +@given("a skill YAML string with MCP server missing transport") +def step_given_mcp_missing_transport(context: Context) -> None: + """Provide a skill YAML with an MCP server missing transport.""" + context.skill_yaml_string = """\ +name: local/bad-mcp +mcp_servers: + - name: srv +""" + + +@given("a skill YAML string with agent folder missing path") +def step_given_agent_folder_missing_path(context: Context) -> None: + """Provide a skill YAML with an agent folder missing path.""" + context.skill_yaml_string = """\ +name: local/bad-folder +agent_skill_folders: + - name: missing-path-folder +""" + + +@given("a skill YAML string with an unknown top-level field") +def step_given_unknown_field(context: Context) -> None: + """Provide a skill YAML with an extra unknown field.""" + context.skill_yaml_string = """\ +name: local/extra-field +unknown_field: should fail +""" + + +@given("a None skill YAML string") +def step_given_none_yaml(context: Context) -> None: + """Set YAML string to None.""" + context.skill_yaml_string = None # type: ignore[assignment] + + +@given("an empty skill YAML string") +def step_given_empty_yaml(context: Context) -> None: + """Set YAML string to empty.""" + context.skill_yaml_string = "" + + +@given("a skill YAML string that is a list") +def step_given_yaml_list(context: Context) -> None: + """Set YAML string to a list instead of a mapping.""" + context.skill_yaml_string = "- item1\n- item2\n" + + +@given("a None skill YAML file path") +def step_given_none_file_path(context: Context) -> None: + """Set file path to None.""" + context.skill_yaml_file = None # type: ignore[assignment] + + +@given('the skill YAML directory path "{dir_path}"') +def step_given_yaml_directory_path(context: Context, dir_path: str) -> None: + """Set file path to a directory.""" + project_root = Path(__file__).resolve().parents[2] + context.skill_yaml_file = str(project_root / dir_path) + + +@given("a skill YAML string with camelCase keys") +def step_given_camel_case_yaml(context: Context) -> None: + """Provide a skill YAML with camelCase keys.""" + context.skill_yaml_string = _CAMEL_CASE_YAML + + +@given('a skill YAML string using env var "${{{var}}}" for mcp command') +def step_given_yaml_with_env_var(context: Context, var: str) -> None: + """Provide a skill YAML with an environment variable reference in MCP.""" + context.skill_yaml_string = f"""\ +name: local/env-skill +mcp_servers: + - name: srv + transport: stdio + command: ${{{var}}} +""" + + +# ──────────────────────────────────────────────────────────── +# When steps +# ──────────────────────────────────────────────────────────── + + +@when("I validate the skill schema") +def step_when_validate_skill_schema(context: Context) -> None: + """Validate the skill YAML string via SkillConfigSchema.""" + try: + context.skill_config = SkillConfigSchema.from_yaml(context.skill_yaml_string) + context.skill_schema_error = None + except (ValidationError, ValueError) as exc: + context.skill_config = None + context.skill_schema_error = str(exc) + + +@when("I validate the skill schema from file") +def step_when_validate_from_file(context: Context) -> None: + """Validate a skill YAML file via SkillConfigSchema.""" + try: + context.skill_config = SkillConfigSchema.from_yaml_file( + context.skill_yaml_file + ) + context.skill_schema_error = None + except (ValidationError, ValueError, FileNotFoundError) as exc: + context.skill_config = None + context.skill_schema_error = str(exc) + + +@when("I validate the skill schema expecting failure") +def step_when_validate_expecting_failure(context: Context) -> None: + """Validate the skill YAML, expecting it to fail.""" + try: + context.skill_config = SkillConfigSchema.from_yaml(context.skill_yaml_string) + context.skill_schema_error = None + except (ValidationError, ValueError) as exc: + context.skill_config = None + context.skill_schema_error = str(exc) + + +@when("I validate the skill schema from file expecting failure") +def step_when_validate_file_expecting_failure(context: Context) -> None: + """Validate a skill YAML file, expecting it to fail.""" + try: + context.skill_config = SkillConfigSchema.from_yaml_file( + context.skill_yaml_file + ) + context.skill_schema_error = None + except (ValidationError, ValueError, FileNotFoundError, TypeError) as exc: + context.skill_config = None + context.skill_schema_error = str(exc) + + +# ──────────────────────────────────────────────────────────── +# Then steps +# ──────────────────────────────────────────────────────────── + + +@then("the skill schema validation should succeed") +def step_then_validation_succeeds(context: Context) -> None: + """Assert that schema validation succeeded.""" + assert context.skill_config is not None, ( + f"Expected validation to succeed but got error: {context.skill_schema_error}" + ) + + +@then("the skill schema validation should fail") +def step_then_validation_fails(context: Context) -> None: + """Assert that schema validation failed.""" + assert context.skill_config is None, "Expected validation to fail but it succeeded" + assert context.skill_schema_error is not None + + +@then('the skill schema error should mention "{text}"') +def step_then_error_mentions(context: Context, text: str) -> None: + """Assert the error message contains expected text.""" + assert context.skill_schema_error is not None, "No error was captured" + assert text.lower() in context.skill_schema_error.lower(), ( + f"Expected error to mention '{text}', got: {context.skill_schema_error}" + ) + + +@then('the skill config name should be "{expected}"') +def step_then_config_name(context: Context, expected: str) -> None: + """Assert the parsed config name.""" + assert context.skill_config is not None + assert context.skill_config.name == expected + + +@then('the skill config description should be "{expected}"') +def step_then_config_description(context: Context, expected: str) -> None: + """Assert the parsed config description.""" + assert context.skill_config is not None + assert context.skill_config.description == expected + + +@then("the skill config should have {count:d} tools") +def step_then_tool_count(context: Context, count: int) -> None: + """Assert the number of tool references.""" + assert context.skill_config is not None + assert len(context.skill_config.tools) == count + + +@then("the skill config should have {count:d} inline tools") +def step_then_inline_tool_count(context: Context, count: int) -> None: + """Assert the number of inline tools.""" + assert context.skill_config is not None + assert len(context.skill_config.inline_tools) == count + + +@then("the skill config should have {count:d} includes") +def step_then_include_count(context: Context, count: int) -> None: + """Assert the number of includes.""" + assert context.skill_config is not None + assert len(context.skill_config.includes) == count + + +@then("the skill config should have {count:d} mcp servers") +def step_then_mcp_count(context: Context, count: int) -> None: + """Assert the number of MCP servers.""" + assert context.skill_config is not None + assert len(context.skill_config.mcp_servers) == count + + +@then("the skill config should have {count:d} agent skill folders") +def step_then_agent_folder_count(context: Context, count: int) -> None: + """Assert the number of agent skill folders.""" + assert context.skill_config is not None + assert len(context.skill_config.agent_skill_folders) == count + + +@then("the skill config tools list should be empty") +def step_then_tools_empty(context: Context) -> None: + """Assert tools list is empty.""" + assert context.skill_config is not None + assert context.skill_config.tools == [] + + +@then("the skill config inline_tools list should be empty") +def step_then_inline_tools_empty(context: Context) -> None: + """Assert inline_tools list is empty.""" + assert context.skill_config is not None + assert context.skill_config.inline_tools == [] + + +@then("the skill config includes list should be empty") +def step_then_includes_empty(context: Context) -> None: + """Assert includes list is empty.""" + assert context.skill_config is not None + assert context.skill_config.includes == [] + + +@then("the skill config mcp_servers list should be empty") +def step_then_mcp_servers_empty(context: Context) -> None: + """Assert mcp_servers list is empty.""" + assert context.skill_config is not None + assert context.skill_config.mcp_servers == [] + + +@then("the skill config agent_skill_folders list should be empty") +def step_then_agent_folders_empty(context: Context) -> None: + """Assert agent_skill_folders list is empty.""" + assert context.skill_config is not None + assert context.skill_config.agent_skill_folders == [] + + +@then('skill tool {idx:d} name should be "{expected}"') +def step_then_tool_name(context: Context, idx: int, expected: str) -> None: + """Assert a tool reference name by index.""" + assert context.skill_config is not None + assert context.skill_config.tools[idx].name == expected + + +@then('skill tool {idx:d} description should be "{expected}"') +def step_then_tool_description(context: Context, idx: int, expected: str) -> None: + """Assert a tool reference description by index.""" + assert context.skill_config is not None + assert context.skill_config.tools[idx].description == expected + + +@then("skill tool {idx:d} writes should be true") +def step_then_tool_writes_true(context: Context, idx: int) -> None: + """Assert a tool reference writes flag is True.""" + assert context.skill_config is not None + assert context.skill_config.tools[idx].writes is True + + +@then("skill tool {idx:d} checkpointable should be false") +def step_then_tool_checkpointable_false(context: Context, idx: int) -> None: + """Assert a tool reference checkpointable flag is False.""" + assert context.skill_config is not None + assert context.skill_config.tools[idx].checkpointable is False + + +@then('skill inline tool {idx:d} name should be "{expected}"') +def step_then_inline_name(context: Context, idx: int, expected: str) -> None: + """Assert an inline tool name by index.""" + assert context.skill_config is not None + assert context.skill_config.inline_tools[idx].name == expected + + +@then('skill inline tool {idx:d} source should be "{expected}"') +def step_then_inline_source(context: Context, idx: int, expected: str) -> None: + """Assert an inline tool source by index.""" + assert context.skill_config is not None + assert context.skill_config.inline_tools[idx].source == expected + + +@then("skill inline tool {idx:d} writes should be true") +def step_then_inline_writes_true(context: Context, idx: int) -> None: + """Assert an inline tool writes flag is True.""" + assert context.skill_config is not None + assert context.skill_config.inline_tools[idx].writes is True + + +@then("skill inline tool {idx:d} checkpointable should be true") +def step_then_inline_checkpointable_true(context: Context, idx: int) -> None: + """Assert an inline tool checkpointable flag is True.""" + assert context.skill_config is not None + assert context.skill_config.inline_tools[idx].checkpointable is True + + +@then('skill mcp server {idx:d} name should be "{expected}"') +def step_then_mcp_name(context: Context, idx: int, expected: str) -> None: + """Assert an MCP server name by index.""" + assert context.skill_config is not None + assert context.skill_config.mcp_servers[idx].name == expected + + +@then('skill mcp server {idx:d} transport should be "{expected}"') +def step_then_mcp_transport(context: Context, idx: int, expected: str) -> None: + """Assert an MCP server transport by index.""" + assert context.skill_config is not None + assert context.skill_config.mcp_servers[idx].transport == expected + + +@then('skill mcp server {idx:d} command should be "{expected}"') +def step_then_mcp_command(context: Context, idx: int, expected: str) -> None: + """Assert an MCP server command by index.""" + assert context.skill_config is not None + assert context.skill_config.mcp_servers[idx].command == expected + + +@then('skill mcp server {idx:d} url should be "{expected}"') +def step_then_mcp_url(context: Context, idx: int, expected: str) -> None: + """Assert an MCP server url by index.""" + assert context.skill_config is not None + assert context.skill_config.mcp_servers[idx].url == expected + + +@then("skill mcp server {idx:d} tool filter include should have {count:d} items") +def step_then_mcp_filter_include_count( + context: Context, idx: int, count: int +) -> None: + """Assert the number of tool filter include entries.""" + assert context.skill_config is not None + mcp = context.skill_config.mcp_servers[idx] + assert mcp.tool_filter is not None + assert len(mcp.tool_filter.include) == count + + +@then("skill mcp server {idx:d} tool filter exclude should have {count:d} items") +def step_then_mcp_filter_exclude_count( + context: Context, idx: int, count: int +) -> None: + """Assert the number of tool filter exclude entries.""" + assert context.skill_config is not None + mcp = context.skill_config.mcp_servers[idx] + assert mcp.tool_filter is not None + assert len(mcp.tool_filter.exclude) == count + + +@then('skill agent folder {idx:d} path should be "{expected}"') +def step_then_agent_folder_path(context: Context, idx: int, expected: str) -> None: + """Assert an agent skill folder path by index.""" + assert context.skill_config is not None + assert context.skill_config.agent_skill_folders[idx].path == expected + + +@then('skill agent folder {idx:d} name should be "{expected}"') +def step_then_agent_folder_name(context: Context, idx: int, expected: str) -> None: + """Assert an agent skill folder name by index.""" + assert context.skill_config is not None + assert context.skill_config.agent_skill_folders[idx].name == expected + + +@then('the skill config model_dump should contain key "{key}"') +def step_then_model_dump_contains_key(context: Context, key: str) -> None: + """Assert the model_dump dict contains a key.""" + assert context.skill_config is not None + data: dict[str, Any] = context.skill_config.model_dump() + assert key in data, f"Key '{key}' not found in model_dump: {list(data.keys())}" + diff --git a/implementation_plan.md b/implementation_plan.md index 1d1cf6bb1..974ea86a4 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -162,23 +162,6 @@ The following work from the previous implementation has been completed and will - `features/action_model.feature` (22 scenarios) - All new tests pass, existing tests unaffected -**2026-02-13**: Stage A2b.gamma Complete (Aditya) - Action YAML Schema + Examples + Loader -- Created `docs/schema/action.schema.yaml` — formal schema definition for action YAML config files, covering all fields (name, description, strategy_actor, execution_actor, definition_of_done, arguments, invariants, automation_profile, inputs_schema, optional actors, behaviour flags, state), versioning, namespaced name patterns, key normalization mapping. -- Created 5 example action YAML configs under `examples/actions/`: simple.yaml, invariant-heavy.yaml, read-only.yaml, estimation-actor.yaml, inputs-schema.yaml. -- Created `src/cleveragents/action/schema.py` with `ActionConfigSchema` Pydantic model: - - `from_yaml(yaml_string)` and `from_yaml_file(path)` factory methods - - camelCase→snake_case key normalization with deprecation warnings - - `${ENV_VAR}` environment variable interpolation - - Invariant normalization: trim whitespace, drop blanks, de-duplicate preserving order - - Clear, actionable error messages for every validation failure mode - - Nested `ActionArgumentSchema` for typed arguments with validation - - `extra="forbid"` prevents unknown keys -- Behave tests: `features/action_schema.feature` — 31 scenarios / 140 steps covering valid schemas, missing fields, invalid names, invalid types, invariant normalization, key normalization, env var interpolation, serialization, edge cases (None/empty/list/directory inputs). -- Robot smoke tests: `robot/action_schema.robot` — 6 test cases validating all example YAMLs + invalid rejection. -- ASV benchmarks: `benchmarks/action_schema_bench.py` — 3 suites (validation, file-load, serialization). -- Coverage: 98% overall (action/schema.py: 98%, action/__init__.py: 100%). -- All nox sessions pass: lint, typecheck, unit_tests, integration_tests, benchmark, security_scan, coverage_report. - **2026-02-05**: Stage A3 Complete - PlanLifecycleService - Created `src/cleveragents/application/services/plan_lifecycle_service.py` with: - Full plan lifecycle management (Action -> Strategize -> Execute -> Apply -> Applied) @@ -796,6 +779,23 @@ The following work from the previous implementation has been completed and will - Verification: lint 0 findings, typecheck 0 errors, 39 new scenarios pass, 97% coverage (session.py 98%) - Branch: `feature/m3-session-domain` (based on `feature/m3-tool-domain`); PR pending +**2026-02-13**: Stage A2b.gamma Complete (Aditya) - Action YAML Schema + Examples + Loader +- Created `docs/schema/action.schema.yaml` — formal schema definition for action YAML config files, covering all fields (name, description, strategy_actor, execution_actor, definition_of_done, arguments, invariants, automation_profile, inputs_schema, optional actors, behaviour flags, state), versioning, namespaced name patterns, key normalization mapping. +- Created 5 example action YAML configs under `examples/actions/`: simple.yaml, invariant-heavy.yaml, read-only.yaml, estimation-actor.yaml, inputs-schema.yaml. +- Created `src/cleveragents/action/schema.py` with `ActionConfigSchema` Pydantic model: + - `from_yaml(yaml_string)` and `from_yaml_file(path)` factory methods + - camelCase→snake_case key normalization with deprecation warnings + - `${ENV_VAR}` environment variable interpolation + - Invariant normalization: trim whitespace, drop blanks, de-duplicate preserving order + - Clear, actionable error messages for every validation failure mode + - Nested `ActionArgumentSchema` for typed arguments with validation + - `extra="forbid"` prevents unknown keys +- Behave tests: `features/action_schema.feature` — 31 scenarios / 140 steps covering valid schemas, missing fields, invalid names, invalid types, invariant normalization, key normalization, env var interpolation, serialization, edge cases (None/empty/list/directory inputs). +- Robot smoke tests: `robot/action_schema.robot` — 6 test cases validating all example YAMLs + invalid rejection. +- ASV benchmarks: `benchmarks/action_schema_bench.py` — 3 suites (validation, file-load, serialization). +- Coverage: 98% overall (action/schema.py: 98%, action/__init__.py: 100%). +- All nox sessions pass: lint, typecheck, unit_tests, integration_tests, benchmark, security_scan, coverage_report. + **2026-02-14**: Stage A4b.action Complete - Action CLI Spec Alignment [Jeff] - Rewrote `src/cleveragents/cli/commands/action.py`: config-only `action create --config `, removed `action available` command, added `--namespace`/`--state`/REGEX filters to `action list`, enhanced output with `Short Name`/`Definition of Done` summary. - Updated 3 existing feature files (action_cli.feature, action_cli_uncovered_lines.feature, action_cli_edge_cases_coverage.feature) to use config-only create flow and remove `available` command references. @@ -829,6 +829,22 @@ The following work from the previous implementation has been completed and will - Verification: lint 0, typecheck 0, 2630 scenarios passed, 97% total coverage. Tool builtins 97-100% coverage. - Branch: `feature/m1-tool-builtins` (based on `feature/m1-tool-runtime-core`) - 2026-02-14: Commit traceability sweep found no matching git log entries for skill schema/examples, skill CLI, provider actors, MCP config/adapter, actor compiler, or skill registry persistence; Section 2B traceability tasks remain open. + +**2026-02-16**: C0.skill.schema Complete (Aditya) - Skill YAML Schema + Examples + Loader +- Created `docs/schema/skill.schema.yaml` — formal schema definition for skill YAML config files, covering all fields (name, description, tools, inline_tools, includes, mcp_servers, agent_skill_folders), versioning, namespaced name patterns, key normalization mapping, additionalProperties: false at all levels. +- Created 5 example skill YAML configs under `examples/skills/`: single-tool.yaml, composed.yaml, inline-tool.yaml, validation-only.yaml, mcp-backed.yaml. +- Created `src/cleveragents/skills/schema.py` with `SkillConfigSchema` Pydantic model: + - `from_yaml(yaml_string)` and `from_yaml_file(path)` factory methods + - camelCase→snake_case key normalization with deprecation warnings + - `${ENV_VAR}` environment variable interpolation + - Clear, actionable error messages for every validation failure mode + - Nested models: `SkillToolRefSchema` (tool refs with overrides), `SkillInlineToolSchema` (name/source/code/input_schema/writes/checkpointable/side_effects), `SkillIncludeSchema`, `SkillMcpServerSchema` (name/transport/command/args/env/url/headers/tool_filter), `SkillMcpToolFilterSchema`, `SkillAgentFolderSchema` + - `extra="forbid"` at all nesting levels to reject unknown keys +- Behave: 36 scenarios / 169 steps in `features/skill_schema.feature` + `features/steps/skill_schema_steps.py` +- Robot: 6 test cases in `robot/skill_schema.robot` + `robot/helper_skill_schema.py` +- ASV: 3 benchmark suites in `benchmarks/skill_schema_bench.py` +- Quality: lint 0 findings, typecheck 0 errors, all Behave/Robot tests pass. + --- ## Roadmap @@ -1425,15 +1441,6 @@ No standalone Q0-Advanced commits planned. Advanced QA enhancements are bundled - [X] Git [Aditya]: `git checkout master` - [X] Git [Aditya]: `git branch -d feature/m1-action-schema` - [X] Quality [Aditya]: Verify coverage >=97% via `nox -s coverage_report`. Coverage result: 98% total (action/schema.py: 98%, action/__init__.py: 100%). - **Notes (A2b.gamma)**: - - Schema: `docs/schema/action.schema.yaml` — full field definitions with types, patterns, defaults, constraints. - - Examples: 5 configs under `examples/actions/` — simple, invariant-heavy, read-only, estimation-actor, inputs-schema. - - Validation helper: `src/cleveragents/action/schema.py` — Pydantic `ActionConfigSchema` model with `from_yaml()` and `from_yaml_file()` factories. - - Features: camelCase→snake_case key normalization with deprecation warnings, `${ENV_VAR}` interpolation, invariant trim/dedup/drop-blanks, clear error messages for every validation failure. - - Behave tests: 31 scenarios / 140 steps in `features/action_schema.feature`. - - Robot smoke: 6 test cases in `robot/action_schema.robot` (5 valid examples + 1 invalid rejection). - - ASV benchmarks: 3 suites (validation, file-load, serialization) in `benchmarks/action_schema_bench.py`. - - All nox sessions pass: lint ✓, typecheck (0 errors) ✓, unit_tests (2222 scenarios) ✓, integration_tests (208 passed) ✓, benchmark ✓, security_scan ✓, coverage (98%) ✓. **Parallel Group A2c: Plan Phase + Apply State Rebaseline (M1-critical)** **PARALLEL SUBTRACK A2c.alpha [Jeff]**: Plan phase/state alignment + CLI output updates diff --git a/robot/helper_skill_schema.py b/robot/helper_skill_schema.py new file mode 100644 index 000000000..0f49d8404 --- /dev/null +++ b/robot/helper_skill_schema.py @@ -0,0 +1,56 @@ +"""Robot Framework helper for skill YAML schema validation. + +Provides a CLI-style interface for Robot to invoke schema validation +on skill YAML files. Exit code 0 = success, 1 = failure. + +Usage: + python robot/helper_skill_schema.py validate + python robot/helper_skill_schema.py validate-invalid +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from cleveragents.skills.schema import SkillConfigSchema # noqa: E402 + + +def main() -> int: + """Entry point called by Robot Framework ``Run Process``.""" + if len(sys.argv) < 3: + print("Usage: helper_skill_schema.py ") + return 1 + + command = sys.argv[1] + yaml_path = sys.argv[2] + + if command == "validate": + try: + config = SkillConfigSchema.from_yaml_file(yaml_path) + print(f"skill-schema-ok: {config.name}") + return 0 + except Exception as exc: + print(f"skill-schema-fail: {exc}") + return 1 + + if command == "validate-invalid": + try: + SkillConfigSchema.from_yaml_file(yaml_path) + print("skill-schema-unexpected-success") + return 1 + except Exception as exc: + print(f"skill-schema-expected-fail: {exc}") + return 0 + + print(f"Unknown command: {command}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/robot/skill_schema.robot b/robot/skill_schema.robot new file mode 100644 index 000000000..0e4d7457c --- /dev/null +++ b/robot/skill_schema.robot @@ -0,0 +1,51 @@ +*** Settings *** +Documentation Smoke tests for Skill YAML schema validation +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_skill_schema.py + +*** Test Cases *** +Validate Single-Tool Skill YAML + [Documentation] Parse the single-tool example skill YAML and assert success + ${result}= Run Process ${PYTHON} ${HELPER} validate ${CURDIR}/../examples/skills/single-tool.yaml cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} skill-schema-ok + +Validate Composed Skill YAML + [Documentation] Parse the composed example skill YAML and assert success + ${result}= Run Process ${PYTHON} ${HELPER} validate ${CURDIR}/../examples/skills/composed.yaml cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} skill-schema-ok + +Validate Inline-Tool Skill YAML + [Documentation] Parse the inline-tool example skill YAML and assert success + ${result}= Run Process ${PYTHON} ${HELPER} validate ${CURDIR}/../examples/skills/inline-tool.yaml cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} skill-schema-ok + +Validate Validation-Only Skill YAML + [Documentation] Parse the validation-only example skill YAML and assert success + ${result}= Run Process ${PYTHON} ${HELPER} validate ${CURDIR}/../examples/skills/validation-only.yaml cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} skill-schema-ok + +Validate MCP-Backed Skill YAML + [Documentation] Parse the MCP-backed example skill YAML and assert success + ${result}= Run Process ${PYTHON} ${HELPER} validate ${CURDIR}/../examples/skills/mcp-backed.yaml cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} skill-schema-ok + +Reject Invalid Skill YAML + [Documentation] Attempt to parse an intentionally invalid YAML and assert failure + ${invalid_yaml}= Set Variable ${TEMPDIR}${/}invalid_skill.yaml + Create File ${invalid_yaml} description: missing required fields\n + ${result}= Run Process ${PYTHON} ${HELPER} validate-invalid ${invalid_yaml} cwd=${WORKSPACE} + Log ${result.stdout} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} skill-schema-expected-fail + diff --git a/src/cleveragents/skills/__init__.py b/src/cleveragents/skills/__init__.py new file mode 100644 index 000000000..560408f48 --- /dev/null +++ b/src/cleveragents/skills/__init__.py @@ -0,0 +1,16 @@ +"""Skill YAML configuration schema and validation. + +This package provides the ``SkillConfigSchema`` Pydantic model for +loading, validating, and normalizing skill YAML configuration files. + +Usage:: + + from cleveragents.skills.schema import SkillConfigSchema + + config = SkillConfigSchema.from_yaml_file("examples/skills/single-tool.yaml") +""" + +from cleveragents.skills.schema import SkillConfigSchema + +__all__ = ["SkillConfigSchema"] + diff --git a/src/cleveragents/skills/schema.py b/src/cleveragents/skills/schema.py new file mode 100644 index 000000000..3a2cdefa9 --- /dev/null +++ b/src/cleveragents/skills/schema.py @@ -0,0 +1,529 @@ +"""Skill YAML configuration schema, validation, and loading. + +Provides :class:`SkillConfigSchema`, a Pydantic model that: + +* Loads raw YAML (string or file) and validates it against the skill schema. +* Normalizes camelCase keys to snake_case before validation. +* Interpolates ``${ENV_VAR}`` placeholders from environment variables. +* Produces clear, actionable error messages for every validation failure. + +Schema definition lives in ``docs/schema/skill.schema.yaml``. +Example configs live under ``examples/skills/``. + +Based on: +- docs/specification.md — Skill Configuration Files + JSON Schema +- implementation_plan.md — Task C0.skill.schema +""" + +from __future__ import annotations + +import logging +import os +import re +from pathlib import Path +from typing import Any + +import yaml +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_validator, +) + +logger = logging.getLogger(__name__) + +# ──────────────────────────────────────────────────────────── +# Constants +# ──────────────────────────────────────────────────────────── + +#: Pattern for ``/`` with hyphens, underscores, alphanum. +NAMESPACED_NAME_RE = re.compile( + r"^[a-zA-Z0-9][a-zA-Z0-9_-]*/[a-zA-Z0-9][a-zA-Z0-9_-]*$" +) + +#: Pattern for ``${VAR}`` environment variable references. +_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + +#: Supported MCP transport protocols. +_VALID_TRANSPORTS = frozenset({"stdio", "sse", "streamable-http"}) + +#: camelCase → snake_case mapping for top-level and nested keys. +_CAMEL_TO_SNAKE: dict[str, str] = { + "inlineTools": "inline_tools", + "mcpServers": "mcp_servers", + "agentSkillFolders": "agent_skill_folders", + "inputSchema": "input_schema", + "sideEffects": "side_effects", + "toolFilter": "tool_filter", +} + + +# ──────────────────────────────────────────────────────────── +# Nested models +# ──────────────────────────────────────────────────────────── + + +class SkillToolRefSchema(BaseModel): + """Schema for a tool reference within a skill. + + References a named tool from the Tool Registry by its namespaced name. + """ + + name: str = Field( + ..., + min_length=1, + description="Fully qualified name of a tool in the Tool Registry.", + ) + description: str | None = Field( + default=None, + description="Override the tool's registered description.", + ) + writes: bool | None = Field( + default=None, + description="Override the tool's writes capability flag.", + ) + checkpointable: bool | None = Field( + default=None, + description="Override the tool's checkpointable capability flag.", + ) + + @field_validator("name") + @classmethod + def validate_namespaced_name(cls, v: str) -> str: + """Validate that the tool name follows namespace/name pattern.""" + if not NAMESPACED_NAME_RE.match(v): + raise ValueError( + f"Invalid tool reference name '{v}'. " + "Names must follow the namespace/name format " + "(e.g. 'builtin/read_file'). " + "Only alphanumeric, hyphens, and underscores are allowed." + ) + return v + + model_config = ConfigDict( + str_strip_whitespace=True, + extra="forbid", + ) + + +class SkillInlineToolSchema(BaseModel): + """Schema for an anonymous inline tool definition within a skill. + + Inline tools are not registered in the Tool Registry and exist + only within the scope of the skill that defines them. + """ + + name: str = Field( + ..., + min_length=1, + description="Tool name, unique within this skill.", + ) + description: str | None = Field( + default=None, + description="Human-readable description of the tool's purpose.", + ) + source: str = Field( + ..., + description="Source type. Must be 'custom' for inline tools.", + ) + code: str = Field( + ..., + min_length=1, + description="Python code defining the tool's behavior.", + ) + input_schema: dict[str, Any] | None = Field( + default=None, + description="JSON Schema describing the tool's input parameters.", + ) + writes: bool = Field( + False, + description="Whether this tool performs write operations.", + ) + checkpointable: bool = Field( + False, + description="Whether this tool supports checkpointing.", + ) + side_effects: list[str] = Field( + default_factory=list, + description="Descriptions of side effects.", + ) + + @field_validator("source") + @classmethod + def validate_source(cls, v: str) -> str: + """Validate that source is 'custom'.""" + if v != "custom": + raise ValueError( + f"Invalid inline tool source '{v}'. " + "Inline tools must have source: custom." + ) + return v + + model_config = ConfigDict( + str_strip_whitespace=True, + extra="forbid", + ) + + +class SkillIncludeSchema(BaseModel): + """Schema for an included skill reference. + + When a skill includes another, the included skill's tools are + merged into the including skill's tool set. + """ + + name: str = Field( + ..., + min_length=1, + description="Fully qualified skill name to include.", + ) + description: str | None = Field( + default=None, + description="Override the included skill's description.", + ) + + @field_validator("name") + @classmethod + def validate_namespaced_name(cls, v: str) -> str: + """Validate that the include name follows namespace/name pattern.""" + if not NAMESPACED_NAME_RE.match(v): + raise ValueError( + f"Invalid include name '{v}'. " + "Names must follow the namespace/name format " + "(e.g. 'local/file-reader'). " + "Only alphanumeric, hyphens, and underscores are allowed." + ) + return v + + model_config = ConfigDict( + str_strip_whitespace=True, + extra="forbid", + ) + + +class SkillMcpToolFilterSchema(BaseModel): + """Schema for MCP server tool filtering. + + Controls which tools from the MCP server are exposed. + """ + + include: list[str] = Field( + default_factory=list, + description="Whitelist of tool names to expose.", + ) + exclude: list[str] = Field( + default_factory=list, + description="Blacklist of tool names to hide.", + ) + + model_config = ConfigDict( + str_strip_whitespace=True, + extra="forbid", + ) + + +class SkillMcpServerSchema(BaseModel): + """Schema for an MCP server specification within a skill. + + MCP servers expose remote tools via stdio, SSE, or streamable-http + transport protocols. + """ + + name: str = Field( + ..., + min_length=1, + description="Identifier for the MCP server.", + ) + transport: str = Field( + ..., + description="Transport protocol: stdio | sse | streamable-http.", + ) + command: str | None = Field( + default=None, + description="Command to start the server (required for stdio).", + ) + args: list[str] = Field( + default_factory=list, + description="Command-line arguments for the server command.", + ) + env: dict[str, str] = Field( + default_factory=dict, + description="Environment variables for the server.", + ) + url: str | None = Field( + default=None, + description="Server URL (required for sse/streamable-http).", + ) + headers: dict[str, str] = Field( + default_factory=dict, + description="HTTP headers for remote server connections.", + ) + tool_filter: SkillMcpToolFilterSchema | None = Field( + default=None, + description="Filter which tools to expose.", + ) + + @field_validator("transport") + @classmethod + def validate_transport(cls, v: str) -> str: + """Validate transport is a supported protocol.""" + v_lower = v.lower() + if v_lower not in _VALID_TRANSPORTS: + valid = ", ".join(sorted(_VALID_TRANSPORTS)) + raise ValueError( + f"Invalid MCP transport '{v}'. " + f"Allowed transports: {valid}." + ) + return v_lower + + model_config = ConfigDict( + str_strip_whitespace=True, + extra="forbid", + ) + + +class SkillAgentFolderSchema(BaseModel): + """Schema for an Agent Skills Standard folder reference. + + Points to a directory containing a SKILL.md file that defines + an Agent Skills Standard tool bundle. + """ + + path: str = Field( + ..., + min_length=1, + description="Path to the folder containing SKILL.md.", + ) + name: str | None = Field( + default=None, + description="Override the skill bundle name.", + ) + + model_config = ConfigDict( + str_strip_whitespace=True, + extra="forbid", + ) + + +# ──────────────────────────────────────────────────────────── +# Main schema model +# ──────────────────────────────────────────────────────────── + + +class SkillConfigSchema(BaseModel): + """Pydantic model for a skill YAML configuration file. + + Validates all fields described in ``docs/schema/skill.schema.yaml``. + + Create instances via the factory class methods: + - :meth:`from_yaml` — parse a YAML string + - :meth:`from_yaml_file` — parse a YAML file from disk + """ + + # ─── Identity ─────────────────────────────────────────── + name: str = Field( + ..., + description="Namespaced skill name in namespace/name format.", + ) + description: str | None = Field( + default=None, + description="Human-readable description of what this skill provides.", + ) + + # ─── Tool References ──────────────────────────────────── + tools: list[SkillToolRefSchema] = Field( + default_factory=list, + description="References to named tools from the Tool Registry.", + ) + + # ─── Inline Tools ─────────────────────────────────────── + inline_tools: list[SkillInlineToolSchema] = Field( + default_factory=list, + description="Anonymous tool definitions within this skill.", + ) + + # ─── Includes ─────────────────────────────────────────── + includes: list[SkillIncludeSchema] = Field( + default_factory=list, + description="Other skills whose tools are merged in.", + ) + + # ─── MCP Servers ──────────────────────────────────────── + mcp_servers: list[SkillMcpServerSchema] = Field( + default_factory=list, + description="MCP server specifications.", + ) + + # ─── Agent Skill Folders ──────────────────────────────── + agent_skill_folders: list[SkillAgentFolderSchema] = Field( + default_factory=list, + description="Agent Skills Standard folder references.", + ) + + # ─── Pydantic Config ─────────────────────────────────── + model_config = ConfigDict( + str_strip_whitespace=True, + extra="forbid", + ) + + # ──────────────────────────────────────────────────────── + # Field validators + # ──────────────────────────────────────────────────────── + + @field_validator("name") + @classmethod + def validate_namespaced_name(cls, v: str) -> str: + """Validate that name follows namespace/name pattern.""" + if not NAMESPACED_NAME_RE.match(v): + raise ValueError( + f"Invalid skill name '{v}'. " + "Names must follow the namespace/name format " + "(e.g. 'local/file-reader'). " + "Only alphanumeric, hyphens, and underscores are allowed." + ) + return v + + # ──────────────────────────────────────────────────────── + # Factory class methods + # ──────────────────────────────────────────────────────── + + @classmethod + def from_yaml(cls, yaml_string: str) -> SkillConfigSchema: + """Parse and validate a skill YAML string. + + Args: + yaml_string: Raw YAML content. + + Returns: + Validated ``SkillConfigSchema`` instance. + + Raises: + ValueError: If the YAML is not a mapping or is empty. + pydantic.ValidationError: If schema validation fails. + """ + if yaml_string is None: + raise ValueError( + "YAML string cannot be None. Provide a valid YAML string." + ) + if not yaml_string.strip(): + raise ValueError( + "YAML string is empty. Provide a valid skill YAML configuration." + ) + + raw = yaml.safe_load(yaml_string) + if not isinstance(raw, dict): + raise ValueError( + f"Skill YAML must be a mapping (key: value), " + f"got {type(raw).__name__}." + ) + + normalized = _normalize_keys(raw) + interpolated = _interpolate_env_vars(normalized) + return cls.model_validate(interpolated) + + @classmethod + def from_yaml_file(cls, path: str | Path) -> SkillConfigSchema: + """Load and validate a skill YAML file from disk. + + Args: + path: Path to the YAML file. + + Returns: + Validated ``SkillConfigSchema`` instance. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If file content is invalid. + pydantic.ValidationError: If schema validation fails. + """ + if path is None: + raise ValueError( + "File path cannot be None. " + "Provide a valid path to a skill YAML file." + ) + + filepath = Path(path) + if not filepath.exists(): + raise FileNotFoundError( + f"Skill YAML file not found: {filepath}. " + "Check the file path and try again." + ) + if not filepath.is_file(): + raise ValueError( + f"Path is not a file: {filepath}. " + "Provide a path to a YAML file, not a directory." + ) + + content = filepath.read_text(encoding="utf-8") + return cls.from_yaml(content) + + +# ──────────────────────────────────────────────────────────── +# Internal helpers +# ──────────────────────────────────────────────────────────── + + +def _normalize_keys(data: dict[str, Any]) -> dict[str, Any]: + """Normalize camelCase keys to snake_case recursively. + + Logs a warning for every camelCase key that is converted. + """ + result: dict[str, Any] = {} + for key, value in data.items(): + snake_key = _CAMEL_TO_SNAKE.get(key, key) + if snake_key != key: + logger.warning( + "Deprecated camelCase key '%s' normalized to '%s'. " + "Please update your YAML to use snake_case.", + key, + snake_key, + ) + + if isinstance(value, dict): + result[snake_key] = _normalize_keys(value) + elif isinstance(value, list): + result[snake_key] = [ + _normalize_keys(item) if isinstance(item, dict) else item + for item in value + ] + else: + result[snake_key] = value + return result + + +def _interpolate_env_vars(data: dict[str, Any]) -> dict[str, Any]: + """Replace ``${VAR}`` references with environment variable values. + + Only string values are interpolated. Missing environment variables + are left as-is (no error) to allow deferred resolution. + """ + result: dict[str, Any] = {} + for key, value in data.items(): + if isinstance(value, str): + result[key] = _ENV_VAR_RE.sub(_env_replacer, value) + elif isinstance(value, dict): + result[key] = _interpolate_env_vars(value) + elif isinstance(value, list): + result[key] = [ + ( + _interpolate_env_vars(item) + if isinstance(item, dict) + else ( + _ENV_VAR_RE.sub(_env_replacer, item) + if isinstance(item, str) + else item + ) + ) + for item in value + ] + else: + result[key] = value + return result + + +def _env_replacer(match: re.Match[str]) -> str: + """Replace a single ``${VAR}`` match with its env value.""" + var_name = match.group(1) + return os.environ.get(var_name, match.group(0)) +