Files
temp/features/steps/skill_add_include_validation_steps.py
freemo 1f94b7e8d5 fix(skills): validate included skills are registered before adding skill
- Implemented a pre-registration validation in SkillService.add_skill() to ensure all skills listed in the includes field are already registered in self._skills before accepting the new skill registration. This prevents registering a skill that depends on non-existent skills.
- The validation applies to both the initial add and when update=True (re-registration), ensuring dependencies are consistently enforced across both creation and update flows.

- If any included skills are missing, a clear ValueError is raised listing all missing skill names with guidance to register them first, providing complete visibility rather than failing on the first missing item.

- Added new Behave feature file features/skill_add_include_validation.feature with 8 scenarios covering: single missing include, multiple missing includes, partial includes, all includes registered (success), update with missing include, and service-level assertions. Also added step definitions in features/steps/skill_add_include_validation_steps.py.

- Updated features/skill_cli.feature to pre-register local/file-reader in scenarios that use the full skill YAML (which includes local/file-reader) to reflect the new validation behavior.

- Updated features/steps/skill_cli_coverage_r3_steps.py to pre-register local/new-include in the update-with-different-includes scenario to align with the updated validation flow.

- Rationale and design decisions:
  - Validation is placed in the service layer (SkillService.add_skill()) rather than the CLI layer to ensure consistent enforcement regardless of how the service is invoked.
  - The error message reports all missing includes at once for better user guidance and faster remediation.
  - Tests that relied on the full skill YAML were updated to pre-register dependencies to accurately reflect the validation changes and maintain test integrity.

ISSUES CLOSED: #2555
2026-04-05 04:36:52 +00:00

227 lines
7.3 KiB
Python

"""Step definitions for skill_add_include_validation.feature.
Tests that ``agents skill add`` validates included skills are registered
before accepting a new skill registration (issue #2555).
"""
from __future__ import annotations
import os
import tempfile
from behave import given, then
from behave.runner import Context
from cleveragents.cli.commands.skill import (
_get_skill_service,
)
from cleveragents.cli.commands.skill import app as skill_app # noqa: F401
from cleveragents.skills.schema import SkillConfigSchema
# ---------------------------------------------------------------------------
# YAML fixtures
# ---------------------------------------------------------------------------
_SINGLE_MISSING_INCLUDE_YAML = """\
name: local/needs-missing-dep
description: "Skill that includes an unregistered skill"
tools:
- name: builtin/read_file
includes:
- name: local/missing-dep
"""
_TWO_MISSING_INCLUDES_YAML = """\
name: local/needs-two-deps
description: "Skill that includes two unregistered skills"
tools:
- name: builtin/read_file
includes:
- name: local/dep-a
- name: local/dep-b
"""
_PARTIAL_INCLUDES_YAML = """\
name: local/needs-partial-deps
description: "Skill that includes one registered and one unregistered skill"
tools:
- name: builtin/read_file
includes:
- name: local/dep-registered
- name: local/dep-missing
"""
_VALID_INCLUDE_YAML = """\
name: local/needs-base-tools
description: "Skill that includes a registered skill"
tools:
- name: builtin/read_file
includes:
- name: local/base-tools
"""
_UPDATE_WITH_MISSING_INCLUDE_YAML = """\
name: local/composable
description: "Updated composable skill with unregistered include"
tools:
- name: builtin/read_file
includes:
- name: local/not-there
"""
_PARTIAL_REGISTERED_AND_MISSING_YAML = """\
name: local/needs-partial-deps
description: "Skill that includes one registered and one unregistered skill"
tools:
- name: builtin/read_file
includes:
- name: local/dep-registered
- name: local/dep-missing
"""
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _write_temp_yaml(context: Context, content: str) -> str:
"""Write YAML content to a temporary file and register cleanup."""
fd, path = tempfile.mkstemp(suffix=".yaml")
with os.fdopen(fd, "w") as fh:
fh.write(content)
if not hasattr(context, "_skill_cleanup"):
context._skill_cleanup = []
context._skill_cleanup.append(path)
return path
# ---------------------------------------------------------------------------
# Given steps
# ---------------------------------------------------------------------------
@given('a skill config YAML that includes "local/missing-dep" at a temp path')
def step_yaml_single_missing_include(context: Context) -> None:
"""Write a skill YAML that includes one unregistered skill."""
context.skill_yaml_path = _write_temp_yaml(context, _SINGLE_MISSING_INCLUDE_YAML)
@given(
'a skill config YAML that includes "local/dep-a" and "local/dep-b" at a temp path'
)
def step_yaml_two_missing_includes(context: Context) -> None:
"""Write a skill YAML that includes two unregistered skills."""
context.skill_yaml_path = _write_temp_yaml(context, _TWO_MISSING_INCLUDES_YAML)
@given(
'a skill config YAML that includes "local/dep-registered" and "local/dep-missing" at a temp path'
)
def step_yaml_partial_includes(context: Context) -> None:
"""Write a skill YAML that includes one registered and one unregistered skill."""
context.skill_yaml_path = _write_temp_yaml(
context, _PARTIAL_REGISTERED_AND_MISSING_YAML
)
@given('a skill config YAML that includes "local/base-tools" at a temp path')
def step_yaml_valid_include(context: Context) -> None:
"""Write a skill YAML that includes a registered skill."""
context.skill_yaml_path = _write_temp_yaml(context, _VALID_INCLUDE_YAML)
@given(
'a skill config YAML for "local/composable" that includes "local/not-there" at a temp path'
)
def step_yaml_update_with_missing_include(context: Context) -> None:
"""Write a skill YAML for updating an existing skill with an unregistered include."""
context.skill_yaml_path = _write_temp_yaml(
context, _UPDATE_WITH_MISSING_INCLUDE_YAML
)
# ---------------------------------------------------------------------------
# Then steps — service-level assertions
# ---------------------------------------------------------------------------
@then("adding a skill with unregistered include via service raises ValueError")
def step_service_raises_for_unregistered_include(context: Context) -> None:
"""Assert SkillService.add_skill raises ValueError when include is missing."""
service = _get_skill_service()
schema = SkillConfigSchema(
name="local/test-validation",
description="Test skill",
includes=[{"name": "local/does-not-exist"}],
)
try:
service.add_skill(schema)
raise AssertionError(
"Expected ValueError for unregistered include, but no exception was raised"
)
except ValueError as exc:
msg = str(exc)
assert "not registered" in msg, (
f"Expected 'not registered' in error message, got: {msg}"
)
assert "local/does-not-exist" in msg, (
f"Expected missing skill name in error message, got: {msg}"
)
@then(
"adding a skill with two unregistered includes via service raises ValueError listing both"
)
def step_service_raises_listing_both_missing(context: Context) -> None:
"""Assert ValueError lists all missing included skills."""
service = _get_skill_service()
schema = SkillConfigSchema(
name="local/test-multi-validation",
description="Test skill",
includes=[
{"name": "local/missing-one"},
{"name": "local/missing-two"},
],
)
try:
service.add_skill(schema)
raise AssertionError(
"Expected ValueError for unregistered includes, but no exception was raised"
)
except ValueError as exc:
msg = str(exc)
assert "local/missing-one" in msg, (
f"Expected 'local/missing-one' in error message, got: {msg}"
)
assert "local/missing-two" in msg, (
f"Expected 'local/missing-two' in error message, got: {msg}"
)
@then('updating "{name}" with unregistered include via service raises ValueError')
def step_service_update_raises_for_unregistered_include(
context: Context, name: str
) -> None:
"""Assert SkillService.add_skill with update=True also validates includes."""
service = _get_skill_service()
schema = SkillConfigSchema(
name=name,
description="Updated skill",
includes=[{"name": "local/unregistered-dep"}],
)
try:
service.add_skill(schema, update=True)
raise AssertionError(
"Expected ValueError for unregistered include on update, "
"but no exception was raised"
)
except ValueError as exc:
msg = str(exc)
assert "not registered" in msg, (
f"Expected 'not registered' in error message, got: {msg}"
)
assert "local/unregistered-dep" in msg, (
f"Expected missing skill name in error message, got: {msg}"
)