forked from HAL9000/cleveragents-core
1f94b7e8d5
- Implemented a pre-registration validation in SkillService.add_skill() to ensure all skills listed in the includes field are already registered in self._skills before accepting the new skill registration. This prevents registering a skill that depends on non-existent skills. - The validation applies to both the initial add and when update=True (re-registration), ensuring dependencies are consistently enforced across both creation and update flows. - If any included skills are missing, a clear ValueError is raised listing all missing skill names with guidance to register them first, providing complete visibility rather than failing on the first missing item. - Added new Behave feature file features/skill_add_include_validation.feature with 8 scenarios covering: single missing include, multiple missing includes, partial includes, all includes registered (success), update with missing include, and service-level assertions. Also added step definitions in features/steps/skill_add_include_validation_steps.py. - Updated features/skill_cli.feature to pre-register local/file-reader in scenarios that use the full skill YAML (which includes local/file-reader) to reflect the new validation behavior. - Updated features/steps/skill_cli_coverage_r3_steps.py to pre-register local/new-include in the update-with-different-includes scenario to align with the updated validation flow. - Rationale and design decisions: - Validation is placed in the service layer (SkillService.add_skill()) rather than the CLI layer to ensure consistent enforcement regardless of how the service is invoked. - The error message reports all missing includes at once for better user guidance and faster remediation. - Tests that relied on the full skill YAML were updated to pre-register dependencies to accurately reflect the validation changes and maintain test integrity. ISSUES CLOSED: #2555
227 lines
7.3 KiB
Python
227 lines
7.3 KiB
Python
"""Step definitions for skill_add_include_validation.feature.
|
|
|
|
Tests that ``agents skill add`` validates included skills are registered
|
|
before accepting a new skill registration (issue #2555).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
|
|
from behave import given, then
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.cli.commands.skill import (
|
|
_get_skill_service,
|
|
)
|
|
from cleveragents.cli.commands.skill import app as skill_app # noqa: F401
|
|
from cleveragents.skills.schema import SkillConfigSchema
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# YAML fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_SINGLE_MISSING_INCLUDE_YAML = """\
|
|
name: local/needs-missing-dep
|
|
description: "Skill that includes an unregistered skill"
|
|
tools:
|
|
- name: builtin/read_file
|
|
includes:
|
|
- name: local/missing-dep
|
|
"""
|
|
|
|
_TWO_MISSING_INCLUDES_YAML = """\
|
|
name: local/needs-two-deps
|
|
description: "Skill that includes two unregistered skills"
|
|
tools:
|
|
- name: builtin/read_file
|
|
includes:
|
|
- name: local/dep-a
|
|
- name: local/dep-b
|
|
"""
|
|
|
|
_PARTIAL_INCLUDES_YAML = """\
|
|
name: local/needs-partial-deps
|
|
description: "Skill that includes one registered and one unregistered skill"
|
|
tools:
|
|
- name: builtin/read_file
|
|
includes:
|
|
- name: local/dep-registered
|
|
- name: local/dep-missing
|
|
"""
|
|
|
|
_VALID_INCLUDE_YAML = """\
|
|
name: local/needs-base-tools
|
|
description: "Skill that includes a registered skill"
|
|
tools:
|
|
- name: builtin/read_file
|
|
includes:
|
|
- name: local/base-tools
|
|
"""
|
|
|
|
_UPDATE_WITH_MISSING_INCLUDE_YAML = """\
|
|
name: local/composable
|
|
description: "Updated composable skill with unregistered include"
|
|
tools:
|
|
- name: builtin/read_file
|
|
includes:
|
|
- name: local/not-there
|
|
"""
|
|
|
|
_PARTIAL_REGISTERED_AND_MISSING_YAML = """\
|
|
name: local/needs-partial-deps
|
|
description: "Skill that includes one registered and one unregistered skill"
|
|
tools:
|
|
- name: builtin/read_file
|
|
includes:
|
|
- name: local/dep-registered
|
|
- name: local/dep-missing
|
|
"""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _write_temp_yaml(context: Context, content: str) -> str:
|
|
"""Write YAML content to a temporary file and register cleanup."""
|
|
fd, path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(content)
|
|
if not hasattr(context, "_skill_cleanup"):
|
|
context._skill_cleanup = []
|
|
context._skill_cleanup.append(path)
|
|
return path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a skill config YAML that includes "local/missing-dep" at a temp path')
|
|
def step_yaml_single_missing_include(context: Context) -> None:
|
|
"""Write a skill YAML that includes one unregistered skill."""
|
|
context.skill_yaml_path = _write_temp_yaml(context, _SINGLE_MISSING_INCLUDE_YAML)
|
|
|
|
|
|
@given(
|
|
'a skill config YAML that includes "local/dep-a" and "local/dep-b" at a temp path'
|
|
)
|
|
def step_yaml_two_missing_includes(context: Context) -> None:
|
|
"""Write a skill YAML that includes two unregistered skills."""
|
|
context.skill_yaml_path = _write_temp_yaml(context, _TWO_MISSING_INCLUDES_YAML)
|
|
|
|
|
|
@given(
|
|
'a skill config YAML that includes "local/dep-registered" and "local/dep-missing" at a temp path'
|
|
)
|
|
def step_yaml_partial_includes(context: Context) -> None:
|
|
"""Write a skill YAML that includes one registered and one unregistered skill."""
|
|
context.skill_yaml_path = _write_temp_yaml(
|
|
context, _PARTIAL_REGISTERED_AND_MISSING_YAML
|
|
)
|
|
|
|
|
|
@given('a skill config YAML that includes "local/base-tools" at a temp path')
|
|
def step_yaml_valid_include(context: Context) -> None:
|
|
"""Write a skill YAML that includes a registered skill."""
|
|
context.skill_yaml_path = _write_temp_yaml(context, _VALID_INCLUDE_YAML)
|
|
|
|
|
|
@given(
|
|
'a skill config YAML for "local/composable" that includes "local/not-there" at a temp path'
|
|
)
|
|
def step_yaml_update_with_missing_include(context: Context) -> None:
|
|
"""Write a skill YAML for updating an existing skill with an unregistered include."""
|
|
context.skill_yaml_path = _write_temp_yaml(
|
|
context, _UPDATE_WITH_MISSING_INCLUDE_YAML
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then steps — service-level assertions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("adding a skill with unregistered include via service raises ValueError")
|
|
def step_service_raises_for_unregistered_include(context: Context) -> None:
|
|
"""Assert SkillService.add_skill raises ValueError when include is missing."""
|
|
service = _get_skill_service()
|
|
schema = SkillConfigSchema(
|
|
name="local/test-validation",
|
|
description="Test skill",
|
|
includes=[{"name": "local/does-not-exist"}],
|
|
)
|
|
try:
|
|
service.add_skill(schema)
|
|
raise AssertionError(
|
|
"Expected ValueError for unregistered include, but no exception was raised"
|
|
)
|
|
except ValueError as exc:
|
|
msg = str(exc)
|
|
assert "not registered" in msg, (
|
|
f"Expected 'not registered' in error message, got: {msg}"
|
|
)
|
|
assert "local/does-not-exist" in msg, (
|
|
f"Expected missing skill name in error message, got: {msg}"
|
|
)
|
|
|
|
|
|
@then(
|
|
"adding a skill with two unregistered includes via service raises ValueError listing both"
|
|
)
|
|
def step_service_raises_listing_both_missing(context: Context) -> None:
|
|
"""Assert ValueError lists all missing included skills."""
|
|
service = _get_skill_service()
|
|
schema = SkillConfigSchema(
|
|
name="local/test-multi-validation",
|
|
description="Test skill",
|
|
includes=[
|
|
{"name": "local/missing-one"},
|
|
{"name": "local/missing-two"},
|
|
],
|
|
)
|
|
try:
|
|
service.add_skill(schema)
|
|
raise AssertionError(
|
|
"Expected ValueError for unregistered includes, but no exception was raised"
|
|
)
|
|
except ValueError as exc:
|
|
msg = str(exc)
|
|
assert "local/missing-one" in msg, (
|
|
f"Expected 'local/missing-one' in error message, got: {msg}"
|
|
)
|
|
assert "local/missing-two" in msg, (
|
|
f"Expected 'local/missing-two' in error message, got: {msg}"
|
|
)
|
|
|
|
|
|
@then('updating "{name}" with unregistered include via service raises ValueError')
|
|
def step_service_update_raises_for_unregistered_include(
|
|
context: Context, name: str
|
|
) -> None:
|
|
"""Assert SkillService.add_skill with update=True also validates includes."""
|
|
service = _get_skill_service()
|
|
schema = SkillConfigSchema(
|
|
name=name,
|
|
description="Updated skill",
|
|
includes=[{"name": "local/unregistered-dep"}],
|
|
)
|
|
try:
|
|
service.add_skill(schema, update=True)
|
|
raise AssertionError(
|
|
"Expected ValueError for unregistered include on update, "
|
|
"but no exception was raised"
|
|
)
|
|
except ValueError as exc:
|
|
msg = str(exc)
|
|
assert "not registered" in msg, (
|
|
f"Expected 'not registered' in error message, got: {msg}"
|
|
)
|
|
assert "local/unregistered-dep" in msg, (
|
|
f"Expected missing skill name in error message, got: {msg}"
|
|
)
|