"""Step definitions for skill flattening and capability summary tests. All step text patterns are prefixed with ``flatten`` to avoid collisions with existing step definitions in the Behave step registry. """ from __future__ import annotations from typing import Any from behave import given, then, when from behave.runner import Context from cleveragents.domain.models.core.skill import ( ResolvedToolEntry, Skill, SkillCapabilitySummary, SkillInclude, SkillInlineTool, SkillResolver, ) from cleveragents.domain.models.core.tool import ToolCapability, ToolSource from cleveragents.skills.protocol import SkillDefinition, SkillMetadata from cleveragents.skills.registry import SkillRegistry # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _ensure_flatten_registry(context: Context) -> dict[str, Skill]: """Return (and lazily create) the flatten skill lookup dict.""" if not hasattr(context, "flatten_registry"): context.flatten_registry = {} return context.flatten_registry # type: ignore[no-any-return] def _make_skill_definition(skill: Skill) -> SkillDefinition: """Build a ``SkillDefinition`` wrapping *skill* with default metadata.""" resolver = SkillResolver() resolved = resolver.resolve_tools(skill, {}) meta = SkillMetadata.from_skill(skill, resolved=resolved) return SkillDefinition( skill=skill, resolved_tools=resolved, metadata=meta, ) # --------------------------------------------------------------------------- # Given steps (all prefixed with "flatten") # --------------------------------------------------------------------------- @given('a flatten registry skill "{name}" with tool refs "{refs}"') def flatten_given_skill_refs(context: Context, name: str, refs: str) -> None: """Register a skill with tool refs in the flatten lookup.""" registry = _ensure_flatten_registry(context) ref_list = [r.strip() for r in refs.split(",") if r.strip()] registry[name] = Skill( name=name, description=f"Skill {name}", tool_refs=ref_list, ) @given('a flatten registry skill "{name}" including "{inc}" with refs "{refs}"') def flatten_given_skill_include_refs( context: Context, name: str, inc: str, refs: str ) -> None: """Register a skill with one include and own tool refs.""" registry = _ensure_flatten_registry(context) ref_list = [r.strip() for r in refs.split(",") if r.strip()] includes = [SkillInclude(name=i.strip()) for i in inc.split(",") if i.strip()] registry[name] = Skill( name=name, description=f"Skill {name}", tool_refs=ref_list, includes=includes, ) @given('a flatten registry skill "{name}" with {count:d} inline tools') def flatten_given_skill_inline(context: Context, name: str, count: int) -> None: """Register a skill with *count* inline tools.""" registry = _ensure_flatten_registry(context) inlines = [ SkillInlineTool( description=f"Inline tool {i}", source=ToolSource.CUSTOM, code=f"return {i}", ) for i in range(count) ] registry[name] = Skill( name=name, description=f"Skill {name}", anonymous_tools=inlines, ) @given('a flatten registry skill "{name}" including "{inc}" with no refs') def flatten_given_skill_include_no_refs(context: Context, name: str, inc: str) -> None: """Register a skill that includes another with no own refs.""" registry = _ensure_flatten_registry(context) includes = [SkillInclude(name=i.strip()) for i in inc.split(",") if i.strip()] registry[name] = Skill( name=name, description=f"Skill {name}", includes=includes, ) @given( 'a flatten registry skill "{name}" including "{inc}"' " with overrides timeout {timeout:d}" ) def flatten_given_skill_include_overrides( context: Context, name: str, inc: str, timeout: int ) -> None: """Register a skill that includes another with per-include overrides.""" registry = _ensure_flatten_registry(context) includes = [ SkillInclude( name=inc.strip(), overrides={"timeout": timeout}, ) ] registry[name] = Skill( name=name, description=f"Skill {name}", includes=includes, ) @given( 'a flatten registry skill "{name}" including "{inc}"' ' with overrides on non-overridable field "{field}"' ) def flatten_given_skill_include_bad_override( context: Context, name: str, inc: str, field: str ) -> None: """Register a skill whose include overrides a non-overridable field.""" registry = _ensure_flatten_registry(context) includes = [ SkillInclude( name=inc.strip(), overrides={field: "forbidden-value"}, ) ] registry[name] = Skill( name=name, description=f"Skill {name}", includes=includes, ) @given('a flatten registry skill "{name}" with mixed capability inline tools') def flatten_given_skill_capability_mix(context: Context, name: str) -> None: """Register a skill with inline tools having diverse capabilities.""" registry = _ensure_flatten_registry(context) inlines = [ SkillInlineTool( description="Read-only tool", source=ToolSource.CUSTOM, code="pass", capability=ToolCapability(read_only=True), ), SkillInlineTool( description="Write tool with side effects", source=ToolSource.CUSTOM, code="pass", capability=ToolCapability( writes=True, side_effects=["filesystem"], ), ), SkillInlineTool( description="Checkpointable tool", source=ToolSource.CUSTOM, code="pass", capability=ToolCapability(checkpointable=True), ), ] registry[name] = Skill( name=name, description=f"Skill {name}", anonymous_tools=inlines, ) @given('a flatten skill registry with skill "{name}" having {count:d} refs') def flatten_given_full_registry(context: Context, name: str, count: int) -> None: """Create a full SkillRegistry and register a skill with *count* refs.""" refs = [f"local/tool-{i}" for i in range(count)] skill = Skill(name=name, description=f"Skill {name}", tool_refs=refs) defn = _make_skill_definition(skill) reg = SkillRegistry() reg.register(defn) context.flatten_skill_registry = reg @given("an empty flatten skill registry") def flatten_given_empty_registry(context: Context) -> None: """Create an empty SkillRegistry.""" context.flatten_skill_registry = SkillRegistry() @given("the flatten registry skills are registered in the skill registry") def flatten_register_all_in_skill_registry(context: Context) -> None: """Register all skills from the flatten lookup into a SkillRegistry. Uses minimal metadata (no resolution) because the skill graph may contain cycles that would fail resolution. """ registry = _ensure_flatten_registry(context) sr = SkillRegistry() for skill in registry.values(): meta = SkillMetadata( name=skill.name, description=skill.description, ) defn = SkillDefinition( skill=skill, resolved_tools=[], metadata=meta, ) sr.register(defn) context.flatten_skill_registry = sr @given( 'a flatten registry skill "{name}" including "{inc}"' " with overrides priority {prio:d} and extra key" ) def flatten_given_skill_include_merge_overrides( context: Context, name: str, inc: str, prio: int ) -> None: """Register a skill whose include overrides have multiple keys.""" registry = _ensure_flatten_registry(context) includes = [ SkillInclude( name=inc.strip(), overrides={"priority": prio, "extra": "yes"}, ) ] registry[name] = Skill( name=name, description=f"Skill {name}", includes=includes, ) @given( 'a flatten registry skill "{name}" with tool refs "{refs}"' " and overrides priority {prio:d}" ) def flatten_given_skill_refs_with_overrides( context: Context, name: str, refs: str, prio: int ) -> None: """Register a skill with tool refs that have skill-level overrides.""" registry = _ensure_flatten_registry(context) ref_list = [r.strip() for r in refs.split(",") if r.strip()] overrides: dict[str, dict[str, Any]] = {} for ref in ref_list: overrides[ref] = {"priority": prio} registry[name] = Skill( name=name, description=f"Skill {name}", tool_refs=ref_list, overrides=overrides, ) # --------------------------------------------------------------------------- # When steps # --------------------------------------------------------------------------- @when('I flatten the skill "{name}"') def flatten_when_resolve(context: Context, name: str) -> None: """Resolve/flatten a skill from the flatten registry.""" registry = _ensure_flatten_registry(context) skill = registry[name] resolver = SkillResolver() context.flatten_result = resolver.resolve_tools(skill, registry) context.flatten_error = None @when('I try to flatten the skill "{name}"') def flatten_when_try_resolve(context: Context, name: str) -> None: """Try to flatten; capture errors.""" registry = _ensure_flatten_registry(context) skill = registry[name] resolver = SkillResolver() context.flatten_result = None context.flatten_error = None try: context.flatten_result = resolver.resolve_tools(skill, registry) except ValueError as exc: context.flatten_error = exc @when('I flatten the skill "{name}" and compute summary') def flatten_when_resolve_summary(context: Context, name: str) -> None: """Flatten a skill and compute the capability summary.""" registry = _ensure_flatten_registry(context) skill = registry[name] resolver = SkillResolver() entries = resolver.resolve_tools(skill, registry) context.flatten_result = entries context.flatten_summary = resolver.compute_capability_summary(skill, entries) @when('I call the flatten tools method for "{name}"') def flatten_when_tools_method(context: Context, name: str) -> None: """Call SkillRegistry.tools() on a registered skill.""" sr: SkillRegistry = context.flatten_skill_registry entries, summary = sr.tools(name) context.flatten_tools_entries = entries context.flatten_tools_summary = summary @when('I call flatten validate_plan with skills "{skills_csv}"') def flatten_when_validate_plan(context: Context, skills_csv: str) -> None: """Call SkillRegistry.validate_plan().""" sr: SkillRegistry = context.flatten_skill_registry skill_list = [s.strip() for s in skills_csv.split(",") if s.strip()] context.flatten_validation_errors = sr.validate_plan({"skills": skill_list}) # --------------------------------------------------------------------------- # Then steps # --------------------------------------------------------------------------- @then("the flatten result should have {count:d} entries") def flatten_then_count(context: Context, count: int) -> None: """Check flattened entry count.""" actual = len(context.flatten_result) assert actual == count, ( f"Expected {count} entries, got {actual}: " f"{[e.name for e in context.flatten_result]}" ) @then('the flatten entry at index {idx:d} should be "{expected}"') def flatten_then_entry_at(context: Context, idx: int, expected: str) -> None: """Check entry name at a given index.""" entries: list[ResolvedToolEntry] = context.flatten_result assert idx < len(entries), f"Index {idx} out of range ({len(entries)} entries)" actual = entries[idx].name assert actual == expected, ( f"Expected entry at {idx} to be '{expected}', got '{actual}'" ) @then("the flatten entry at index {idx:d} should be marked inline") def flatten_then_entry_inline(context: Context, idx: int) -> None: """Check that entry at index is inline.""" entries: list[ResolvedToolEntry] = context.flatten_result assert entries[idx].is_inline is True, ( f"Expected inline at index {idx}, got is_inline=False" ) @then("a flatten cycle error should be raised") def flatten_then_cycle_error(context: Context) -> None: """Verify a cycle error was raised.""" assert context.flatten_error is not None, "Expected cycle error" assert "ycle" in str(context.flatten_error), ( f"Expected cycle error, got: {context.flatten_error}" ) @then('the flatten error should mention "{text}"') def flatten_then_error_mention(context: Context, text: str) -> None: """Check the error message contains text.""" msg = str(context.flatten_error) assert text in msg, f"Expected '{text}' in error: {msg}" @then('the flatten entry "{name}" should have override "{key}" equal to {value:d}') def flatten_then_override_int( context: Context, name: str, key: str, value: int ) -> None: """Check that a resolved entry has a specific int override.""" entries: list[ResolvedToolEntry] = context.flatten_result for entry in entries: if entry.name == name: assert key in entry.overrides, ( f"Override '{key}' not found on '{name}': {entry.overrides}" ) assert entry.overrides[key] == value, ( f"Expected override '{key}'={value}, got {entry.overrides[key]}" ) return raise AssertionError(f"Entry '{name}' not found: {[e.name for e in entries]}") @then('the flatten entry "{name}" should have override "{key}" equal to "{value}"') def flatten_then_override_str( context: Context, name: str, key: str, value: str ) -> None: """Check that a resolved entry has a specific string override.""" entries: list[ResolvedToolEntry] = context.flatten_result for entry in entries: if entry.name == name: assert key in entry.overrides, ( f"Override '{key}' not found on '{name}': {entry.overrides}" ) assert entry.overrides[key] == value, ( f"Expected override '{key}'='{value}', got {entry.overrides[key]}" ) return raise AssertionError(f"Entry '{name}' not found: {[e.name for e in entries]}") @then("a flatten override error should be raised") def flatten_then_override_error(context: Context) -> None: """Verify a non-overridable field error was raised.""" assert context.flatten_error is not None, ( "Expected override error but no error was raised" ) @then("the flatten summary total_tools should be {expected:d}") def flatten_then_summary_total(context: Context, expected: int) -> None: """Check summary total_tools.""" summary: SkillCapabilitySummary = context.flatten_summary assert summary.total_tools == expected, ( f"Expected total_tools={expected}, got {summary.total_tools}" ) @then("the flatten summary read_only_tools should be {expected:d}") def flatten_then_summary_readonly(context: Context, expected: int) -> None: """Check summary read_only_tools.""" summary: SkillCapabilitySummary = context.flatten_summary assert summary.read_only_tools == expected @then("the flatten summary write_tools should be {expected:d}") def flatten_then_summary_writes(context: Context, expected: int) -> None: """Check summary write_tools.""" summary: SkillCapabilitySummary = context.flatten_summary assert summary.write_tools == expected @then("the flatten summary has_side_effects should be true") def flatten_then_summary_side_effects(context: Context) -> None: """Check summary has_side_effects.""" summary: SkillCapabilitySummary = context.flatten_summary assert summary.has_side_effects is True @then("the flatten summary checkpointable_tools should be {expected:d}") def flatten_then_summary_checkpoint(context: Context, expected: int) -> None: """Check summary checkpointable_tools.""" summary: SkillCapabilitySummary = context.flatten_summary assert summary.checkpointable_tools == expected @then("the flatten tools result should contain {count:d} entries") def flatten_then_tools_count(context: Context, count: int) -> None: """Check tools() returned correct entry count.""" actual = len(context.flatten_tools_entries) assert actual == count, f"Expected {count} entries, got {actual}" @then("the flatten tools result should include a capability summary") def flatten_then_tools_summary(context: Context) -> None: """Check tools() returned a SkillCapabilitySummary.""" summary = context.flatten_tools_summary assert isinstance(summary, SkillCapabilitySummary), ( f"Expected SkillCapabilitySummary, got {type(summary)}" ) @then("the flatten validation result should have {count:d} errors") def flatten_then_validation_count(context: Context, count: int) -> None: """Check validate_plan error count.""" actual = len(context.flatten_validation_errors) assert actual == count, ( f"Expected {count} errors, got {actual}: {context.flatten_validation_errors}" ) @then('the flatten validation error should mention "{text}"') def flatten_then_validation_mention(context: Context, text: str) -> None: """Check that at least one validation error mentions text.""" errors: list[str] = context.flatten_validation_errors assert any(text in e for e in errors), f"Expected '{text}' in errors: {errors}" @then('the flatten entry "{name}" source_skill should be "{expected}"') def flatten_then_entry_source(context: Context, name: str, expected: str) -> None: """Check which skill contributed a tool entry.""" entries: list[ResolvedToolEntry] = context.flatten_result for entry in entries: if entry.name == name: assert entry.source_skill == expected, ( f"Expected source_skill='{expected}', got '{entry.source_skill}'" ) return raise AssertionError(f"Entry '{name}' not found: {[e.name for e in entries]}")